<commit_before>import numpy as np
import math
import random
import itertools
import collections

class NeuralNet:
    def __init__(self, n_inputs, n_outputs, n_hiddens, n_hidden_layers, activation_functions ):
        self.n_inputs             = n_inputs             # Number of network input signals
        self.n_outputs            = n_outputs            # Number of desired outputs from the network
        self.n_hiddens            = n_hiddens            # Number of nodes in each hidden layer
        self.n_hidden_layers      = n_hidden_layers      # Number of hidden layers in the network
        self.activation_functions = activation_functions

        assert len(activation_functions) == (n_hidden_layers + 1), \
            "Expected {n_expected} activation functions, but was initialized with {n_received}.".format(
                n_expected = (n_hidden_layers + 1),
                n_received = len(activation_functions)
            )

        if n_hidden_layers == 0:
            # Count the necessary number of weights for the input->output connection.
            # input -> [] -> output
            self.n_weights = (n_inputs + 1) * n_outputs
        else:
            # Count the necessary number of weights summed over all the layers.
            # input -> [n_hiddens -> n_hiddens] -> output
            self.n_weights = (n_inputs + 1) * n_hiddens +\
                             (n_hiddens**2 + n_hiddens) * (n_hidden_layers - 1) +\
                             (n_hiddens * n_outputs) + n_outputs

        # Initialize the network with new randomized weights
        self.set_weights( self.generate_weights() )
    #end

    def generate_weights(self, low = -0.1, high = 0.1):
        # Generate new random weights for all the connections in the network
        if not False:
            # Support NumPy
            return [random.uniform(low,high) for _ in xrange(self.n_weights)]
        else:
            return np.random.uniform(low, high, size=(1,self.n_weights)).tolist()[0]
    #end

    def unpack(self, weight_list ):
        # This method will create a list of weight matrices. Each list element
        # corresponds to the connection between two layers.
        if self.n_hidden_layers == 0:
            return [ np.array(weight_list).reshape(self.n_inputs+1,self.n_outputs) ]
        else:
            weight_layers = [ np.array(weight_list[:(self.n_inputs+1)*self.n_hiddens]).reshape(self.n_inputs+1,self.n_hiddens) ]
            weight_layers += [ np.array(weight_list[(self.n_inputs+1)*self.n_hiddens+(i*(self.n_hiddens**2+self.n_hiddens)):(self.n_inputs+1)*self.n_hiddens+((i+1)*(self.n_hiddens**2+self.n_hiddens))]).reshape(self.n_hiddens+1,self.n_hiddens) for i in xrange(self.n_hidden_layers-1) ]
            weight_layers += [ np.array(weight_list[(self.n_inputs+1)*self.n_hiddens+((self.n_hidden_layers-1)*(self.n_hiddens**2+self.n_hiddens)):]).reshape(self.n_hiddens+1,self.n_outputs) ]

        return weight_layers
    #end

    def set_weights(self, weight_list ):
        # This is a helper method for setting the network weights to a previously defined list.
        # This is useful for utilizing a previously optimized neural network weight set.
        self.weights = self.unpack( weight_list )
    #end

    def get_weights(self, ):
        # This will stack all the weights in the network on a list, which may be saved to the disk.
        return [w for l in self.weights for w in l.flat]
    #end

    def backpropagation(self, trainingset, ERROR_LIMIT = 1e-3, learning_rate = 0.3, momentum_factor = 0.9  ):
        def addBias(A):
            return np.hstack(( np.ones((A.shape[0],1)), A )) # Add 1 as bias.
        #end addBias

        assert trainingset[0].features.shape[0] == self.n_inputs, \
                "ERROR: input size varies from the defined input setting"

        assert trainingset[0].targets.shape[0]  == self.n_outputs, \
                "ERROR: output size varies from the defined output setting"

        training_data    = np.array( [instance.features for instance in trainingset ] )
        training_targets = np.array( [instance.targets for instance in trainingset ] )

        MSE      = ( ) # inf
        neterror = None
        momentum = collections.defaultdict( int )

        epoch    = 0
        while MSE > ERROR_LIMIT:
            epoch += 1

            input_layers      = self.update( training_data, trace=True )
            out               = input_layers[-1]

            error             = training_targets - out
            delta             = error
            MSE               = np.mean( np.power(error,2) )

            loop  = itertools.izip(
                            xrange(len(self.weights)-1, -1, -1),
                            reversed(self.weights),
                            reversed(input_layers[:-1]),
                        )

            for i, weight_layer, input_signals in loop:
                # Loop over the weight layers in reversed order to calculate the deltas

                # Calculate weight change
                dW = learning_rate * np.dot( addBias(input_signals).T, delta ) + momentum_factor * momentum[i]

                if i!= 0:
                    """Do not calculate the delta unnecessarily."""
                    # Skipping the bias weight during calculation.
                    weight_delta = np.dot( delta, weight_layer[1:,:].T )

                    # Calculate the delta for the subsequent layer
                    delta = np.multiply(  weight_delta, self.activation_functions[i-1]( input_signals, derivative=True) )

                # Store the momentum
                momentum[i] = dW

                # Update the weights
                self.weights[ i ] += dW

            if epoch%1000==0:
                # Show the current training status
                print "* current network error (MSE):", MSE

        print "* Converged to error bound (%.4g) with MSE = %.4g." % ( ERROR_LIMIT, MSE )
        print "* Trained for %d epochs." % epoch
    # end backprop

    def update(self, input_values, trace=False ):
        # This is a forward operation in the network. This is how we calculate the network output
        # from a set of input signals.

        output = input_values
        if trace: tracelist = [ output ]

        for i, weight_layer in enumerate(self.weights):
            # Loop over the network layers and calculate the output
            output = np.dot( output, weight_layer[1:,:] ) + weight_layer[0:1,:] # implicit bias
            output = self.activation_functions[i]( output )
            if trace: tracelist.append( output )

        if trace: return tracelist

        return output
    #end

    def save_to_file(self, filename = "network.pkl" ):
        import cPickle
        """
        This save method pickles the parameters of the current network into a
        binary file for persistant storage.
        """
        with open( filename , 'wb') as file:
            store_dict = {
                "n_inputs"             : self.n_inputs,
                "n_outputs"            : self.n_outputs,
                "n_hiddens"            : self.n_hiddens,
                "n_hidden_layers"      : self.n_hidden_layers,
                "activation_functions" : self.activation_functions,
                "n_weights"            : self.n_weights,
                "weights"              : self.weights

            }
            cPickle.dump( store_dict, file, 2 )
    #end

    @staticmethod
    def load_from_file( filename = "network.pkl" ):
        """
        Load the complete configuration of a previously stored network.
        """
        network = NeuralNet( 0, 0, 0, 0, [0] )

        with open( filename , 'rb') as file:
            import cPickle
            store_dict = cPickle.load(file)

            network.n_inputs             = store_dict["n_inputs"]
            network.n_outputs            = store_dict["n_outputs"]
            network.n_hiddens            = store_dict["n_hiddens"]
            network.n_hidden_layers      = store_dict["n_hidden_layers"]
            network.n_weights            = store_dict["n_weights"]
            network.weights              = store_dict["weights"]
            network.activation_functions = store_dict["activation_functions"]

        return network
    #end
#end class<commit_msg>swap signs to conform with the norm<commit_after>import math
import random
import itertools
import collections

PYPY = False
try:
    import numpypy as np
    PYPY = True
except:
    import numpy as np

class NeuralNet:
    def __init__(self, n_inputs, n_outputs, n_hiddens, n_hidden_layers, activation_functions ):
        self.n_inputs             = n_inputs             # Number of network input signals
        self.n_outputs            = n_outputs            # Number of desired outputs from the network
        self.n_hiddens            = n_hiddens            # Number of nodes in each hidden layer
        self.n_hidden_layers      = n_hidden_layers      # Number of hidden layers in the network
        self.activation_functions = activation_functions

        assert len(activation_functions) == (n_hidden_layers + 1), \
            "Expected {n_expected} activation functions, but was initialized with {n_received}.".format(
                n_expected = (n_hidden_layers + 1),
                n_received = len(activation_functions)
            )

        if n_hidden_layers == 0:
            # Count the necessary number of weights for the input->output connection.
            # input -> [] -> output
            self.n_weights = (n_inputs + 1) * n_outputs
        else:
            # Count the necessary number of weights summed over all the layers.
            # input -> [n_hiddens -> n_hiddens] -> output
            self.n_weights = (n_inputs + 1) * n_hiddens +\
                             (n_hiddens**2 + n_hiddens) * (n_hidden_layers - 1) +\
                             (n_hiddens * n_outputs) + n_outputs

        # Initialize the network with new randomized weights
        self.set_weights( self.generate_weights() )
    #end

    def generate_weights(self, low = -0.1, high = 0.1):
        # Generate new random weights for all the connections in the network
        if not False:
            # Support NumPy
            return [random.uniform(low,high) for _ in xrange(self.n_weights)]
        else:
            return np.random.uniform(low, high, size=(1,self.n_weights)).tolist()[0]
    #end

    def unpack(self, weight_list ):
        # This method will create a list of weight matrices. Each list element
        # corresponds to the connection between two layers.
        if self.n_hidden_layers == 0:
            return [ np.array(weight_list).reshape(self.n_inputs+1,self.n_outputs) ]
        else:
            weight_layers = [ np.array(weight_list[:(self.n_inputs+1)*self.n_hiddens]).reshape(self.n_inputs+1,self.n_hiddens) ]
            weight_layers += [ np.array(weight_list[(self.n_inputs+1)*self.n_hiddens+(i*(self.n_hiddens**2+self.n_hiddens)):(self.n_inputs+1)*self.n_hiddens+((i+1)*(self.n_hiddens**2+self.n_hiddens))]).reshape(self.n_hiddens+1,self.n_hiddens) for i in xrange(self.n_hidden_layers-1) ]
            weight_layers += [ np.array(weight_list[(self.n_inputs+1)*self.n_hiddens+((self.n_hidden_layers-1)*(self.n_hiddens**2+self.n_hiddens)):]).reshape(self.n_hiddens+1,self.n_outputs) ]

        return weight_layers
    #end

    def set_weights(self, weight_list ):
        # This is a helper method for setting the network weights to a previously defined list.
        # This is useful for utilizing a previously optimized neural network weight set.
        self.weights = self.unpack( weight_list )
    #end

    def get_weights(self, ):
        # This will stack all the weights in the network on a list, which may be saved to the disk.
        return [w for l in self.weights for w in l.flat]
    #end

    def backpropagation(self, trainingset, ERROR_LIMIT = 1e-3, learning_rate = 0.3, momentum_factor = 0.9  ):
        def addBias(A):
            return np.hstack(( np.ones((A.shape[0],1)), A )) # Add 1 as bias.
        #end addBias

        assert trainingset[0].features.shape[0] == self.n_inputs, \
                "ERROR: input size varies from the defined input setting"

        assert trainingset[0].targets.shape[0]  == self.n_outputs, \
                "ERROR: output size varies from the defined output setting"

        training_data    = np.array( [instance.features for instance in trainingset ] )
        training_targets = np.array( [instance.targets for instance in trainingset ] )

        MSE      = ( ) # inf
        neterror = None
        momentum = collections.defaultdict( int )

        epoch    = 0
        while MSE > ERROR_LIMIT:
            epoch += 1

            input_layers      = self.update( training_data, trace=True )
            out               = input_layers[-1]

            error             = out - training_targets
            delta             = error
            MSE               = np.mean( np.power(error,2) )

            loop  = itertools.izip(
                            xrange(len(self.weights)-1, -1, -1),
                            reversed(self.weights),
                            reversed(input_layers[:-1]),
                        )

            for i, weight_layer, input_signals in loop:
                # Loop over the weight layers in reversed order to calculate the deltas

                # Calculate weight change
                dW = learning_rate * np.dot( addBias(input_signals).T, delta ) + momentum_factor * momentum[i]

                if i!= 0:
                    """Do not calculate the delta unnecessarily."""
                    # Skipping the bias weight during calculation.
                    weight_delta = np.dot( delta, weight_layer[1:,:].T )

                    # Calculate the delta for the subsequent layer
                    delta = np.multiply(  weight_delta, self.activation_functions[i-1]( input_signals, derivative=True) )

                # Store the momentum
                momentum[i] = dW

                # Update the weights
                self.weights[ i ] -= dW

            if epoch%1000==0:
                # Show the current training status
                print "* current network error (MSE):", MSE

        print "* Converged to error bound (%.4g) with MSE = %.4g." % ( ERROR_LIMIT, MSE )
        print "* Trained for %d epochs." % epoch
    # end backprop

    def update(self, input_values, trace=False ):
        # This is a forward operation in the network. This is how we calculate the network output
        # from a set of input signals.

        output = input_values
        if trace: tracelist = [ output ]

        for i, weight_layer in enumerate(self.weights):
            # Loop over the network layers and calculate the output
            output = np.dot( output, weight_layer[1:,:] ) + weight_layer[0:1,:] # implicit bias
            output = self.activation_functions[i]( output )
            if trace: tracelist.append( output )

        if trace: return tracelist

        return output
    #end

    def save_to_file(self, filename = "network.pkl" ):
        import cPickle
        """
        This save method pickles the parameters of the current network into a
        binary file for persistant storage.
        """
        with open( filename , 'wb') as file:
            store_dict = {
                "n_inputs"             : self.n_inputs,
                "n_outputs"            : self.n_outputs,
                "n_hiddens"            : self.n_hiddens,
                "n_hidden_layers"      : self.n_hidden_layers,
                "activation_functions" : self.activation_functions,
                "n_weights"            : self.n_weights,
                "weights"              : self.weights

            }
            cPickle.dump( store_dict, file, 2 )
    #end

    @staticmethod
    def load_from_file( filename = "network.pkl" ):
        """
        Load the complete configuration of a previously stored network.
        """
        network = NeuralNet( 0, 0, 0, 0, [0] )

        with open( filename , 'rb') as file:
            import cPickle
            store_dict = cPickle.load(file)

            network.n_inputs             = store_dict["n_inputs"]
            network.n_outputs            = store_dict["n_outputs"]
            network.n_hiddens            = store_dict["n_hiddens"]
            network.n_hidden_layers      = store_dict["n_hidden_layers"]
            network.n_weights            = store_dict["n_weights"]
            network.weights              = store_dict["weights"]
            network.activation_functions = store_dict["activation_functions"]

        return network
    #end
#end class<|endoftext|>


<commit_before>#!/usr/bin/env python
# Created by Oleg Ivanov

#Ref:

# sample usage: python filebackup.py -s path/source/dirname -d path/destination/dirname
# -i
# -s

import os, sys, getopt, logging
import smtplib
from smtplib import SMTPException
from email.mime.text import MIMEText
import time
from datetime import datetime
import subprocess
from subprocess import PIPE
import shutil
#from os import listdir
import glob
from os.path import isfile, join

#SOURCE_PATH = ""
#DESTINATION_PATH = ""
MAILER_HOST = ""
#MAILER_PORT = ""
MAILER_USERNAME = ""
#MAILER_PASSWORD = ""

COMMAND_COUNTER = 0

def help():
    print(
        "Usage: python filebackup.py [OPTION]...\n" \
        "\n" \
        "-s, --source           path to the source directory\n" \
        "-b, --basedir          directory to archive\n" \
        "-d, --dest             path to the destination directory\n" \
        "-k, --keepcount        number of files to keep in backup destination\n" \
        "-h, --mailerhost       mailerhost\n" \
        "-u, --maileruser       maileruser\n" \
        "-H, --help             this help"
    )

#python filesbackup.py -s test -d myarchive
#python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\backup' -d myarchive -b test -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\orderflex\public' -d 'C:\Users\ch3\Documents\myarchive' -b 'Uploaded' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#Testing: python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\orderflex\public' -d 'C:\Users\ch3\Documents\myarchive' -b 'docs' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#python filesbackup.py -s '/opt/order-lab/orderflex/public' -d /mnt/pathology/view-test-backup/uploadsarchive -b 'Uploaded' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
def start_backup(source, dest, basedir):
    print("source=",source,", dest=",dest,", basedir=",basedir)
    #logging.info("get_site_status: url="+url)

    #https://docs.python.org/3/library/shutil.html#shutil.make_archive
    #>> > from shutil import make_archive
    #>> > import os
    #>> > archive_name = os.path.expanduser(os.path.join('~', 'myarchive'))
    #>> > root_dir = os.path.expanduser(os.path.join('~', '.ssh'))
    #>> > make_archive(archive_name, 'gztar', root_dir)
    #'/Users/tarek/myarchive.tar.gz'
    #archive_name = os.path.expanduser(os.path.join('~', 'myarchive'))

    now = datetime.now()
    #append now.hour
    dest = dest + "_"  + str(now.hour)

    archivefile = ''
    try:
        if basedir != None:
            archivefile = shutil.make_archive(dest, 'gztar', source, base_dir=basedir)
        else:
            archivefile = shutil.make_archive(dest, 'gztar', source)
    except Exception as error:
        print("Error archiving: ",error)
        return error

    #print('archivefile=',archivefile)

    return None

#https://janakiev.com/blog/python-shell-commands/
#https://stackoverflow.com/questions/89228/how-do-i-execute-a-program-or-call-a-system-command
def runCommand(command):
    # try to restart the server
    print("run: " + command)
    #print(os.popen(command).read())
    #output = subprocess.run([command], capture_output=True) #capture_output is for python > 3.7
    output = subprocess.run([command], stdout=PIPE, stderr=PIPE, shell=True)
    print(output)
    # sleep in seconds
    time.sleep(10)

def send_email_alert(mailerhost, fromEmail, toEmailList, emailSubject, emailBody):
    emailBody = emailBody + "\n\n" + datetime.now().strftime('%Y-%B-%d %H:%M:%S')
    msg = MIMEText(emailBody)
    msg['Subject'] = emailSubject
    msg['From'] = fromEmail
    msg['To'] = ', '.join(toEmailList)

    MAILER_PORT = ''

    try:
        #print("MAILER_HOST=" + MAILER_HOST+", MAILER_PORT="+MAILER_PORT)
        smtpObj = smtplib.SMTP(mailerhost, MAILER_PORT)
        #if MAILER_USERNAME != "" and MAILER_PASSWORD != "":
        #    smtpObj.starttls()
        #    smtpObj.login(MAILER_USERNAME, MAILER_PASSWORD)
        smtpObj.sendmail(fromEmail, toEmailList, msg.as_string())
        print("Successfully sent email")
    except SMTPException:
        print("Error: unable to send email")
        #pass

def main(argv):

    print("\n### filesbackup.py "+datetime.now().strftime('%Y-%B-%d %H:%M:%S')+"###")
    #logging.basicConfig(filename='checksites.log',level=logging.INFO)
    #logging.info('main start')

    source = ''           # -s
    basedir = ''          # -b
    dest = ''             # -d
    mailerhost = ''
    maileruser = ''
    receivers = ''  # -r
    fromEmail = ''  # -s
    keepcount = 1

    try:
        opts, args = getopt.getopt(
            argv,
            "s:b:d:h:u:r:f:k:H",
            ["source=", "basedir=", "dest=", "mailerhost=", "maileruser=", "receivers=", "fromemail=", "keepcount=", "help"]
        )
    except getopt.GetoptError:
        print('Parameters error')
        #logging.warning('Parameters error')
        #help()
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-s", "--source"):
            source = arg
            #print('filesbackup.py --urls=' + urls)
        elif opt in ("-b", "--basedir"):
            basedir = arg
        elif opt in ("-d", "--dest"):
            dest = arg
        elif opt in ("-h", "--mailerhost"):             # == "--mailerhost":
            mailerhost = arg
        elif opt in ("-u", "--maileruser"):             # == "--maileruser":
            maileruser = arg
        elif opt in ("-r", "--receivers"):              #Array of the receiver emails
            receivers = arg
        elif opt in ("-f", "--fromemail"):                 #Sender email
            fromEmail = arg
        elif opt in ("-k", "--keepcount"):                 #number of files to keep in backup destination
            keepcount = arg
        elif opt in ("-H", "--help"):
           help()
           #sys.exit()
           return
        else:
            #print('backupfiles.py: invalid option')
            #logging.warning('backupfiles.py: parameter errors')
            help()
            sys.exit(2)

    print('source=',source,', basedir=',basedir, 'dest=',dest, ", mailerhost=",mailerhost,", receivers=",receivers,", fromEmail=",fromEmail,", keepcount=",keepcount)
    #logging.info('urls=' + urls + ', mailerhost=' + mailerhost + ', maileruser=' + maileruser + ', mailerpassword=' + mailerpassword)

    if source == '':
        print('Nothing to do: source is not provided')
        #logging.warning('Nothing to do: source is not provided')
        return

    if basedir == '':
        print('Nothing to do: basedir is not provided')
        return

    if dest == '':
        print('Nothing to do: destination is not provided')
        #logging.warning('Nothing to do: destination is not provided')
        return

    if keepcount == '':
        print('Please provide keepcount - number of files to keep in backup destination. Default keepcount=1')
        keepcount = 1

    toEmailList = ''
    if receivers:
        # receivers is comma separated string of receiver, convert to list
        receivers = receivers.replace(" ", "")
        # receivers is comma separated string of receiver, convert to list
        toEmailList = list(receivers.split(","))

    runCommand('whoami') #testing

    archivefileError = start_backup(source, dest, basedir)

    if archivefileError:
        if mailerhost:
            emailSubject = "Error archiving folder " + basedir
            emailBody = "Error creating archive '" + str(dest) + "' for folder " + str(basedir) + " in " + str(source) + "; Error=" + repr(archivefileError)
            #emailBody = "Error creating archive: " + repr(archivefileError)
            send_email_alert(mailerhost, fromEmail, toEmailList, emailSubject, emailBody)
        else:
            print("Mailer parameters are not provided: Error email has not been sent. Error=",archivefileError)

        print("archivefileError=", archivefileError)
    else:
        print("Archive completed successfully")

    ### remove old files from output directory based on keepcount ###
    #1) take output directory (dest or -d) and get base and filename
    basename, filename = os.path.split(dest)
    print("basename=",basename,", filename=",filename)
    #2) list all files in output directory

    #onlyfiles = [f for f in listdir(basename) if isfile(join(basename, f))]

    # Get list of all files only in the given directory
    onlyfiles = filter(os.path.isfile, glob.glob(basename + '/*'))
    # Sort list of files based on last modification time in ascending order
    onlyfiles = sorted(onlyfiles, key=os.path.getmtime)

    for count, file in enumerate(onlyfiles):
        timestamp_str = time.strftime('%m/%d/%Y :: %H:%M:%S', time.gmtime(os.path.getmtime(file)))
        print("count=",count,", timestamp_str=",timestamp_str, "file=",file)

    if len(onlyfiles) > keepcount:
        lendiff = len(onlyfiles) - int(keepcount)
        for i in range(lendiff):
            print("remove=", onlyfiles[i])
            #os.remove(onlyfiles[i])

    ### EOF remove old files from output directory based on keepcount ###

if __name__ == '__main__':
    #python filesbackup.py -s test -d myarchive

    main(sys.argv[1:])

<commit_msg>testing cron<commit_after>#!/usr/bin/env python
# Created by Oleg Ivanov

#Ref:

# sample usage: python filebackup.py -s path/source/dirname -d path/destination/dirname
# -i
# -s

import os, sys, getopt, logging
import smtplib
from smtplib import SMTPException
from email.mime.text import MIMEText
import time
from datetime import datetime
import subprocess
from subprocess import PIPE
import shutil
#from os import listdir
import glob
from os.path import isfile, join

#SOURCE_PATH = ""
#DESTINATION_PATH = ""
MAILER_HOST = ""
#MAILER_PORT = ""
MAILER_USERNAME = ""
#MAILER_PASSWORD = ""

COMMAND_COUNTER = 0

def help():
    print(
        "Usage: python filebackup.py [OPTION]...\n" \
        "\n" \
        "-s, --source           path to the source directory\n" \
        "-b, --basedir          directory to archive\n" \
        "-d, --dest             path to the destination directory\n" \
        "-k, --keepcount        number of files to keep in backup destination\n" \
        "-h, --mailerhost       mailerhost\n" \
        "-u, --maileruser       maileruser\n" \
        "-H, --help             this help"
    )

#python filesbackup.py -s test -d myarchive
#python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\backup' -d myarchive -b test -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\orderflex\public' -d 'C:\Users\ch3\Documents\myarchive' -b 'Uploaded' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#Testing: python filesbackup.py -s 'C:\Users\ch3\Documents\MyDocs\WCMC\ORDER\order-lab\orderflex\public' -d 'C:\Users\ch3\Documents\myarchive' -b 'docs' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
#python filesbackup.py -s '/opt/order-lab/orderflex/public' -d /mnt/pathology/view-test-backup/uploadsarchive -b 'Uploaded' -h "smtp.med.cornell.edu" -f oli2002@med.cornell.edu -r oli2002@med.cornell.edu
def start_backup(source, dest, basedir):
    print("source=",source,", dest=",dest,", basedir=",basedir)
    #logging.info("get_site_status: url="+url)

    #https://docs.python.org/3/library/shutil.html#shutil.make_archive
    #>> > from shutil import make_archive
    #>> > import os
    #>> > archive_name = os.path.expanduser(os.path.join('~', 'myarchive'))
    #>> > root_dir = os.path.expanduser(os.path.join('~', '.ssh'))
    #>> > make_archive(archive_name, 'gztar', root_dir)
    #'/Users/tarek/myarchive.tar.gz'
    #archive_name = os.path.expanduser(os.path.join('~', 'myarchive'))

    now = datetime.now()
    #append now.hour
    dest = dest + "_"  + str(now.hour)

    archivefile = ''
    try:
        if basedir != None:
            archivefile = shutil.make_archive(dest, 'gztar', source, base_dir=basedir)
        else:
            archivefile = shutil.make_archive(dest, 'gztar', source)
    except Exception as error:
        print("Error archiving: ",error)
        return error

    #print('archivefile=',archivefile)

    return None

#https://janakiev.com/blog/python-shell-commands/
#https://stackoverflow.com/questions/89228/how-do-i-execute-a-program-or-call-a-system-command
def runCommand(command):
    # try to restart the server
    print("run: " + command)
    #print(os.popen(command).read())
    #output = subprocess.run([command], capture_output=True) #capture_output is for python > 3.7
    output = subprocess.run([command], stdout=PIPE, stderr=PIPE, shell=True)
    print(output)
    # sleep in seconds
    time.sleep(10)

def send_email_alert(mailerhost, fromEmail, toEmailList, emailSubject, emailBody):
    emailBody = emailBody + "\n\n" + datetime.now().strftime('%Y-%B-%d %H:%M:%S')
    msg = MIMEText(emailBody)
    msg['Subject'] = emailSubject
    msg['From'] = fromEmail
    msg['To'] = ', '.join(toEmailList)

    MAILER_PORT = ''

    try:
        #print("MAILER_HOST=" + MAILER_HOST+", MAILER_PORT="+MAILER_PORT)
        smtpObj = smtplib.SMTP(mailerhost, MAILER_PORT)
        #if MAILER_USERNAME != "" and MAILER_PASSWORD != "":
        #    smtpObj.starttls()
        #    smtpObj.login(MAILER_USERNAME, MAILER_PASSWORD)
        smtpObj.sendmail(fromEmail, toEmailList, msg.as_string())
        print("Successfully sent email")
    except SMTPException:
        print("Error: unable to send email")
        #pass

def main(argv):

    print("\n### filesbackup.py "+datetime.now().strftime('%Y-%B-%d %H:%M:%S')+"###")
    #logging.basicConfig(filename='checksites.log',level=logging.INFO)
    #logging.info('main start')

    source = ''           # -s
    basedir = ''          # -b
    dest = ''             # -d
    mailerhost = ''
    maileruser = ''
    receivers = ''  # -r
    fromEmail = ''  # -s
    keepcount = 1

    try:
        opts, args = getopt.getopt(
            argv,
            "s:b:d:h:u:r:f:k:H",
            ["source=", "basedir=", "dest=", "mailerhost=", "maileruser=", "receivers=", "fromemail=", "keepcount=", "help"]
        )
    except getopt.GetoptError:
        print('Parameters error')
        #logging.warning('Parameters error')
        #help()
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-s", "--source"):
            source = arg
            #print('filesbackup.py --urls=' + urls)
        elif opt in ("-b", "--basedir"):
            basedir = arg
        elif opt in ("-d", "--dest"):
            dest = arg
        elif opt in ("-h", "--mailerhost"):             # == "--mailerhost":
            mailerhost = arg
        elif opt in ("-u", "--maileruser"):             # == "--maileruser":
            maileruser = arg
        elif opt in ("-r", "--receivers"):              #Array of the receiver emails
            receivers = arg
        elif opt in ("-f", "--fromemail"):                 #Sender email
            fromEmail = arg
        elif opt in ("-k", "--keepcount"):                 #number of files to keep in backup destination
            keepcount = arg
        elif opt in ("-H", "--help"):
           help()
           #sys.exit()
           return
        else:
            #print('backupfiles.py: invalid option')
            #logging.warning('backupfiles.py: parameter errors')
            help()
            sys.exit(2)

    print('source=',source,', basedir=',basedir, 'dest=',dest, ", mailerhost=",mailerhost,", receivers=",receivers,", fromEmail=",fromEmail,", keepcount=",keepcount)
    #logging.info('urls=' + urls + ', mailerhost=' + mailerhost + ', maileruser=' + maileruser + ', mailerpassword=' + mailerpassword)

    if source == '':
        print('Nothing to do: source is not provided')
        #logging.warning('Nothing to do: source is not provided')
        return

    if basedir == '':
        print('Nothing to do: basedir is not provided')
        return

    if dest == '':
        print('Nothing to do: destination is not provided')
        #logging.warning('Nothing to do: destination is not provided')
        return

    if keepcount == '':
        print('Please provide keepcount - number of files to keep in backup destination. Default keepcount=1')
        keepcount = 1

    toEmailList = ''
    if receivers:
        # receivers is comma separated string of receiver, convert to list
        receivers = receivers.replace(" ", "")
        # receivers is comma separated string of receiver, convert to list
        toEmailList = list(receivers.split(","))

    runCommand('whoami') #testing

    archivefileError = start_backup(source, dest, basedir)

    if archivefileError:
        if mailerhost:
            emailSubject = "Error archiving folder " + basedir
            emailBody = "Error creating archive '" + str(dest) + "' for folder " + str(basedir) + " in " + str(source) + "; Error=" + repr(archivefileError)
            #emailBody = "Error creating archive: " + repr(archivefileError)
            send_email_alert(mailerhost, fromEmail, toEmailList, emailSubject, emailBody)
        else:
            print("Mailer parameters are not provided: Error email has not been sent. Error=",archivefileError)

        print("archivefileError=", archivefileError)
    else:
        print("Archive completed successfully")

    ### remove old files from output directory based on keepcount ###
    #1) take output directory (dest or -d) and get base and filename
    basename, filename = os.path.split(dest)
    print("basename=",basename,", filename=",filename)
    #2) list all files in output directory

    #onlyfiles = [f for f in listdir(basename) if isfile(join(basename, f))]

    # Get list of all files only in the given directory
    onlyfiles = filter(os.path.isfile, glob.glob(basename + '/*'))
    # Sort list of files based on last modification time in ascending order
    onlyfiles = sorted(onlyfiles, key=os.path.getmtime)

    for count, file in enumerate(onlyfiles):
        timestamp_str = time.strftime('%m/%d/%Y :: %H:%M:%S', time.gmtime(os.path.getmtime(file)))
        print("count=",count,", timestamp_str=",timestamp_str, "file=",file)

    if len(onlyfiles) > int(keepcount):
        lendiff = len(onlyfiles) - int(keepcount)
        for i in range(lendiff):
            print("remove=", onlyfiles[i])
            #os.remove(onlyfiles[i])

    ### EOF remove old files from output directory based on keepcount ###

if __name__ == '__main__':
    #python filesbackup.py -s test -d myarchive

    main(sys.argv[1:])

<|endoftext|>


<commit_before>""""
Creates the station collection in the database from the facility.dat file. Existing collection will be dropped.
"""

from pymongo import MongoClient
import string
import re
import argparse
import ast
import aka_call_sign
from radiostations.aka_call_sign import get_aka_sign

class Facility(object):
    def __init__(self, data):
        self.fields = data.split('|')
        self.city=self.fields[0]
        self.state=self.fields[1]
        self.callsign=self.fields[5]
        self.displaysign=self.callsign
        self.channel=self.fields[6]
        self.frequency=self.fields[9]
        self.service=self.fields[10]
        self.facility=self.fields[14]
        self.insert_template = string.Template('{"facility-id": "$fac", ' + \
                           '"fcc-call-sign": "$csign", ' + \
                           '"aka-call-sign": "$asign", ' + \
                           '"display-sign":"$dsign",' + \
                           '"freq": $freq, ' + \
                           '"amfm": "$amfm", ' + \
                           '"antennas": [], ' + \
                           '"address": {"city": "$city", "state": "$st"}, ' + \
                           '}')

    def to_display_sign(self):
        '''
        Create the display sign. Some stations use a different call sign than the official one, just a few.
        '''
        call_sign = get_aka_sign(self.callsign)

        if self.service == "FM" and call_sign[-3:] != "-FM":
            call_sign = call_sign + "-FM"

        return call_sign + ' ' + '{:g}'.format(float(self.frequency))

    def to_aka_sign(self):
        call_sign = get_aka_sign(self.callsign)

        if self.service == "FM" and call_sign[-3:] != "-FM":
            call_sign = call_sign + "-FM"

        return call_sign

    def to_insert_string(self):
        return self.insert_template.substitute(fac=self.facility, csign=self.callsign, asign=self.to_aka_sign(), dsign=self.to_display_sign(), freq=float(self.frequency),
                                               amfm=self.service, city=self.city, st=self.state)
    def to_dict(self):
        return ast.literal_eval(self.to_insert_string())

def get_facilities(facility_file):
    checkCallSign = re.compile(r"(?:[K,W][A-Z]{2,3})(?:-AM/FM|-AM|-FM)?")
    checkCallSignSvc = re.compile(r"(AM|FM)$")

    with open(facility_file, 'r') as fac:

        for fac_data in fac:

            facility = Facility(fac_data)

            call_sign_match = checkCallSign.match(facility.callsign)
            if not call_sign_match:
                continue;

            if not checkCallSignSvc.match(facility.service):
                continue

            yield facility.to_dict()

def main(argv=None):

    argparser = argparse.ArgumentParser(description="Create station collection from fcc facility data")
    argparser.add_argument('--facility_file', dest='facility_file', default='facility.dat', help="path to facility file (i.e. facility.dat) defaults to facility.dat in current directory")
    argparser.add_argument('--collection', dest='collection', default='stations', help='name of collection to create, default is stations. Drops existing collection if it exists')
    argparser.add_argument('--db_name', dest='dbname', required=True, help='name of mongodb database, required')
    args = argparser.parse_args()

    # assumes the database server is listening @ localhost:27017
    client = MongoClient()
    db = client[args.dbname]
    db.drop_collection(args.collection)
    stations = db.create_collection(args.collection)

    stations.insert(get_facilities(args.facility_file))
    stations.create_index("fcc-call-sign")
    stations.create_index("aka-call-sign")
    stations.create_index("facility-id")

    print str(stations.count()) + " stations inserted"

if __name__ == "__main__":
    main()

<commit_msg>Apply str.title to city name<commit_after>""""
Creates the station collection in the database from the facility.dat file. Existing collection will be dropped.
"""

from pymongo import MongoClient
import string
import re
import argparse
import ast
from radiostations.aka_call_sign import get_aka_sign

class Facility(object):
    def __init__(self, data):
        self.fields = data.split('|')
        self.city=str.title(self.fields[0])
        self.state=self.fields[1]
        self.callsign=self.fields[5]
        self.displaysign=self.callsign
        self.channel=self.fields[6]
        self.frequency=self.fields[9]
        self.service=self.fields[10]
        self.facility=self.fields[14]
        self.insert_template = string.Template('{"facility-id": "$fac", ' + \
                           '"fcc-call-sign": "$csign", ' + \
                           '"aka-call-sign": "$asign", ' + \
                           '"display-sign":"$dsign",' + \
                           '"freq": $freq, ' + \
                           '"amfm": "$amfm", ' + \
                           '"antennas": [], ' + \
                           '"address": {"city": "$city", "state": "$st"}, ' + \
                           '}')

    def to_display_sign(self):
        '''
        Create the display sign. Some stations use a different call sign than the official one, just a few.
        '''
        call_sign = get_aka_sign(self.callsign)

        if self.service == "FM" and call_sign[-3:] != "-FM":
            call_sign = call_sign + "-FM"

        return call_sign + ' ' + '{:g}'.format(float(self.frequency))

    def to_aka_sign(self):
        call_sign = get_aka_sign(self.callsign)

        if self.service == "FM" and call_sign[-3:] != "-FM":
            call_sign = call_sign + "-FM"

        return call_sign

    def to_insert_string(self):
        return self.insert_template.substitute(fac=self.facility, csign=self.callsign, asign=self.to_aka_sign(), dsign=self.to_display_sign(), freq=float(self.frequency),
                                               amfm=self.service, city=self.city, st=self.state)
    def to_dict(self):
        return ast.literal_eval(self.to_insert_string())

def get_facilities(facility_file):
    checkCallSign = re.compile(r"(?:[K,W][A-Z]{2,3})(?:-AM/FM|-AM|-FM)?")
    checkCallSignSvc = re.compile(r"(AM|FM)$")

    with open(facility_file, 'r') as fac:

        for fac_data in fac:

            facility = Facility(fac_data)

            call_sign_match = checkCallSign.match(facility.callsign)
            if not call_sign_match:
                continue;

            if not checkCallSignSvc.match(facility.service):
                continue

            yield facility.to_dict()

def main(argv=None):

    argparser = argparse.ArgumentParser(description="Create station collection from fcc facility data")
    argparser.add_argument('--facility_file', dest='facility_file', default='facility.dat', help="path to facility file (i.e. facility.dat) defaults to facility.dat in current directory")
    argparser.add_argument('--collection', dest='collection', default='stations', help='name of collection to create, default is stations. Drops existing collection if it exists')
    argparser.add_argument('--db_name', dest='dbname', required=True, help='name of mongodb database, required')
    args = argparser.parse_args()

    # assumes the database server is listening @ localhost:27017
    client = MongoClient()
    db = client[args.dbname]
    db.drop_collection(args.collection)
    stations = db.create_collection(args.collection)

    stations.insert(get_facilities(args.facility_file))
    stations.create_index("fcc-call-sign")
    stations.create_index("aka-call-sign")
    stations.create_index("facility-id")

    print str(stations.count()) + " stations inserted"

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import gevent
import pytest

from raiden.tests.utils.transfer import (
    assert_synched_channels,
)
from raiden.tests.utils.blockchain import wait_until_block
from raiden.blockchain.abi import (
    CONTRACT_MANAGER,
    EVENT_CHANNEL_CLOSED,
    EVENT_CHANNEL_NEW_BALANCE,
    EVENT_CHANNEL_SETTLED,
)
from raiden.blockchain.events import (
    ALL_EVENTS,
    get_all_channel_manager_events,
    get_all_netting_channel_events,
    get_all_registry_events,
)
from raiden.utils import address_encoder

def event_dicts_are_equal(dict1, dict2):
    for k, v in dict1.items():
        if k not in dict2:
            return False
        if k == 'block_number':
            continue

        v2 = dict2[k]
        if isinstance(v2, str) and v2.startswith('0x'):
            v2 = v2[2:]
        if isinstance(v, str) and v.startswith('0x'):
            v = v[2:]
        if v2 != v:
            return False

    return True

@pytest.mark.parametrize('number_of_nodes', [2])
@pytest.mark.parametrize('channels_per_node', [0])
def test_event_new_channel(raiden_chain, deposit, settle_timeout, events_poll_timeout):
    app0, app1 = raiden_chain  # pylint: disable=unbalanced-tuple-unpacking

    token_address = app0.raiden.default_registry.token_addresses()[0]

    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 0
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 0

    token0 = app0.raiden.chain.token(token_address)
    graph0 = app0.raiden.default_registry.manager_by_token(token_address)

    token1 = app1.raiden.chain.token(token_address)

    netcontract_address = graph0.new_netting_channel(
        app1.raiden.address,
        settle_timeout,
    )

    netting_channel0 = app0.raiden.chain.netting_channel(netcontract_address)
    netting_channel1 = app1.raiden.chain.netting_channel(netcontract_address)

    gevent.sleep(events_poll_timeout)

    # channel is created but not opened and without funds
    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 1
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 1

    channel0 = list(app0.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501
    channel1 = list(app1.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501

    assert_synched_channels(
        channel0, 0, [],
        channel1, 0, [],
    )

    token0.approve(netcontract_address, deposit)
    netting_channel0.deposit(deposit)

    gevent.sleep(events_poll_timeout)

    # channel is open but single funded
    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 1
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 1

    channel0 = list(app0.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501
    channel1 = list(app1.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501

    assert_synched_channels(
        channel0, deposit, [],
        channel1, 0, [],
    )

    token1.approve(netcontract_address, deposit)
    netting_channel1.deposit(deposit)

    gevent.sleep(events_poll_timeout)

    # channel is open and funded by both participants
    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 1
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 1

    channel0 = list(app0.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501
    channel1 = list(app1.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501

    assert_synched_channels(
        channel0, deposit, [],
        channel1, deposit, [],
    )

@pytest.mark.parametrize('number_of_nodes', [2])
@pytest.mark.parametrize('channels_per_node', [0])
@pytest.mark.parametrize('cached_genesis', [None])
def test_query_events(raiden_chain, deposit, settle_timeout, events_poll_timeout):
    app0, app1 = raiden_chain  # pylint: disable=unbalanced-tuple-unpacking

    token_address = app0.raiden.default_registry.token_addresses()[0]

    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 0
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 0

    token0 = app0.raiden.chain.token(token_address)
    manager0 = app0.raiden.default_registry.manager_by_token(token_address)

    events = get_all_registry_events(
        app0.raiden.chain,
        app0.raiden.default_registry.address,
        events=ALL_EVENTS,
        from_block=0,
        to_block='latest',
    )

    assert len(events) == 1
    assert event_dicts_are_equal(events[0], {
        '_event_type': b'TokenAdded',
        'channel_manager_address': address_encoder(manager0.address),
        'token_address': address_encoder(token_address),
        'block_number': 'ignore',
    })

    events = get_all_registry_events(
        app0.raiden.chain,
        app0.raiden.default_registry.address,
        events=ALL_EVENTS,
        from_block=999999998,
        to_block=999999999,
    )
    assert not events

    netcontract_address = manager0.new_netting_channel(
        app1.raiden.address,
        settle_timeout,
    )

    events = get_all_channel_manager_events(
        app0.raiden.chain,
        manager0.address,
        events=ALL_EVENTS,
        from_block=0,
        to_block='latest',
    )

    assert len(events) == 1
    assert event_dicts_are_equal(events[0], {
        '_event_type': b'ChannelNew',
        'settle_timeout': settle_timeout,
        'netting_channel': address_encoder(netcontract_address),
        'participant1': address_encoder(app0.raiden.address),
        'participant2': address_encoder(app1.raiden.address),
        'block_number': 'ignore',
    })

    events = get_all_channel_manager_events(
        app0.raiden.chain,
        manager0.address,
        events=ALL_EVENTS,
        from_block=999999998,
        to_block=999999999,
    )
    assert not events

    netting_channel0 = app0.raiden.chain.netting_channel(netcontract_address)

    gevent.sleep(events_poll_timeout * 2)

    # channel is created but not opened and without funds
    assert len(app0.raiden.token_to_channelgraph[token_address].address_to_channel) == 1
    assert len(app1.raiden.token_to_channelgraph[token_address].address_to_channel) == 1

    channel0 = list(app0.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501
    channel1 = list(app1.raiden.token_to_channelgraph[token_address].address_to_channel.values())[0]  # noqa: E501

    assert_synched_channels(
        channel0, 0, [],
        channel1, 0, [],
    )

    token0.approve(netcontract_address, deposit)
    netting_channel0.deposit(deposit)

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        netting_channel_address=netcontract_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        netcontract_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_NEW_BALANCE)],
    )

    assert len(all_netting_channel_events) == 1
    assert len(events) == 1

    new_balance_event = {
        '_event_type': b'ChannelNewBalance',
        'token_address': address_encoder(token_address),
        'participant': address_encoder(app0.raiden.address),
        'balance': deposit,
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], new_balance_event)
    assert event_dicts_are_equal(events[0], new_balance_event)

    channel0.external_state.close(None)

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        netting_channel_address=netcontract_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        netcontract_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_CLOSED)],
    )

    assert len(all_netting_channel_events) == 2
    assert len(events) == 1

    closed_event = {
        '_event_type': b'ChannelClosed',
        'closing_address': address_encoder(app0.raiden.address),
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], closed_event)
    assert event_dicts_are_equal(events[0], closed_event)

    settle_expiration = app0.raiden.chain.block_number() + settle_timeout + 1
    wait_until_block(app0.raiden.chain, settle_expiration)

    channel1.external_state.settle()

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        netting_channel_address=netcontract_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        netcontract_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_SETTLED)],
    )

    assert len(all_netting_channel_events) == 3
    assert len(events) == 1

    settled_event = {
        '_event_type': b'ChannelSettled',
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], settled_event)
    assert event_dicts_are_equal(events[0], settled_event)
<commit_msg>fix test_events<commit_after># -*- coding: utf-8 -*-
import gevent
import pytest

from raiden.api.python import RaidenAPI2
from raiden.blockchain.abi import (
    CONTRACT_MANAGER,
    EVENT_CHANNEL_CLOSED,
    EVENT_CHANNEL_NEW_BALANCE,
    EVENT_CHANNEL_SETTLED,
)
from raiden.blockchain.events import (
    ALL_EVENTS,
    get_all_channel_manager_events,
    get_all_netting_channel_events,
    get_all_registry_events,
)
from raiden.tests.utils.transfer import (
    assert_synched_channel_state,
    get_channelstate,
    pending_mediated_transfer,
)
from raiden.tests.utils.blockchain import wait_until_block
from raiden.tests.utils.network import CHAIN
from raiden.transfer import views, channel
from raiden.utils import address_encoder, sha3

def event_dicts_are_equal(dict1, dict2):
    for k, v in dict1.items():
        if k not in dict2:
            return False
        if k == 'block_number':
            continue

        v2 = dict2[k]
        if isinstance(v2, str) and v2.startswith('0x'):
            v2 = v2[2:]
        if isinstance(v, str) and v.startswith('0x'):
            v = v[2:]
        if v2 != v:
            return False

    return True

@pytest.mark.parametrize('number_of_nodes', [2])
@pytest.mark.parametrize('channels_per_node', [0])
def test_channel_new(raiden_chain, events_poll_timeout, token_addresses):
    app0, app1 = raiden_chain  # pylint: disable=unbalanced-tuple-unpacking
    registry_address = app0.raiden.default_registry.address
    token_address = token_addresses[0]

    channelcount0 = views.total_token_network_channels(
        views.state_from_app(app0),
        registry_address,
        token_address,
    )

    RaidenAPI2(app0.raiden).channel_open(token_address, app1.raiden.address)
    gevent.sleep(events_poll_timeout)

    # The channel is created but without funds
    channelcount1 = views.total_token_network_channels(
        views.state_from_app(app0),
        registry_address,
        token_address,
    )
    assert channelcount0 + 1 == channelcount1

@pytest.mark.parametrize('privatekey_seed', ['event_new_channel:{}'])
@pytest.mark.parametrize('number_of_nodes', [2])
@pytest.mark.parametrize('channels_per_node', [0])
def test_channel_deposit(raiden_chain, deposit, events_poll_timeout, token_addresses):
    app0, app1 = raiden_chain
    token_address = token_addresses[0]

    channel0 = get_channelstate(app0, app1, token_address)
    channel1 = get_channelstate(app1, app0, token_address)
    assert channel0 is None
    assert channel1 is None

    RaidenAPI2(app0.raiden).channel_open(token_address, app1.raiden.address)
    gevent.sleep(events_poll_timeout)

    assert_synched_channel_state(
        token_address,
        app0, 0, [],
        app1, 0, [],
    )

    RaidenAPI2(app0.raiden).channel_deposit(token_address, app1.raiden.address, deposit)
    gevent.sleep(events_poll_timeout)

    assert_synched_channel_state(
        token_address,
        app0, deposit, [],
        app1, 0, [],
    )

    RaidenAPI2(app1.raiden).channel_deposit(token_address, app0.raiden.address, deposit)
    gevent.sleep(events_poll_timeout)

    assert_synched_channel_state(
        token_address,
        app0, deposit, [],
        app1, deposit, [],
    )

@pytest.mark.parametrize('number_of_nodes', [2])
@pytest.mark.parametrize('channels_per_node', [0])
@pytest.mark.parametrize('cached_genesis', [None])
def test_query_events(raiden_chain, token_addresses, deposit, settle_timeout, events_poll_timeout):
    app0, app1 = raiden_chain  # pylint: disable=unbalanced-tuple-unpacking
    registry_address = app0.raiden.default_registry.address
    token_address = token_addresses[0]

    manager0 = app0.raiden.default_registry.manager_by_token(token_address)

    channelcount0 = views.total_token_network_channels(
        views.state_from_app(app0),
        registry_address,
        token_address,
    )

    events = get_all_registry_events(
        app0.raiden.chain,
        app0.raiden.default_registry.address,
        events=ALL_EVENTS,
        from_block=0,
        to_block='latest',
    )

    assert len(events) == 1
    assert event_dicts_are_equal(events[0], {
        '_event_type': b'TokenAdded',
        'channel_manager_address': address_encoder(manager0.address),
        'token_address': address_encoder(token_address),
        'block_number': 'ignore',
    })

    events = get_all_registry_events(
        app0.raiden.chain,
        app0.raiden.default_registry.address,
        events=ALL_EVENTS,
        from_block=999999998,
        to_block=999999999,
    )
    assert not events

    channel_address = RaidenAPI2(app0.raiden).channel_open(token_address, app1.raiden.address)
    gevent.sleep(events_poll_timeout * 2)

    events = get_all_channel_manager_events(
        app0.raiden.chain,
        manager0.address,
        events=ALL_EVENTS,
        from_block=0,
        to_block='latest',
    )

    assert len(events) == 1
    assert event_dicts_are_equal(events[0], {
        '_event_type': b'ChannelNew',
        'settle_timeout': settle_timeout,
        'netting_channel': address_encoder(channel_address),
        'participant1': address_encoder(app0.raiden.address),
        'participant2': address_encoder(app1.raiden.address),
        'block_number': 'ignore',
    })

    events = get_all_channel_manager_events(
        app0.raiden.chain,
        manager0.address,
        events=ALL_EVENTS,
        from_block=999999998,
        to_block=999999999,
    )
    assert not events

    # channel is created but not opened and without funds
    channelcount1 = views.total_token_network_channels(
        views.state_from_app(app0),
        registry_address,
        token_address,
    )
    assert channelcount0 + 1 == channelcount1

    assert_synched_channel_state(
        token_address,
        app0, 0, [],
        app1, 0, [],
    )

    RaidenAPI2(app0.raiden).channel_deposit(token_address, app1.raiden.address, deposit)
    gevent.sleep(events_poll_timeout * 2)

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        channel_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        channel_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_NEW_BALANCE)],
    )

    assert len(all_netting_channel_events) == 1
    assert len(events) == 1

    new_balance_event = {
        '_event_type': b'ChannelNewBalance',
        'token_address': address_encoder(token_address),
        'participant': address_encoder(app0.raiden.address),
        'balance': deposit,
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], new_balance_event)
    assert event_dicts_are_equal(events[0], new_balance_event)

    RaidenAPI2(app0.raiden).channel_close(token_address, app1.raiden.address)
    gevent.sleep(events_poll_timeout * 2)

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        netting_channel_address=channel_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        channel_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_CLOSED)],
    )

    assert len(all_netting_channel_events) == 2
    assert len(events) == 1

    closed_event = {
        '_event_type': b'ChannelClosed',
        'closing_address': address_encoder(app0.raiden.address),
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], closed_event)
    assert event_dicts_are_equal(events[0], closed_event)

    settle_expiration = app0.raiden.chain.block_number() + settle_timeout + 5
    wait_until_block(app0.raiden.chain, settle_expiration)

    all_netting_channel_events = get_all_netting_channel_events(
        app0.raiden.chain,
        netting_channel_address=channel_address,
        from_block=0,
        to_block='latest',
    )

    events = get_all_netting_channel_events(
        app0.raiden.chain,
        channel_address,
        events=[CONTRACT_MANAGER.get_event_id(EVENT_CHANNEL_SETTLED)],
    )

    assert len(all_netting_channel_events) == 3
    assert len(events) == 1

    settled_event = {
        '_event_type': b'ChannelSettled',
        'block_number': 'ignore',
    }

    assert event_dicts_are_equal(all_netting_channel_events[-1], settled_event)
    assert event_dicts_are_equal(events[0], settled_event)

@pytest.mark.xfail(reason='out-of-gas for unlock and settle')
@pytest.mark.parametrize('number_of_nodes', [3])
@pytest.mark.parametrize('channels_per_node', [CHAIN])
def test_secret_revealed(raiden_chain, deposit, settle_timeout, token_addresses):
    app0, app1, app2 = raiden_chain
    token_address = token_addresses[0]

    amount = 10
    identifier = 1
    secret = pending_mediated_transfer(
        raiden_chain,
        token_address,
        amount,
        identifier,
    )
    hashlock = sha3(secret)

    gevent.sleep(.1)  # wait for the messages

    channel_state2_1 = get_channelstate(app2, app1, token_address)

    # the secret hasn't been revealed yet (through messages)

    assert len(channel_state2_1.our_state.hashlocks_to_pendinglocks) == 1
    proofs = list(channel.get_known_unlocks(channel_state2_1.our_state))
    assert not proofs

    channel.register_secret(channel_state2_1, secret, hashlock)

    # Close the channel
    netting_channel_proxy = app2.raiden.chain.netting_channel(channel_state2_1.identifier)
    netting_channel_proxy.channel_close(channel_state2_1.partner_state.balance_proof)

    # Reveal the secret through the blockchain (this needs to emit the
    # SecretRevealed event)
    unlock_proofs = channel.get_known_unlocks(channel_state2_1.partner_state)
    netting_channel_proxy.withdraw(app2.raiden.address, unlock_proofs)

    settle_expiration = app0.raiden.chain.block_number() + settle_timeout
    wait_until_block(app0.raiden.chain, settle_expiration)

    assert_synched_channel_state(
        token_address,
        app1, deposit - amount, [],
        app2, deposit + amount, [],
    )

    assert_synched_channel_state(
        token_address,
        app0, deposit - amount, [],
        app1, deposit + amount, [],
    )
<|endoftext|>


<commit_before>from rest_framework import VERSION

version = tuple(map(int, VERSION.split(".")))

if version[:2] >= (3, 10):
    from rest_framework.schemas.coreapi import is_custom_action
else:
    from rest_framework.schemas.generators import is_custom_action
<commit_msg>Lint ignore drf compat<commit_after>from rest_framework import VERSION

version = tuple(map(int, VERSION.split(".")))

if version[:2] >= (3, 10):
    from rest_framework.schemas.coreapi import is_custom_action  # noqa
else:
    from rest_framework.schemas.generators import is_custom_action  # noqa
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function

from scipy.stats import norm

import numpy as np

from . import common_args
from ..util import read_param_file

from multiprocessing import Pool, cpu_count
from functools import partial
try:
    from itertools import zip_longest
except ImportError:
    # Python 2
    from itertools import izip_longest as zip_longest

def analyze(problem, Y, calc_second_order=True, num_resamples=100,
            conf_level=0.95, print_to_console=False, parallel=False,
            n_processors=None):
    """Perform Sobol Analysis on model outputs.

    Returns a dictionary with keys 'S1', 'S1_conf', 'ST', and 'ST_conf', where
    each entry is a list of size D (the number of parameters) containing the
    indices in the same order as the parameter file.  If calc_second_order is
    True, the dictionary also contains keys 'S2' and 'S2_conf'.

    Parameters
    ----------
    problem : dict
        The problem definition
    Y : numpy.array
        A NumPy array containing the model outputs
    calc_second_order : bool
        Calculate second-order sensitivities (default True)
    num_resamples : int
        The number of resamples (default 100)
    conf_level : float
        The confidence interval level (default 0.95)
    print_to_console : bool
        Print results directly to console (default False)

    References
    ----------
    .. [1] Sobol, I. M. (2001).  "Global sensitivity indices for nonlinear
           mathematical models and their Monte Carlo estimates."  Mathematics
           and Computers in Simulation, 55(1-3):271-280,
           doi:10.1016/S0378-4754(00)00270-6.
    .. [2] Saltelli, A. (2002).  "Making best use of model evaluations to
           compute sensitivity indices."  Computer Physics Communications,
           145(2):280-297, doi:10.1016/S0010-4655(02)00280-1.
    .. [3] Saltelli, A., P. Annoni, I. Azzini, F. Campolongo, M. Ratto, and
           S. Tarantola (2010).  "Variance based sensitivity analysis of model
           output.  Design and estimator for the total sensitivity index."
           Computer Physics Communications, 181(2):259-270,
           doi:10.1016/j.cpc.2009.09.018.

    Examples
    --------
    >>> X = saltelli.sample(problem, 1000)
    >>> Y = Ishigami.evaluate(X)
    >>> Si = sobol.analyze(problem, Y, print_to_console=True)
    """
    # determining if groups are defined and adjusting the number
    # of rows in the cross-sampled matrix accordingly
    if problem['groups'] == None:
        D = problem['num_vars']
    else:
        D = len(problem['groups'][1])

    if calc_second_order and Y.size % (2 * D + 2) == 0:
        N = int(Y.size / (2 * D + 2))
    elif not calc_second_order and Y.size % (D + 2) == 0:
        N = int(Y.size / (D + 2))
    else:
        raise RuntimeError("""
        Incorrect number of samples in model output file.
        Confirm that calc_second_order matches option used during sampling.""")

    if conf_level < 0 or conf_level > 1:
        raise RuntimeError("Confidence level must be between 0-1.")

    A,B,AB,BA = separate_output_values(Y, D, N, calc_second_order)
    r = np.random.randint(N, size=(N, num_resamples))
    Z = norm.ppf(0.5 + conf_level / 2)

    if not parallel:
        S = create_Si_dict(D, calc_second_order)

        for j in range(D):
            S['S1'][j] = first_order(A, AB[:, j], B)
            S['S1_conf'][j] = Z * first_order(A[r], AB[r,j], B[r]).std(ddof=1)
            S['ST'][j] = total_order(A, AB[:, j], B)
            S['ST_conf'][j] = Z * total_order(A[r], AB[r,j], B[r]).std(ddof=1)

        # Second order (+conf.)
        if calc_second_order:
            for j in range(D):
                for k in range(j + 1, D):
                    S['S2'][j, k] = second_order(
                        A, AB[:, j], AB[:, k], BA[:, j], B)
                    S['S2_conf'][j, k] = Z * second_order(A[r], AB[r, j],
                        AB[r, k], BA[r, j], B[r]).std(ddof=1)

    else:
        tasks, n_processors = create_task_list(D, calc_second_order, n_processors)

        func = partial(sobol_parallel, Z, A, AB, BA, B, r)
        pool = Pool(n_processors)
        S_list = pool.map_async(func, tasks)
        pool.close()
        pool.join()

        S = Si_list_to_dict(S_list.get(), D, calc_second_order)

    # Print results to console
    if print_to_console:
        print_indices(S, problem, calc_second_order)

    return S

def first_order(A, AB, B):
    # First order estimator following Saltelli et al. 2010 CPC, normalized by
    # sample variance
    return np.mean(B * (AB - A), axis=0) / np.var(np.r_[A, B], axis=0)

def total_order(A, AB, B):
    # Total order estimator following Saltelli et al. 2010 CPC, normalized by
    # sample variance
    return 0.5 * np.mean((A - AB) ** 2, axis=0) / np.var(np.r_[A, B], axis=0)

def second_order(A, ABj, ABk, BAj, B):
    # Second order estimator following Saltelli 2002
    Vjk = np.mean(BAj * ABk - A * B, axis=0) / np.var(np.r_[A, B], axis=0)
    Sj = first_order(A, ABj, B)
    Sk = first_order(A, ABk, B)

    return Vjk - Sj - Sk

def create_Si_dict(D, calc_second_order):
    # initialize empty dict to store sensitivity indices
    S = dict((k, np.empty(D)) for k in ('S1', 'S1_conf', 'ST', 'ST_conf'))

    if calc_second_order:
        S['S2'] = np.empty((D, D))
        S['S2'][:] = np.nan
        S['S2_conf'] = np.empty((D, D))
        S['S2_conf'][:] = np.nan

    return S

def separate_output_values(Y, D, N, calc_second_order):
    AB = np.empty((N, D))
    BA = np.empty((N, D)) if calc_second_order else None
    step = 2 * D + 2 if calc_second_order else D + 2

    A = Y[0:Y.size:step]
    B = Y[(step - 1):Y.size:step]
    for j in range(D):
        AB[:, j] = Y[(j + 1):Y.size:step]
        if calc_second_order:
            BA[:, j] = Y[(j + 1 + D):Y.size:step]

    return A,B,AB,BA

def sobol_parallel(Z, A, AB, BA, B, r, tasks):
    sobol_indices = []
    for d,j,k in tasks:
        if d == 'S1':
            s = first_order(A, AB[:, j], B)
        elif d == 'S1_conf':
            s = Z * first_order(A[r], AB[r, j], B[r]).std(ddof=1)
        elif d == 'ST':
            s = total_order(A, AB[:, j], B)
        elif d == 'ST_conf':
            s = Z * total_order(A[r], AB[r, j], B[r]).std(ddof=1)
        elif d == 'S2':
            s = second_order(A, AB[:, j], AB[:, k], BA[:, j], B)
        elif d == 'S2_conf':
            s = Z * second_order(A[r], AB[r, j], AB[r, k],
                                               BA[r, j], B[r]).std(ddof=1)
        sobol_indices.append([d, j, k, s])

    return sobol_indices

def create_task_list(D, calc_second_order, n_processors):
    # Create list with one entry (key, parameter 1, parameter 2) per sobol
    # index (+conf.). This is used to supply parallel tasks to multiprocessing.Pool
    tasks_first_order = [[d, j, None] for j in range(D) for d in ('S1', 'S1_conf', 'ST', 'ST_conf')]

    # Add second order (+conf.) to tasks
    tasks_second_order = []
    if calc_second_order:
        tasks_second_order = [[d, j, k] for j in range(D) for k in
                            range(j + 1, D) for d in ('S2', 'S2_conf')]

    if n_processors == None:
        n_processors = min(cpu_count(), len(tasks_first_order) + len(tasks_second_order))

    if not calc_second_order:
        tasks = np.array_split(tasks_first_order, n_processors)
    else:
        # merges both lists alternating its elements and splits the resulting list into n_processors sublists
        tasks = np.array_split([v for v in sum(
            zip_longest(tasks_first_order[::-1], tasks_second_order), ())
                if v is not None], n_processors)

    return tasks, n_processors

def Si_list_to_dict(S_list, D, calc_second_order):
    # Convert the parallel output into the regular dict format for printing/returning
    S = create_Si_dict(D, calc_second_order)
    L = []
    for l in S_list: # first reformat to flatten
        L += l

    for s in L:  # First order (+conf.)
        if s[2] is None:
            S[s[0]][s[1]] = s[3]
        else:
            S[s[0]][s[1], s[2]] = s[3]

    return S

def print_indices(S, problem, calc_second_order):
    # Output to console
    if problem['groups'] == None:
        D = problem['num_vars']
        print('Parameter S1 S1_conf ST ST_conf')

        for j in range(D):
            print('%s %f %f %f %f' % (problem['names'][j], S['S1'][
                j], S['S1_conf'][j], S['ST'][j], S['ST_conf'][j]))

        if calc_second_order:
            print('\nParameter_1 Parameter_2 S2 S2_conf')

            for j in range(D):
                for k in range(j + 1, D):
                    print("%s %s %f %f" % (problem['names'][j], problem[
                        'names'][k], S['S2'][j, k], S['S2_conf'][j, k]))

    else:
        D = len(problem['groups'][1])
        print('Group S1 S1_conf ST ST_conf')

        for j in range(D):
            print('%s %f %f %f %f' % (problem['groups'][1][j], S['S1'][
                j], S['S1_conf'][j], S['ST'][j], S['ST_conf'][j]))

        if calc_second_order:
            print('\nGroup_1 Group_2 S2 S2_conf')

            for j in range(D):
                for k in range(j + 1, D):
                    print("%s %s %f %f" % (problem['groups'][1][j], problem[
                    'groups'][1][k], S['S2'][j, k], S['S2_conf'][j, k]))

if __name__ == "__main__":
    parser = common_args.create()
    parser.add_argument('--max-order', type=int, required=False, default=2,
                        choices=[1, 2],
                        help='Maximum order of sensitivity indices to '
                             'calculate')
    parser.add_argument('-r', '--resamples', type=int, required=False,
                        default=1000,
                        help='Number of bootstrap resamples for Sobol '
                             'confidence intervals')
    parser.add_argument('--parallel', action='store_true', help='Makes '
                        'use of parallelization.',
                        dest='parallel')
    parser.add_argument('--processors', type=int, required=False,
                        default=None,
                        help='Number of processors to be used with the ' +
                        'parallel option.', dest='n_processors')
    args = parser.parse_args()

    problem = read_param_file(args.paramfile)
    Y = np.loadtxt(args.model_output_file, delimiter=args.delimiter,
                   usecols=(args.column,))

    analyze(problem, Y, (args.max_order == 2),
            num_resamples=args.resamples, print_to_console=True,
            parallel=args.parallel, n_processors=args.n_processors)
<commit_msg>Replaced '== None' conditions<commit_after>from __future__ import division
from __future__ import print_function

from scipy.stats import norm

import numpy as np

from . import common_args
from ..util import read_param_file

from multiprocessing import Pool, cpu_count
from functools import partial
try:
    from itertools import zip_longest
except ImportError:
    # Python 2
    from itertools import izip_longest as zip_longest

def analyze(problem, Y, calc_second_order=True, num_resamples=100,
            conf_level=0.95, print_to_console=False, parallel=False,
            n_processors=None):
    """Perform Sobol Analysis on model outputs.

    Returns a dictionary with keys 'S1', 'S1_conf', 'ST', and 'ST_conf', where
    each entry is a list of size D (the number of parameters) containing the
    indices in the same order as the parameter file.  If calc_second_order is
    True, the dictionary also contains keys 'S2' and 'S2_conf'.

    Parameters
    ----------
    problem : dict
        The problem definition
    Y : numpy.array
        A NumPy array containing the model outputs
    calc_second_order : bool
        Calculate second-order sensitivities (default True)
    num_resamples : int
        The number of resamples (default 100)
    conf_level : float
        The confidence interval level (default 0.95)
    print_to_console : bool
        Print results directly to console (default False)

    References
    ----------
    .. [1] Sobol, I. M. (2001).  "Global sensitivity indices for nonlinear
           mathematical models and their Monte Carlo estimates."  Mathematics
           and Computers in Simulation, 55(1-3):271-280,
           doi:10.1016/S0378-4754(00)00270-6.
    .. [2] Saltelli, A. (2002).  "Making best use of model evaluations to
           compute sensitivity indices."  Computer Physics Communications,
           145(2):280-297, doi:10.1016/S0010-4655(02)00280-1.
    .. [3] Saltelli, A., P. Annoni, I. Azzini, F. Campolongo, M. Ratto, and
           S. Tarantola (2010).  "Variance based sensitivity analysis of model
           output.  Design and estimator for the total sensitivity index."
           Computer Physics Communications, 181(2):259-270,
           doi:10.1016/j.cpc.2009.09.018.

    Examples
    --------
    >>> X = saltelli.sample(problem, 1000)
    >>> Y = Ishigami.evaluate(X)
    >>> Si = sobol.analyze(problem, Y, print_to_console=True)
    """
    # determining if groups are defined and adjusting the number
    # of rows in the cross-sampled matrix accordingly
    if not problem.get('groups'):
        D = problem['num_vars']
    else:
        D = len(problem['groups'][1])

    if calc_second_order and Y.size % (2 * D + 2) == 0:
        N = int(Y.size / (2 * D + 2))
    elif not calc_second_order and Y.size % (D + 2) == 0:
        N = int(Y.size / (D + 2))
    else:
        raise RuntimeError("""
        Incorrect number of samples in model output file.
        Confirm that calc_second_order matches option used during sampling.""")

    if conf_level < 0 or conf_level > 1:
        raise RuntimeError("Confidence level must be between 0-1.")

    A,B,AB,BA = separate_output_values(Y, D, N, calc_second_order)
    r = np.random.randint(N, size=(N, num_resamples))
    Z = norm.ppf(0.5 + conf_level / 2)

    if not parallel:
        S = create_Si_dict(D, calc_second_order)

        for j in range(D):
            S['S1'][j] = first_order(A, AB[:, j], B)
            S['S1_conf'][j] = Z * first_order(A[r], AB[r,j], B[r]).std(ddof=1)
            S['ST'][j] = total_order(A, AB[:, j], B)
            S['ST_conf'][j] = Z * total_order(A[r], AB[r,j], B[r]).std(ddof=1)

        # Second order (+conf.)
        if calc_second_order:
            for j in range(D):
                for k in range(j + 1, D):
                    S['S2'][j, k] = second_order(
                        A, AB[:, j], AB[:, k], BA[:, j], B)
                    S['S2_conf'][j, k] = Z * second_order(A[r], AB[r, j],
                        AB[r, k], BA[r, j], B[r]).std(ddof=1)

    else:
        tasks, n_processors = create_task_list(D, calc_second_order, n_processors)

        func = partial(sobol_parallel, Z, A, AB, BA, B, r)
        pool = Pool(n_processors)
        S_list = pool.map_async(func, tasks)
        pool.close()
        pool.join()

        S = Si_list_to_dict(S_list.get(), D, calc_second_order)

    # Print results to console
    if print_to_console:
        print_indices(S, problem, calc_second_order)

    return S

def first_order(A, AB, B):
    # First order estimator following Saltelli et al. 2010 CPC, normalized by
    # sample variance
    return np.mean(B * (AB - A), axis=0) / np.var(np.r_[A, B], axis=0)

def total_order(A, AB, B):
    # Total order estimator following Saltelli et al. 2010 CPC, normalized by
    # sample variance
    return 0.5 * np.mean((A - AB) ** 2, axis=0) / np.var(np.r_[A, B], axis=0)

def second_order(A, ABj, ABk, BAj, B):
    # Second order estimator following Saltelli 2002
    Vjk = np.mean(BAj * ABk - A * B, axis=0) / np.var(np.r_[A, B], axis=0)
    Sj = first_order(A, ABj, B)
    Sk = first_order(A, ABk, B)

    return Vjk - Sj - Sk

def create_Si_dict(D, calc_second_order):
    # initialize empty dict to store sensitivity indices
    S = dict((k, np.empty(D)) for k in ('S1', 'S1_conf', 'ST', 'ST_conf'))

    if calc_second_order:
        S['S2'] = np.empty((D, D))
        S['S2'][:] = np.nan
        S['S2_conf'] = np.empty((D, D))
        S['S2_conf'][:] = np.nan

    return S

def separate_output_values(Y, D, N, calc_second_order):
    AB = np.empty((N, D))
    BA = np.empty((N, D)) if calc_second_order else None
    step = 2 * D + 2 if calc_second_order else D + 2

    A = Y[0:Y.size:step]
    B = Y[(step - 1):Y.size:step]
    for j in range(D):
        AB[:, j] = Y[(j + 1):Y.size:step]
        if calc_second_order:
            BA[:, j] = Y[(j + 1 + D):Y.size:step]

    return A,B,AB,BA

def sobol_parallel(Z, A, AB, BA, B, r, tasks):
    sobol_indices = []
    for d,j,k in tasks:
        if d == 'S1':
            s = first_order(A, AB[:, j], B)
        elif d == 'S1_conf':
            s = Z * first_order(A[r], AB[r, j], B[r]).std(ddof=1)
        elif d == 'ST':
            s = total_order(A, AB[:, j], B)
        elif d == 'ST_conf':
            s = Z * total_order(A[r], AB[r, j], B[r]).std(ddof=1)
        elif d == 'S2':
            s = second_order(A, AB[:, j], AB[:, k], BA[:, j], B)
        elif d == 'S2_conf':
            s = Z * second_order(A[r], AB[r, j], AB[r, k],
                                               BA[r, j], B[r]).std(ddof=1)
        sobol_indices.append([d, j, k, s])

    return sobol_indices

def create_task_list(D, calc_second_order, n_processors):
    # Create list with one entry (key, parameter 1, parameter 2) per sobol
    # index (+conf.). This is used to supply parallel tasks to multiprocessing.Pool
    tasks_first_order = [[d, j, None] for j in range(D) for d in ('S1', 'S1_conf', 'ST', 'ST_conf')]

    # Add second order (+conf.) to tasks
    tasks_second_order = []
    if calc_second_order:
        tasks_second_order = [[d, j, k] for j in range(D) for k in
                            range(j + 1, D) for d in ('S2', 'S2_conf')]

    if n_processors is None:
        n_processors = min(cpu_count(), len(tasks_first_order) + len(tasks_second_order))

    if not calc_second_order:
        tasks = np.array_split(tasks_first_order, n_processors)
    else:
        # merges both lists alternating its elements and splits the resulting list into n_processors sublists
        tasks = np.array_split([v for v in sum(
            zip_longest(tasks_first_order[::-1], tasks_second_order), ())
                if v is not None], n_processors)

    return tasks, n_processors

def Si_list_to_dict(S_list, D, calc_second_order):
    # Convert the parallel output into the regular dict format for printing/returning
    S = create_Si_dict(D, calc_second_order)
    L = []
    for l in S_list: # first reformat to flatten
        L += l

    for s in L:  # First order (+conf.)
        if s[2] is None:
            S[s[0]][s[1]] = s[3]
        else:
            S[s[0]][s[1], s[2]] = s[3]

    return S

def print_indices(S, problem, calc_second_order):
    # Output to console
    if not problem.get('groups'):
        D = problem['num_vars']
        print('Parameter S1 S1_conf ST ST_conf')

        for j in range(D):
            print('%s %f %f %f %f' % (problem['names'][j], S['S1'][
                j], S['S1_conf'][j], S['ST'][j], S['ST_conf'][j]))

        if calc_second_order:
            print('\nParameter_1 Parameter_2 S2 S2_conf')

            for j in range(D):
                for k in range(j + 1, D):
                    print("%s %s %f %f" % (problem['names'][j], problem[
                        'names'][k], S['S2'][j, k], S['S2_conf'][j, k]))

    else:
        D = len(problem['groups'][1])
        print('Group S1 S1_conf ST ST_conf')

        for j in range(D):
            print('%s %f %f %f %f' % (problem['groups'][1][j], S['S1'][
                j], S['S1_conf'][j], S['ST'][j], S['ST_conf'][j]))

        if calc_second_order:
            print('\nGroup_1 Group_2 S2 S2_conf')

            for j in range(D):
                for k in range(j + 1, D):
                    print("%s %s %f %f" % (problem['groups'][1][j], problem[
                    'groups'][1][k], S['S2'][j, k], S['S2_conf'][j, k]))

if __name__ == "__main__":
    parser = common_args.create()
    parser.add_argument('--max-order', type=int, required=False, default=2,
                        choices=[1, 2],
                        help='Maximum order of sensitivity indices to '
                             'calculate')
    parser.add_argument('-r', '--resamples', type=int, required=False,
                        default=1000,
                        help='Number of bootstrap resamples for Sobol '
                             'confidence intervals')
    parser.add_argument('--parallel', action='store_true', help='Makes '
                        'use of parallelization.',
                        dest='parallel')
    parser.add_argument('--processors', type=int, required=False,
                        default=None,
                        help='Number of processors to be used with the ' +
                        'parallel option.', dest='n_processors')
    args = parser.parse_args()

    problem = read_param_file(args.paramfile)
    Y = np.loadtxt(args.model_output_file, delimiter=args.delimiter,
                   usecols=(args.column,))

    analyze(problem, Y, (args.max_order == 2),
            num_resamples=args.resamples, print_to_console=True,
            parallel=args.parallel, n_processors=args.n_processors)
<|endoftext|>