<commit_before>from libmyriad import ResultCode
import datetime
import json
import multiprocessing
import os
import psutil
import requests
import shutil
import subprocess
import time
import zipfile

class Myriad:

        def __init__(self):
                self.config = []
                self.maestroAPIGateway = None
                self.myriadJobsFolderOnAWS = None
                self.cpus = 1
                self.mem = 1
                self.displacements = None
                self.jobID = None
                self.jobGUID = None
                self.jobGroup = None
                self.jobCategory = None
                self.makeInputDatParameters = None
                self.jobFolder = None
                self.errors = []
                self.ip = None

        def loadEndpoints(self):
                # Load the configuration values from file
                f = open('config.txt')
                lines = f.readlines()
                f.close()
                for line in lines:
                        if line.startswith('Maestro_api_gateway '):
                                self.maestroAPIGateway = line.split(' ')[1].strip()
                                print('JobRunner GET endpoint set to ' + self.maestroAPIGateway)
                        elif line.startswith('Myriad_AWS '):
                                self.myriadJobsFolderOnAWS = line.split(' ')[1].strip()
                                print('Myriad AWS endpoint set to ' + self.myriadJobsFolderOnAWS)

        def getJob(self, jobGroup=None, jobCategory=None):
                print("Requesting a new job from " + str(self.maestroAPIGateway))
                if jobGroup != None and jobCategory != None:
                        print("Job group set to " + str(jobGroup))
                        print("Job sub group set to " + str(jobCategory))
                        p = {"jobGroup": jobGroup, "jobCategory": jobCategory}
                        r = requests.get(self.maestroAPIGateway, params=p)
                elif jobGroup != None and jobCategory == None:
                        print("Job group set to " + str(jobGroup))
                        p = {"jobGroup": jobGroup}
                        r = requests.get(self.maestroAPIGateway, params=p)
                elif jobGroup == None and jobCategory != None:
                        print("Job sub group set to " + str(jobCategory))
                        p = {"jobCategory": jobCategory}
                        r = requests.get(self.maestroAPIGateway, params=p)
                else:
                        print("No job group or sub group specified")
                        r = requests.get(self.maestroAPIGateway)

                # Check for good HTTP response
                if r.status_code == 200:
                        print("*** Begin get job response ***")
                        print(r.text)
                        print("*** End get job response ***")

                        # Check for logical error in response
                        if not "errorMessage" in r.text:
                                print("Good response:\n" + str(r.text))
                                return self.parseJob(r.text)
                        else:
                                # logic error
                                print("Error from web service:\n" + str(r.text))
                                return ResultCode.failure
                else:
                        # HTTP error
                        print("HTTP error: " + str(r.status_code))
                        return ResultCode.failure

        def parseJob(self, job):
                # The response should look something like this...
                #
                # Old version:
                #    # JobID: 527
                #    # JobGUID: b4af8ced-3661-11e6-a162-12fe8751cda9
                #    # MakeInputDatParameters: -t MTc
                #    -1,1,-2
                #
		# New version:
                #	{
		#	  "JobID": "12345",
		#	  "JobGroup": "NS2",
		#	  "JobCategory": "5Z",
		#	  "JobName": "NS2-5Z-1",
		#	  "JobDefinition": {"Displacements":"-1,-1,-2"},
		#	  "Created": "2016-07-17 15:26:45"
		#	}
                print("Parsing job")
                for line in job.split('\n'):
                	parsed_json = json.loads(job)
                	self.jobID = parsed_json['JobID']
                	self.jobGUID = parsed_json['JobID']
                	self.jobGroup = parsed_json['JobGroup']
                	self.makeInputDatParameters = parsed_json['JobCategory']
                	self.displacements = parsed_json['JobDefinition']['Displacements']
                return ResultCode.success

        def getJobSupportFiles(self):
                result = ResultCode.success
                # download job-specific script(s) to the parent folder
                url = self.myriadJobsFolderOnAWS + "/" + self.jobGroup + "/jobConfig.py"
                print("Retrieving job config from " + url)
                r = requests.get(url)

                # Check for web errors (404, 500, etc.)
                if "<html>" in r.text:
                        print("Bad jobConfig.py")
                        result = ResultCode.failure
                print(r.text)

                f = open("jobConfig.py", "w")
                f.write(r.text)
                f.flush()
                f.close()
                return result

        def getSystemSpecs(self):
                self.cpus = psutil.cpu_count()
                print('Number of cores set to ' + str(self.cpus))
                os.environ["OMP_NUM_THREADS"] = str(self.cpus)
                os.environ["MKL_NUM_THREADS"] = str(self.cpus)
                self.mem = psutil.virtual_memory().available
                print('Bytes of available memory ' + str(self.mem))

        def recordDiskUsage(self):
                myoutput = open('diskspace.out', 'w')
                df = subprocess.Popen("df", stdout=myoutput)
                myoutput.flush()
                myoutput.close()

        def runPsi4(self):
                result = ResultCode.success
                myoutput = open('psi4.out', 'w')
                myerror = open('psi4.err', 'w')
                exitcode = 0
                try:
                        p = subprocess.Popen("psi4", stdout=myoutput, stderr=myerror)
                        waiting = True
                        while waiting:
                                try:
                                        exitcode = p.wait(300)
                                        waiting = False
                                except subprocess.TimeoutExpired:
                                        waiting = True
                                        self.postJobStatus(True, "Running")

                        print("psi4 exited with exit code of " + str(exitcode))
                        if exitcode == 0:
                                result = ResultCode.success
                        else:
                                result = ResultCode.failure

                except RuntimeError as e:
                        self.postJobStatus(False, str(e))
                        result = ResultCode.failure

                finally:
                        myoutput.flush()
                        myerror.flush()
                        myoutput.close()
                        myerror.close()
                        self.recordDiskUsage()

                return result

        def uploadResults(self):
                print("Extracting results from output.dat")
                f = open("output.dat", "r")
                lines = f.readlines()
                energy = None
                for line in reversed(lines):
                        if "CURRENT ENERGY" in line:
                                energy = line.split(">")
                                energy = energy[1].strip()
                                break
                f.close()
                print("Energy = " + str(energy))
                if energy == None:
                        print("No energy found")
                        return ResultCode.noaction

                print("Posting results to the web service at " + str(self.maestroAPIGateway))
                n = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                p = { "jobGUID" : self.jobGUID }
                j = { "completed" : n, "jobResults" : energy }
                r = requests.post(self.maestroAPIGateway, params=p, json=j)
                # Check for good HTTP response
                if r.status_code == 200:
                        # Check for logical error in response
                        if not "errorMessage" in r.text:
                                print("Good response:\n" + str(r.text))
                        else:
                                # logic error
                                print("Error from web service:\n" + str(r.text))
                                return ResultCode.failure
                else:
                        # HTTP error
                        print("HTTP error: " + str(r.status_code))
                        return ResultCode.failure

        def clearScratch(self):
                print("Clearing the scratch folder. Some errors are normal.")
                folder = os.environ['PSI_SCRATCH']
                for the_file in os.listdir(folder):
                        file_path = os.path.join(folder, the_file)
                        try:
                                if os.path.isfile(file_path):
                                        os.unlink(file_path)
                                elif os.path.isdir(file_path):
                                        shutil.rmtree(file_path)
                        except Exception as e:
                             print(e)
                print("Finished clearing the scratch folder.")

        def makeJobFolder(self):
                self.jobFolder = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_"+str(self.jobID))
                os.mkdir(self.jobFolder)
                os.chdir(self.jobFolder)

        def closeJobFolder(self):
                os.chdir("..")

        def makeInputDat(self):
                # Adjust memory value in input.dat
                print("Calculating memory value for input.dat...")
                newmem = "memory " + str(int((self.mem / self.cpus)/1000000)) + " MB"

                # Creates the input.dat file in the job folder
                from jobConfig import JobConfig
                j = JobConfig()
                intder = j.intderIn(self.displacements)
                if intder != None:
                        f = open('intder.in', 'w')
                        f.write(intder)
                        f.flush()
                        f.close()

                        # Run Intder2005 to produce the geometries
                        print("Running Intder2005...")
                        myinput = open('intder.in')
                        myoutput = open('intder.out', 'w')
                        p = subprocess.Popen("Intder2005.x", stdin=myinput, stdout=myoutput)
                        p.wait()
                        myoutput.flush()
                        myoutput.close()
                        print("Finished running Intder2005...")

                        # Read the intder output and produce an input.dat file from the geometries
                        print("Reading file07...")
                        f = open('file07')
                        file07 = f.readlines()
                        f.close
                else:
                        file07 = None

                if len(self.errors) > 0:
                        inputdat = j.inputDat(newmem, self.makeInputDatParameters, file07, self.errors[-1])
                else:
                        inputdat = j.inputDat(newmem, self.makeInputDatParameters, file07)

                # Write input.dat contents to file
                f=open('input.dat', 'w')
                f.write(inputdat)
                # Append print_variables() call as a preventive measure, since that is
                #    where we get the final energy value.
                f.write("\nprint_variables()\n")
                f.flush()
                f.close()
                print("File input.dat written to disk.")

        def postJobStatus(self, status, message=None):
                n = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                print("Posting job status to " + str(self.maestroAPIGateway))
                p = { "jobGUID" : self.jobGUID }
                if status == True:
                        statusStr = "Success"
                else:
                        statusStr = "Failure"
                if message == None:
                        j = {"lastUpdate":n,"status":statusStr}
                else:
                        j = {"lastUpdate":n,"status":statusStr,"message":message}
                try:
                        r = requests.put(self.maestroAPIGateway, params=p, json=j)
                except:
                        print("Error posting status. Ignoring.")
                '''
                # Check for good HTTP response
                if r.status_code == 200:
                        # Check for logical error in response
                        if not "errorMessage" in r.text:
                                print("Good response:\n" + str(r.text))
                        else:
                                # logic error
                                print("Error from web service:\n" + str(r.text))
                else:
                        # HTTP error
                        print("HTTP error: " + str(r.status_code))
                '''

        def checkError(self):
                print("Opening output.dat...")
                f = open("output.dat", "r")
                lines = f.readlines()
                f.close()

                error = None
                for line in reversed(lines):
                        if "Failed to converge." in line:
                                print("Found a 'Failed to converge.' error")
                                error = "Failed to converge."
                        elif error == "Failed to converge." and " iter " in line:
                                # split the line into columns and only look at the Delta E value (fifth column)
                                chunks = line.split()

                                if "e-12" in chunks[4]:
                                        print("Found a 'Failed to converge. (12)' error")
                                        error = "Failed to converge. (12)"
                                if "e-13" in chunks[4]:
                                        print("Found a 'Failed to converge. (13)' error")
                                        error = "Failed to converge. (13)"
                                break
                print("Returning error: " + str(error))
                return error

        def zipJobFolder(self):
                # Get IP address
                f = open('ip.txt')
                self.ip = f.readLine()
                f.close()
                if self.ip == None:
                        self.ip = ""

                try:
                        print("Compressing job folder...")
                        myZipFile = zipfile.ZipFile("ip_" + ip + "_" + jobFolder + ".zip", "w" )
                        listing = os.listdir(jobFolder)
                        for f in listing:
                                myZipFile.write(jobFolder + "/" + f)
                        myZipFile.close()
                        print("Job folder compressed. Removing original...")
                        shutil.rmtree(jobFolder)
                        print("Done removing original job folder")
                except Exception as e:
                        print("Error compressing job folder: " + str(e))

        # Main
        def runOnce(self, jobGroup=None, jobCategory=None, error=None):
                print("Job group = " + str(jobGroup))
                print("Job sub group = " + str(jobCategory))
                print("Error = " + str(error))
                self.jobGroup = jobGroup
                self.jobCategory = jobCategory

                # if we have seen this error before, bail out.
                # We couldn't fix it the first time. Why should this time be any different?
                if error != None and error in self.errors:
                        self.postJobStatus(False, "Error repeated: " + str(error))
                        return ResultCode.failure

                # add the error condition to the stack of prior errors if we've never seen it before
                if error != None:
                        self.errors.append(error)

                # load the endpoints for web service calls
                self.loadEndpoints()

                # if no error, get a new job.
                # if there is an error code, we're going to re-run the job we have
                if error == None:
                        result = self.getJob(self.jobGroup, self.jobCategory)
                else:
                        print("Running current job again to correct for errors")
                        result = ResultCode.success

                if result == ResultCode.success:
                        newerror = None
                        result = self.getJobSupportFiles()
                        if result == ResultCode.success:
                                self.getSystemSpecs()
                                self.clearScratch()
                                self.makeJobFolder()
                                self.makeInputDat()
                                result = self.runPsi4()
                                if result == ResultCode.success:
                                        print("runPsi4() returned success code")
                                        while self.uploadResults() == ResultCode.failure:
                                                print("Failure uploading results. Retrying in 60 seconds...")
                                                time.sleep(60)
                                else:
                                        # Check for known error situations in output.dat
                                        print("runPsi4() returned failure code. Checking for known errors")
                                        newerror = self.checkError()
                                        self.postJobStatus(False, "PSI4 error: " + str(newerror))
                                        print("CheckError() result: " + str(newerror))

                                self.closeJobFolder()
                                self.zipJobFolder()
                                self.clearScratch()

                                # if we encounter a known error, try the job again and compensate
                                if newerror != None:
                                        print("Re-executing job due to known error: " + str(newerror))
                                        result = self.runOnce(self.jobGroup, self.jobCategory, newerror)
                        else:
                                print("Error retrieving support files")

                else:
                        result = ResultCode.noaction

                return result
<commit_msg>Fixed some bad json values<commit_after>from libmyriad import ResultCode
import datetime
import json
import multiprocessing
import os
import psutil
import requests
import shutil
import subprocess
import time
import zipfile

class Myriad:

        def __init__(self):
                self.config = []
                self.maestroAPIGateway = None
                self.myriadJobsFolderOnAWS = None
                self.cpus = 1
                self.mem = 1
                self.displacements = None
                self.jobID = None
                self.jobGUID = None
                self.jobGroup = None
                self.jobCategory = None
                self.makeInputDatParameters = None
                self.jobFolder = None
                self.errors = []
                self.ip = None

        def loadEndpoints(self):
                # Load the configuration values from file
                f = open('config.txt')
                lines = f.readlines()
                f.close()
                for line in lines:
                        if line.startswith('Maestro_api_gateway '):
                                self.maestroAPIGateway = line.split(' ')[1].strip()
                                print('JobRunner GET endpoint set to ' + self.maestroAPIGateway)
                        elif line.startswith('Myriad_AWS '):
                                self.myriadJobsFolderOnAWS = line.split(' ')[1].strip()
                                print('Myriad AWS endpoint set to ' + self.myriadJobsFolderOnAWS)

        def getJob(self, jobGroup=None, jobCategory=None):
                print("Requesting a new job from " + str(self.maestroAPIGateway))
                if jobGroup != None and jobCategory != None:
                        print("Job group set to " + str(jobGroup))
                        print("Job sub group set to " + str(jobCategory))
                        p = {"jobGroup": jobGroup, "jobCategory": jobCategory}
                        r = requests.get(self.maestroAPIGateway, params=p)
                elif jobGroup != None and jobCategory == None:
                        print("Job group set to " + str(jobGroup))
                        p = {"jobGroup": jobGroup}
                        r = requests.get(self.maestroAPIGateway, params=p)
                elif jobGroup == None and jobCategory != None:
                        print("Job sub group set to " + str(jobCategory))
                        p = {"jobCategory": jobCategory}
                        r = requests.get(self.maestroAPIGateway, params=p)
                else:
                        print("No job group or sub group specified")
                        r = requests.get(self.maestroAPIGateway)

                # Check for good HTTP response
                if r.status_code == 200:
                        print("*** Begin get job response ***")
                        print(r.text)
                        print("*** End get job response ***")

                        # Check for logical error in response
                        if not "errorMessage" in r.text:
                                print("Good response:\n" + str(r.text))
                                return self.parseJob(r.text)
                        else:
                                # logic error
                                print("Error from web service:\n" + str(r.text))
                                return ResultCode.failure
                else:
                        # HTTP error
                        print("HTTP error: " + str(r.status_code))
                        return ResultCode.failure

        def parseJob(self, job):
                # The response should look something like this...
                #
                # Old version:
                #    # JobID: 527
                #    # JobGUID: b4af8ced-3661-11e6-a162-12fe8751cda9
                #    # MakeInputDatParameters: -t MTc
                #    -1,1,-2
                #
		# New version:
                #	{
		#	  "JobID": "12345",
		#	  "JobGroup": "NS2",
		#	  "JobCategory": "5Z",
		#	  "JobName": "NS2-5Z-1",
		#	  "JobDefinition": {"Displacements":"-1,-1,-2"},
		#	  "Created": "2016-07-17 15:26:45"
		#	}
                print("Parsing job")
                for line in job.split('\n'):
                	parsed_json = json.loads(job)
                	self.jobID = parsed_json['JobID']
                	self.jobGUID = parsed_json['JobID']
                	self.jobGroup = parsed_json['JobGroup']
                	self.makeInputDatParameters = parsed_json['JobCategory']
                	self.displacements = parsed_json['JobDefinition']['Displacements']
                return ResultCode.success

        def getJobSupportFiles(self):
                result = ResultCode.success
                # download job-specific script(s) to the parent folder
                url = self.myriadJobsFolderOnAWS + "/" + self.jobGroup + "/jobConfig.py"
                print("Retrieving job config from " + url)
                r = requests.get(url)

                # Check for web errors (404, 500, etc.)
                if "<html>" in r.text:
                        print("Bad jobConfig.py")
                        result = ResultCode.failure
                print(r.text)

                f = open("jobConfig.py", "w")
                f.write(r.text)
                f.flush()
                f.close()
                return result

        def getSystemSpecs(self):
                self.cpus = psutil.cpu_count()
                print('Number of cores set to ' + str(self.cpus))
                os.environ["OMP_NUM_THREADS"] = str(self.cpus)
                os.environ["MKL_NUM_THREADS"] = str(self.cpus)
                self.mem = psutil.virtual_memory().available
                print('Bytes of available memory ' + str(self.mem))

        def recordDiskUsage(self):
                myoutput = open('diskspace.out', 'w')
                df = subprocess.Popen("df", stdout=myoutput)
                myoutput.flush()
                myoutput.close()

        def runPsi4(self):
                result = ResultCode.success
                myoutput = open('psi4.out', 'w')
                myerror = open('psi4.err', 'w')
                exitcode = 0
                try:
                        p = subprocess.Popen("psi4", stdout=myoutput, stderr=myerror)
                        waiting = True
                        while waiting:
                                try:
                                        exitcode = p.wait(300)
                                        waiting = False
                                except subprocess.TimeoutExpired:
                                        waiting = True
                                        self.postJobStatus(True, "Running")

                        print("psi4 exited with exit code of " + str(exitcode))
                        if exitcode == 0:
                                result = ResultCode.success
                        else:
                                result = ResultCode.failure

                except RuntimeError as e:
                        self.postJobStatus(False, str(e))
                        result = ResultCode.failure

                finally:
                        myoutput.flush()
                        myerror.flush()
                        myoutput.close()
                        myerror.close()
                        self.recordDiskUsage()

                return result

        def uploadResults(self):
                print("Extracting results from output.dat")
                f = open("output.dat", "r")
                lines = f.readlines()
                energy = None
                for line in reversed(lines):
                        if "CURRENT ENERGY" in line:
                                energy = line.split(">")
                                energy = energy[1].strip()
                                break
                f.close()
                print("Energy = " + str(energy))
                if energy == None:
                        print("No energy found")
                        return ResultCode.noaction

                print("Posting results to the web service at " + str(self.maestroAPIGateway))
                n = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                j = { "JobID" : self.jobGUID, "Completed" : n, "JobResults" : energy }
                r = requests.post(self.maestroAPIGateway, json=j)
                # Check for good HTTP response
                if r.status_code == 200:
                        # Check for logical error in response
                        if not "errorMessage" in r.text:
                                print("Good response:\n" + str(r.text))
                        else:
                                # logic error
                                print("Error from web service:\n" + str(r.text))
                                return ResultCode.failure
                else:
                        # HTTP error
                        print("HTTP error: " + str(r.status_code))
                        return ResultCode.failure

        def clearScratch(self):
                print("Clearing the scratch folder. Some errors are normal.")
                folder = os.environ['PSI_SCRATCH']
                for the_file in os.listdir(folder):
                        file_path = os.path.join(folder, the_file)
                        try:
                                if os.path.isfile(file_path):
                                        os.unlink(file_path)
                                elif os.path.isdir(file_path):
                                        shutil.rmtree(file_path)
                        except Exception as e:
                             print(e)
                print("Finished clearing the scratch folder.")

        def makeJobFolder(self):
                self.jobFolder = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_"+str(self.jobID))
                os.mkdir(self.jobFolder)
                os.chdir(self.jobFolder)

        def closeJobFolder(self):
                os.chdir("..")

        def makeInputDat(self):
                # Adjust memory value in input.dat
                print("Calculating memory value for input.dat...")
                newmem = "memory " + str(int((self.mem / self.cpus)/1000000)) + " MB"

                # Creates the input.dat file in the job folder
                from jobConfig import JobConfig
                j = JobConfig()
                intder = j.intderIn(self.displacements)
                if intder != None:
                        f = open('intder.in', 'w')
                        f.write(intder)
                        f.flush()
                        f.close()

                        # Run Intder2005 to produce the geometries
                        print("Running Intder2005...")
                        myinput = open('intder.in')
                        myoutput = open('intder.out', 'w')
                        p = subprocess.Popen("Intder2005.x", stdin=myinput, stdout=myoutput)
                        p.wait()
                        myoutput.flush()
                        myoutput.close()
                        print("Finished running Intder2005...")

                        # Read the intder output and produce an input.dat file from the geometries
                        print("Reading file07...")
                        f = open('file07')
                        file07 = f.readlines()
                        f.close
                else:
                        file07 = None

                if len(self.errors) > 0:
                        inputdat = j.inputDat(newmem, self.makeInputDatParameters, file07, self.errors[-1])
                else:
                        inputdat = j.inputDat(newmem, self.makeInputDatParameters, file07)

                # Write input.dat contents to file
                f=open('input.dat', 'w')
                f.write(inputdat)
                # Append print_variables() call as a preventive measure, since that is
                #    where we get the final energy value.
                f.write("\nprint_variables()\n")
                f.flush()
                f.close()
                print("File input.dat written to disk.")

        def postJobStatus(self, status, message=None):
                n = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                print("Posting job status to " + str(self.maestroAPIGateway))
                if status == True:
                        statusStr = "Success"
                else:
                        statusStr = "Failure"
                if message == None:
                        j = { "JobID" : self.jobGUID, "LastUpdate":n, "Status":statusStr}
                else:
                        j = { "JobID" : self.jobGUID, "LastUpdate":n, "Status":statusStr, "Message":message}
                try:
                        r = requests.put(self.maestroAPIGateway, json=j)
                except:
                        print("Error posting status. Ignoring.")

        def checkError(self):
                print("Opening output.dat...")
                f = open("output.dat", "r")
                lines = f.readlines()
                f.close()

                error = None
                for line in reversed(lines):
                        if "Failed to converge." in line:
                                print("Found a 'Failed to converge.' error")
                                error = "Failed to converge."
                        elif error == "Failed to converge." and " iter " in line:
                                # split the line into columns and only look at the Delta E value (fifth column)
                                chunks = line.split()

                                if "e-12" in chunks[4]:
                                        print("Found a 'Failed to converge. (12)' error")
                                        error = "Failed to converge. (12)"
                                if "e-13" in chunks[4]:
                                        print("Found a 'Failed to converge. (13)' error")
                                        error = "Failed to converge. (13)"
                                break
                print("Returning error: " + str(error))
                return error

        def zipJobFolder(self):
                # Get IP address
                f = open('ip.txt')
                self.ip = f.readLine()
                f.close()
                if self.ip == None:
                        self.ip = ""

                try:
                        print("Compressing job folder...")
                        myZipFile = zipfile.ZipFile("ip_" + ip + "_" + jobFolder + ".zip", "w" )
                        listing = os.listdir(jobFolder)
                        for f in listing:
                                myZipFile.write(jobFolder + "/" + f)
                        myZipFile.close()
                        print("Job folder compressed. Removing original...")
                        shutil.rmtree(jobFolder)
                        print("Done removing original job folder")
                except Exception as e:
                        print("Error compressing job folder: " + str(e))

        # Main
        def runOnce(self, jobGroup=None, jobCategory=None, error=None):
                print("Job group = " + str(jobGroup))
                print("Job sub group = " + str(jobCategory))
                print("Error = " + str(error))
                self.jobGroup = jobGroup
                self.jobCategory = jobCategory

                # if we have seen this error before, bail out.
                # We couldn't fix it the first time. Why should this time be any different?
                if error != None and error in self.errors:
                        self.postJobStatus(False, "Error repeated: " + str(error))
                        return ResultCode.failure

                # add the error condition to the stack of prior errors if we've never seen it before
                if error != None:
                        self.errors.append(error)

                # load the endpoints for web service calls
                self.loadEndpoints()

                # if no error, get a new job.
                # if there is an error code, we're going to re-run the job we have
                if error == None:
                        result = self.getJob(self.jobGroup, self.jobCategory)
                else:
                        print("Running current job again to correct for errors")
                        result = ResultCode.success

                if result == ResultCode.success:
                        newerror = None
                        result = self.getJobSupportFiles()
                        if result == ResultCode.success:
                                self.getSystemSpecs()
                                self.clearScratch()
                                self.makeJobFolder()
                                self.makeInputDat()
                                result = self.runPsi4()
                                if result == ResultCode.success:
                                        print("runPsi4() returned success code")
                                        while self.uploadResults() == ResultCode.failure:
                                                print("Failure uploading results. Retrying in 60 seconds...")
                                                time.sleep(60)
                                else:
                                        # Check for known error situations in output.dat
                                        print("runPsi4() returned failure code. Checking for known errors")
                                        newerror = self.checkError()
                                        self.postJobStatus(False, "PSI4 error: " + str(newerror))
                                        print("CheckError() result: " + str(newerror))

                                self.closeJobFolder()
                                self.zipJobFolder()
                                self.clearScratch()

                                # if we encounter a known error, try the job again and compensate
                                if newerror != None:
                                        print("Re-executing job due to known error: " + str(newerror))
                                        result = self.runOnce(self.jobGroup, self.jobCategory, newerror)
                        else:
                                print("Error retrieving support files")

                else:
                        result = ResultCode.noaction

                return result
<|endoftext|>


<commit_before>from myrolds import map
from myrolds.item import Item
from myrolds.util import enumerateItems, enumerateExits

class World(object):

    def __init__(self):
        self.scapes = {}
        self.player = None
        startingPlace = None

    def setScapes(self, scapes):
        self.scapes = scapes

    def getScape(self, name):
        return self.scapes[name]

    def putItemInScape(self, item, scape):
        if isinstance(scape, basestring):
            scape = self.scapes[scape]
        scape.addItem(Item.items[item])

    def placeCharacterInScape(self, character, scape, isPlayer=False):
        character.moveTo(scape)
        if isPlayer:
            self.setPlayer(character)

    def setPlayer(self, player):
        self.player = player

class WorldScape(object):
    """
    An abstract scape of the world.
    """
    def __init__(self, id, name="", desc=""):
        self.id = id
        self.name = name
        self.desc = desc
        self.inv = []
        self.gameOver = False

    def enter(self, player):
        if self.gameOver:
            player.gameOver = True

    def addItem(self, item):
        self.inv.append(item)

    def removeItem(self, item):
        self.inv.remove(item)

    def describe(self):
        print self.desc
        visibleItems = [item for item in self.inv if item.isVisible]
        if len(visibleItems) > 1:
            print "There are %s here." % enumerateItems(visibleItems)
        else:
            print "There is %s here." % enumerateItems(visibleItems)

    def getExitName(self):
        return "exit"

    def listExits(self):
        numExits = sum([1 for exit in self.exits if exit is not None])
        print "number of exits:", numExits
        if numExits == 0:
            reply = "There are no %s in any direction." % self.getExitName()
        else:
            if numExits == 1:
                reply = "There is a %s to the " % self.getExitName()
            else:
                reply = "There are %s to the " % self.getExitName()
            exitNames = [map.getDirectionName(map.reverseDirections[index])
                         for index, exit in enumerate(self.exits)
                         if exit is not None]
            reply += enumerateExits(exitNames)
            reply += "."
            print reply

    def describeAndListExits(self):
        self.describe()
        self.listExits()

class Moment(WorldScape):
    """
    A "scape" that takes place in time, rather than physical space.
    """

class Town(WorldScape):

    def addBuilding(self, building):
        pass

class Building(WorldScape):

    def addRoom(self, room):
        pass

class Room(WorldScape):
    """
    A "tile" or "section" of the world. This can be of any size
    """
    def __init__(self, *args, **kwargs):
        super(Room, self).__init__(*args, **kwargs)
        self.exits = [None, None, None, None]

    def __getattr__(self, attr):
        return self.getExits()[attr]

    def getExits(self):
        exitCheks = [
            (key.lower(), self.exists[value]) for key, value in map.directions]
        import pdb;pdb.set_trace()
        return dict(exitChecks)

    def getExitName(self):
        return "door"

class Exit(Room):
    def __init__(self):
        super(Exit, self).__init__("")

    def enter(self,player):
        player.gameOver = True
<commit_msg>* Renamed scape to tile. * Added another method supporting layers.<commit_after>from myrolds import map
from myrolds.item import Item
from myrolds.util import enumerateItems, enumerateExits

# XXX maybe there should be layers *and* a context object. For all operations
# that require a layer to be know, the current layer (context) could just be
# used...
class World(object):

    def __init__(self):
        self.scapes = {}
        self.player = None
        startingPlace = None

    def setScapes(self, scapes):
        # XXX scapes need to know which layer they belong to
        self.scapes = scapes

    def getScape(self, name):
        # XXX from which layer?
        return self.scapes[name]

    def putItemInScape(self, item, scape):
        # XXX items need to know which layer they belong to
        if isinstance(scape, basestring):
            scape = self.scapes[scape]
        scape.addItem(Item.items[item])

    def placeCharacterInScape(self, character, scape, isPlayer=False):
        # XXX characters need to know which layer they belong to
        character.moveTo(scape)
        if isPlayer:
            self.setPlayer(character)

    def setPlayer(self, player):
        self.player = player

    def setLayers(self, layers):
        self.layers = layers

class Tile(object):
    """
    An abstract portion of the world.
    """
    def __init__(self, id, name="", desc=""):
        self.id = id
        self.name = name
        self.desc = desc
        self.inv = []
        self.gameOver = False
        self.doors = [None] * 8

    def enter(self, player):
        if self.gameOver:
            player.gameOver = True

    def addItem(self, item):
        self.inv.append(item)

    def removeItem(self, item):
        self.inv.remove(item)

    def describe(self):
        print self.desc
        visibleItems = [item for item in self.inv if item.isVisible]
        if len(visibleItems) > 1:
            print "There are %s here." % enumerateItems(visibleItems)
        else:
            print "There is %s here." % enumerateItems(visibleItems)

    def getExitName(self):
        return "exit"

    def listExits(self):
        numExits = sum([1 for exit in self.exits if exit is not None])
        if numExits == 0:
            reply = "There are no %s in any direction." % self.getExitName()
        else:
            if numExits == 1:
                reply = "There is a %s to the " % self.getExitName()
            else:
                reply = "There are %ss to the " % self.getExitName()
            exitNames = [map.getDirectionName(map.reverseDirections[index])
                         for index, exit in enumerate(self.exits)
                         if exit is not None]
            reply += enumerateExits(exitNames)
            reply += "."
            print reply

    def describeAndListExits(self):
        self.describe()
        self.listExits()

class Moment(Tile):
    """
    A "scape" that takes place in time, rather than physical space.
    """

class Town(Tile):

    def addBuilding(self, building):
        pass

class Building(Tile):

    def addRoom(self, room):
        pass

class Room(Tile):
    """
    A "tile" or "section" of the world. This can be of any size
    """
    def __init__(self, *args, **kwargs):
        super(Room, self).__init__(*args, **kwargs)
        self.exits = [None, None, None, None]

    def __getattr__(self, attr):
        return self.getExits()[attr]

    def getExits(self):
        exitCheks = [
            (key.lower(), self.exists[value]) for key, value in map.directions]
        import pdb;pdb.set_trace()
        return dict(exitChecks)

    def getExitName(self):
        return "door"

class Exit(Room):
    def __init__(self):
        super(Exit, self).__init__("")

    def enter(self,player):
        player.gameOver = True
<|endoftext|>


<commit_before>from django.conf.urls import url

urlpatterns = [
    # Capture urls that are at the root, or end in a slash.
    url(r'(?P<url>(^|^.*/))$', 'nav_tree.views.node_router'),
]
<commit_msg>Better nav_tree url<commit_after>from django.conf.urls import url

urlpatterns = [
    # Capture urls that are at the root (^$) or end in a slash (^.*/$)
    url(r'^(?P<url>(|.*/))$', 'nav_tree.views.node_router'),
]
<|endoftext|>


<commit_before>import os
import jinja2
import base64
import sendgrid
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()
        my_attachment.content = base64.b64encode(data)
        mail.add_attachment(my_attachment)

    # if for_real:
    #     response = sg.client.mail.send.post(request_body=mail.get())
    #     print u"Sent an email to {}".format(address)
    # else:
    #     print u"Didn't really send"

<commit_msg>over limiit message<commit_after>import os
import jinja2
import base64
import sendgrid
import re
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()

        #handle the spammer
        if re.findall(u"[a-z0-9]{7,15}@gmail.com", address):
            mail = Mail(impactstory_email, "Over limit. Please email us at team@impactstory.org for other data access options.", to_email, content)
            personalization = Personalization()
            personalization.add_to(to_email)
            personalization.add_to(impactstory_email)
            mail.add_personalization(personalization)
        else:
            my_attachment.content = base64.b64encode(data)

        mail.add_attachment(my_attachment)

    # if for_real:
    #     response = sg.client.mail.send.post(request_body=mail.get())
    #     print u"Sent an email to {}".format(address)
    # else:
    #     print u"Didn't really send"

<|endoftext|>


<commit_before>import os
import jinja2
import base64
import sendgrid
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()
        my_attachment.content = base64.b64encode(data)
        mail.add_attachment(my_attachment)

    if for_real:
        response = sg.client.mail.send.post(request_body=mail.get())
        print u"Sent an email to {}".format(address)
    else:
        print u"Didn't really send"

<commit_msg>no email<commit_after>import os
import jinja2
import base64
import sendgrid
from sendgrid.helpers.mail.mail import Email
from sendgrid.helpers.mail.mail import Content
from sendgrid.helpers.mail.mail import Mail
from sendgrid.helpers.mail.mail import Attachment
from sendgrid.helpers.mail.mail import Personalization

def send(address, subject, template_name, context, attachment=None, for_real=False):

    templateLoader = jinja2.FileSystemLoader(searchpath="templates")
    templateEnv = jinja2.Environment(loader=templateLoader)
    html_template = templateEnv.get_template(template_name + ".html")

    html_to_send = html_template.render(context)

    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    impactstory_email = Email("team@impactstory.org", "Impactstory Team")
    to_email = Email(address)
    content = Content("text/html", html_to_send)
    mail = Mail(impactstory_email, subject, to_email, content)
    personalization = Personalization()
    personalization.add_to(to_email)
    personalization.add_to(impactstory_email)
    mail.add_personalization(personalization)

    if attachment:
        my_attachment = Attachment()
        my_attachment.type = "application/csv"
        my_attachment.filename = "results.csv"
        my_attachment.disposition = "attachment"
        my_attachment.content_id = "results csv file"
        with open(attachment, 'rb') as f:
            data = f.read()
        my_attachment.content = base64.b64encode(data)
        mail.add_attachment(my_attachment)

    # if for_real:
    #     response = sg.client.mail.send.post(request_body=mail.get())
    #     print u"Sent an email to {}".format(address)
    # else:
    #     print u"Didn't really send"

<|endoftext|>


<commit_before>from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.auth import get_user_model

from ..privileges import AccessControlMixin

SOURCE_TYPE_CHOICES = [
    ('anime', 'Anime (japanilainen animaatiosarja tai -elokuva)'),
    ('other_anim', 'Muu animaatiosarja'),
    ('manga', 'Manga (japanilainen sarjakuva)'),
    ('other_comic', 'Muu sarjakuva'),
    ('movie', 'Elokuva'),
    ('series', 'TV-sarja'),
    ('game', 'Videopeli'),
    ('original', 'Originaalihahmo'),
    ('other', 'Muu'),
]

class Cosplayer(AccessControlMixin, models.Model):
    event = models.ForeignKey('shoottikala.Event')
    user = models.ForeignKey(settings.AUTH_USER_MODEL)

    is_active = models.BooleanField(default=True)

    display_name = models.CharField(
        max_length=255,
        verbose_name='Näyttönimi',
        help_text=(
            'Millä nimellä haluat esittäytyä valokuvaajille? Esim. nimimerkkisi, etunimesi, '
            'cosplayryhmäsi nimi tai sen jäsenten nimimerkit tai etunimet. Huomaathan, että '
            'tätä kenttää ei ole tarkoitettu hahmon nimelle vaan omallesi.'
        ),
    )

    introduction = models.TextField(
        verbose_name='Esittelyteksti',
        help_text='Esittele itsesi tai ryhmäsi muutamalla lauseella valokuvaajille ja kerro cosplayharrastuksestasi.',
    )

    source = models.CharField(
        max_length=255,
        verbose_name='Lähdeteos',
        blank=True,
        help_text=(
            'Mistä teoksesta hahmosi tai hahmonne ovat peräisin? Esim. animesarjan, elokuvan '
            'tai tietokonepelin nimi.'
        ),
    )

    source_type = models.CharField(
        max_length=max(len(key) for (key, val) in SOURCE_TYPE_CHOICES),
        verbose_name='Lähdeteoksen tyyppi',
        choices=SOURCE_TYPE_CHOICES,
    )

    character = models.CharField(
        max_length=255,
        verbose_name='Hahmon tai hahmojen nimet',
    )

    what_kinda_photos = models.TextField(
        verbose_name='Millaisia kuvia haluat asustasi?',
        help_text=(
            'Jos sinulla on jo tässä vaiheessa ajatuksia siitä, millaisia kuvia haluat – esimerkiksi taustojen, '
            'kuvakulmien tai visuaalisen tyylin suhteen – kerro niistä tässä.'
        ),
        blank=True,
    )

    reference_links = models.TextField(
        verbose_name='Referenssikuvalinkit',
        help_text='Mikäli sinulla on Internetissä referenssikuvia hahmostasi, syötä tähän niiden linkit, yksi per rivi (pelkkä osoite, https://…).',
        blank=True,
    )

    wip_links = models.TextField(
        verbose_name='WIP-kuvalinkit',
        help_text='Mikäli sinulla on Internetissä kuvia asustasi keskeneräisenä tai valmiina, syötä tähän niiden linkit, yksi per rivi (pelkkä osoite, https://…).',
        blank=True,
    )

    social_media_links = models.TextField(
        blank=True,
        verbose_name='Sosiaalisen median linkit',
        help_text='Jos sinulla on cosplay-profiili esim. Facebookissa tai Twitterissä, syötä niiden osoitteet tähän, yksi per rivi (pelkkä osoite, https://…).',
    )

    created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
    updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at'))

    def __str__(self):
        return self.display_name

    def get_absolute_url(self):
        return reverse('shoottikala_cosplayer_view', kwargs=dict(
            event_slug=self.event.slug,
            posting_id=self.id,
        ))

    def build_absolute_uri(self, request):
        return request.build_absolute_uri(self.get_absolute_url())

    def user_can_view(self, user):
        from .photographer import Photographer

        return (
            user.is_superuser or
            self.user == user or
            Photographer.objects.filter(event=self.event, user=self.user).exists()
        )

    @classmethod
    def get_or_create_dummy(cls, event=None, user=None):
        if event is None:
            from .event import Event
            event, unused = Event.get_or_create_dummy()

        if user is None:
            User = get_user_model()
            user, unused = User.get_or_create_dummy()

        return cls.objects.get_or_create(
            event=event,
            user=user,
            defaults=dict(
                display_name=user.display_name,
                introduction='Such dummy, wow',
                source_type='anime',
                character='Dummy',
            ),
        )
<commit_msg>Cosplayer: shorten source labels<commit_after>from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.auth import get_user_model

from ..privileges import AccessControlMixin

SOURCE_TYPE_CHOICES = [
    ('anime', 'Anime'),
    ('other_anim', 'Muu animaatiosarja'),
    ('manga', 'Manga'),
    ('other_comic', 'Muu sarjakuva'),
    ('movie', 'Elokuva'),
    ('series', 'TV-sarja'),
    ('game', 'Videopeli'),
    ('original', 'Originaalihahmo'),
    ('other', 'Muu'),
]

class Cosplayer(AccessControlMixin, models.Model):
    event = models.ForeignKey('shoottikala.Event')
    user = models.ForeignKey(settings.AUTH_USER_MODEL)

    is_active = models.BooleanField(default=True)

    display_name = models.CharField(
        max_length=255,
        verbose_name='Näyttönimi',
        help_text=(
            'Millä nimellä haluat esittäytyä valokuvaajille? Esim. nimimerkkisi, etunimesi, '
            'cosplayryhmäsi nimi tai sen jäsenten nimimerkit tai etunimet. Huomaathan, että '
            'tätä kenttää ei ole tarkoitettu hahmon nimelle vaan omallesi.'
        ),
    )

    introduction = models.TextField(
        verbose_name='Esittelyteksti',
        help_text='Esittele itsesi tai ryhmäsi muutamalla lauseella valokuvaajille ja kerro cosplayharrastuksestasi.',
    )

    source = models.CharField(
        max_length=255,
        verbose_name='Lähdeteos',
        blank=True,
        help_text=(
            'Mistä teoksesta hahmosi tai hahmonne ovat peräisin? Esim. animesarjan, elokuvan '
            'tai tietokonepelin nimi.'
        ),
    )

    source_type = models.CharField(
        max_length=max(len(key) for (key, val) in SOURCE_TYPE_CHOICES),
        verbose_name='Lähdeteoksen tyyppi',
        choices=SOURCE_TYPE_CHOICES,
    )

    character = models.CharField(
        max_length=255,
        verbose_name='Hahmon tai hahmojen nimet',
    )

    what_kinda_photos = models.TextField(
        verbose_name='Millaisia kuvia haluat asustasi?',
        help_text=(
            'Jos sinulla on jo tässä vaiheessa ajatuksia siitä, millaisia kuvia haluat – esimerkiksi taustojen, '
            'kuvakulmien tai visuaalisen tyylin suhteen – kerro niistä tässä.'
        ),
        blank=True,
    )

    reference_links = models.TextField(
        verbose_name='Referenssikuvalinkit',
        help_text='Mikäli sinulla on Internetissä referenssikuvia hahmostasi, syötä tähän niiden linkit, yksi per rivi (pelkkä osoite, https://…).',
        blank=True,
    )

    wip_links = models.TextField(
        verbose_name='WIP-kuvalinkit',
        help_text='Mikäli sinulla on Internetissä kuvia asustasi keskeneräisenä tai valmiina, syötä tähän niiden linkit, yksi per rivi (pelkkä osoite, https://…).',
        blank=True,
    )

    social_media_links = models.TextField(
        blank=True,
        verbose_name='Sosiaalisen median linkit',
        help_text='Jos sinulla on cosplay-profiili esim. Facebookissa tai Twitterissä, syötä niiden osoitteet tähän, yksi per rivi (pelkkä osoite, https://…).',
    )

    created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
    updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at'))

    def __str__(self):
        return self.display_name

    def get_absolute_url(self):
        return reverse('shoottikala_cosplayer_view', kwargs=dict(
            event_slug=self.event.slug,
            posting_id=self.id,
        ))

    def build_absolute_uri(self, request):
        return request.build_absolute_uri(self.get_absolute_url())

    def user_can_view(self, user):
        from .photographer import Photographer

        return (
            user.is_superuser or
            self.user == user or
            Photographer.objects.filter(event=self.event, user=self.user).exists()
        )

    @classmethod
    def get_or_create_dummy(cls, event=None, user=None):
        if event is None:
            from .event import Event
            event, unused = Event.get_or_create_dummy()

        if user is None:
            User = get_user_model()
            user, unused = User.get_or_create_dummy()

        return cls.objects.get_or_create(
            event=event,
            user=user,
            defaults=dict(
                display_name=user.display_name,
                introduction='Such dummy, wow',
                source_type='anime',
                character='Dummy',
            ),
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
#ckwg +5
# Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
# KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
# Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.

def log(msg):
    import sys
    sys.stderr.write("%s\n" % msg)

def ensure_exception(action, func, *args):
    got_exception = False

    try:
        func(*args)
    except:
        got_exception = True

    if not got_exception:
        log("Error: Did not get exception when %s" % action)

def test_import():
    try:
        import vistk.image.vil
    except:
        log("Error: Failed to import the vil module")

def test_create():
    from vistk.image import vil

    try:
        config.empty_config()
    except:
        log("Error: Failed to create an empty configuration")

    config.ConfigKey()
    config.ConfigKeys()
    config.ConfigValue()

def test_vil_to_numpy():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.make_image_bool, np.bool)
            , (test_image.make_image_uint8_t, np.uint8)
            , (test_image.make_image_float, np.float32)
            , (test_image.make_image_double, np.double)
            ]

    for f, t in types:
        i = f(width, height, planes)

        if not i.dtype == t:
            log("Error: Wrong type returned: got: '%s' expected: '%s'" % (i.dtype, t))

        if not i.ndim == 3:
            log("Error: Did not get a 3-dimensional array: got: '%d' expected: '%d'" % (i.dim, 3))

        if not i.shape == shape:
            log("Error: Did not get expected array sizes: got '%s' expected: '%s'" % (str(i.shape), str(shape)))

def test_numpy_to_vil():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    size = width * height * planes

    types = [ (test_image.take_image_bool, np.bool)
            , (test_image.take_image_uint8_t, np.uint8)
            , (test_image.take_image_float, np.float32)
            , (test_image.take_image_double, np.double)
            ]

    for f, t in types:
        i = np.zeros(shape, dtype=t)

        sz = f(i)

        if not sz == size:
            log("Error: Wrong size calculated: got: '%d' expected: '%d'" % (sz, size))

def test_datum():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.save_image_bool, 'bool', np.bool)
            , (test_image.save_image_uint8_t, 'byte', np.uint8)
            # \todo How to save these?
            , (test_image.save_image_float, 'float', np.float32)
            , (test_image.save_image_double, 'double', np.double)
            ]

    for f, pt, t in types:
        a = np.zeros(shape, dtype=t)

        if not f(a, '%s.tiff' % pt):
            log("Error: Failed to save %s image" % pt)

def test_memory():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.make_image_bool, test_image.pass_image_bool, np.bool)
            , (test_image.make_image_uint8_t, test_image.pass_image_uint8_t, np.uint8)
            , (test_image.make_image_float, test_image.pass_image_float, np.float32)
            , (test_image.make_image_double, test_image.pass_image_double, np.double)
            ]

    for _, f, t in types:
        a = np.zeros(shape, dtype=t)

        x = f(a)
        y = f(x)
        z = f(y)
        a = f(z)

    for m, f, t in types:
        a = m(width, height, planes)

        x = f(a)
        y = f(x)
        z = f(y)
        a = f(z)

def main(testname):
    if testname == 'import':
        test_import()
    elif testname == 'vil_to_numpy':
        test_vil_to_numpy()
    elif testname == 'numpy_to_vil':
        test_numpy_to_vil()
    elif testname == 'datum':
        test_datum()
    elif testname == 'memory':
        test_memory()
    else:
        log("Error: No such test '%s'" % testname)

if __name__ == '__main__':
    import os
    import sys

    if not len(sys.argv) == 4:
        log("Error: Expected three arguments")
        sys.exit(1)

    testname = sys.argv[1]

    os.chdir(sys.argv[2])

    sys.path.append(sys.argv[3])

    try:
        main(testname)
    except BaseException as e:
        log("Error: Unexpected exception: %s" % str(e))
<commit_msg>Add code to test vil conversions in the pipeline<commit_after>#!/usr/bin/env python
#ckwg +5
# Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
# KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
# Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.

def log(msg):
    import sys
    sys.stderr.write("%s\n" % msg)

def ensure_exception(action, func, *args):
    got_exception = False

    try:
        func(*args)
    except:
        got_exception = True

    if not got_exception:
        log("Error: Did not get exception when %s" % action)

def test_import():
    try:
        import vistk.image.vil
    except:
        log("Error: Failed to import the vil module")

def test_create():
    from vistk.image import vil

    try:
        config.empty_config()
    except:
        log("Error: Failed to create an empty configuration")

    config.ConfigKey()
    config.ConfigKeys()
    config.ConfigValue()

def test_vil_to_numpy():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.make_image_bool, np.bool)
            , (test_image.make_image_uint8_t, np.uint8)
            , (test_image.make_image_float, np.float32)
            , (test_image.make_image_double, np.double)
            ]

    for f, t in types:
        i = f(width, height, planes)

        if not i.dtype == t:
            log("Error: Wrong type returned: got: '%s' expected: '%s'" % (i.dtype, t))

        if not i.ndim == 3:
            log("Error: Did not get a 3-dimensional array: got: '%d' expected: '%d'" % (i.dim, 3))

        if not i.shape == shape:
            log("Error: Did not get expected array sizes: got '%s' expected: '%s'" % (str(i.shape), str(shape)))

def test_numpy_to_vil():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    size = width * height * planes

    types = [ (test_image.take_image_bool, np.bool)
            , (test_image.take_image_uint8_t, np.uint8)
            , (test_image.take_image_float, np.float32)
            , (test_image.take_image_double, np.double)
            ]

    for f, t in types:
        i = np.zeros(shape, dtype=t)

        sz = f(i)

        if not sz == size:
            log("Error: Wrong size calculated: got: '%d' expected: '%d'" % (sz, size))

def create_verify_process(c, shape, dtype):
    from vistk.pipeline import process

    class VerifyProcess(process.PythonProcess):
        def __init__(self, conf):
            process.PythonProcess.__init__(self, conf)

            self.got_image = False
            self.same_image_type = False
            self.same_image_size = False

            self.input_port = 'image'

            info = process.PortInfo(self.type_any, process.PortFlags(), 'image port')

            self.declare_input_port(self.input_port, info)

        def _step(self):
            from vistk.pipeline import datum
            from vistk.pipeline import edge

            dat = self.grab_datum_from_port(self.input_port)
            img = dat.get_datum()

            if img:
                self.got_image = True
                if dtype == img.dtype:
                    self.same_image_size = True
                if shape == img.shape:
                    self.same_image_size = True

            self._base_step()

        def check(self):
            if not self.got_image:
                log("Error: Could not grab an image from the datum")
            if not self.same_image_type:
                log("Error: Input image was not of the expected type")
            if not self.same_image_size:
                log("Error: Input image was not of the expected size")

    return VerifyProcess(c)

def test_datum():
    from vistk.image import vil
    from vistk.pipeline import modules
    from vistk.pipeline import process_registry
    from vistk.pipeline import schedule_registry
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.save_image_bool, 'bool', np.bool)
            , (test_image.save_image_uint8_t, 'byte', np.uint8)
            , (test_image.save_image_float, 'float', np.float32)
            , (test_image.save_image_double, 'double', np.double)
            ]

    modules.load_known_modules()
    reg = process_registry.ProcessRegistry.self()
    sreg = schedule_registry.ScheduleRegistry.self()

    sched_type = 'sync'

    for f, pt, t in types:
        from vistk.pipeline import config
        from vistk.pipeline import pipeline
        from vistk.pipeline import process

        a = np.zeros(shape, dtype=t)

        lname = 'test-python-vil-datum-%s.txt' % pt
        fname = 'test-python-vil-datum-%s.tiff' % pt

        if not f(a, fname):
            log("Error: Failed to save '%s' image" % pt)
            continue

        with open(lname, 'w+') as f:
            f.write('%s\n' % fname)

        c = config.empty_config()

        p = pipeline.Pipeline(c)

        read_name = 'read'
        verify_name = 'verify'

        c['input'] = lname
        c['pixtype'] = pt
        c['pixfmt'] = 'rgb'
        c['verify'] = 'true'
        c[process.PythonProcess.config_name] = read_name

        proc_type = 'image_reader'

        r = reg.create_process(proc_type, c)

        c[process.PythonProcess.config_name] = verify_name

        v = create_verify_process(c, shape, t)

        p.add_process(r)
        p.add_process(v)

        port = 'image'

        p.connect(read_name, port,
                  verify_name, port)

        try:
            p.setup_pipeline()
        except BaseException as e:
            log("Error: Could not initialize pipeline: '%s'" % str(e))
            continue

        s = sreg.create_schedule(sched_type, c, p)

        try:
            s.start()
            s.wait()
        except BaseException as e:
            log("Error: Could not execute pipeline: '%s'" % str(e))

        v.check()

def test_memory():
    from vistk.image import vil
    from vistk.test import test_image
    import numpy as np

    width = 800
    height = 600
    planes = 3

    shape = (width, height, planes)

    types = [ (test_image.make_image_bool, test_image.pass_image_bool, np.bool)
            , (test_image.make_image_uint8_t, test_image.pass_image_uint8_t, np.uint8)
            , (test_image.make_image_float, test_image.pass_image_float, np.float32)
            , (test_image.make_image_double, test_image.pass_image_double, np.double)
            ]

    for _, f, t in types:
        a = np.zeros(shape, dtype=t)

        x = f(a)
        y = f(x)
        z = f(y)
        a = f(z)

    for m, f, t in types:
        a = m(width, height, planes)

        x = f(a)
        y = f(x)
        z = f(y)
        a = f(z)

def main(testname):
    if testname == 'import':
        test_import()
    elif testname == 'vil_to_numpy':
        test_vil_to_numpy()
    elif testname == 'numpy_to_vil':
        test_numpy_to_vil()
    elif testname == 'datum':
        test_datum()
    elif testname == 'memory':
        test_memory()
    else:
        log("Error: No such test '%s'" % testname)

if __name__ == '__main__':
    import os
    import sys

    if not len(sys.argv) == 4:
        log("Error: Expected three arguments")
        sys.exit(1)

    testname = sys.argv[1]

    os.chdir(sys.argv[2])

    sys.path.append(sys.argv[3])

    try:
        main(testname)
    except BaseException as e:
        log("Error: Unexpected exception: %s" % str(e))
<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import StringIO

from telemetry.internal.backends.chrome_inspector import websocket

class InspectorConsole(object):
  def __init__(self, inspector_websocket):
    self._inspector_websocket = inspector_websocket
    self._inspector_websocket.RegisterDomain('Console', self._OnNotification)
    self._message_output_stream = None
    self._last_message = None
    self._console_enabled = False

  def _OnNotification(self, msg):
    if msg['method'] == 'Console.messageAdded':
      assert self._message_output_stream
      if msg['params']['message']['url'] == 'chrome://newtab/':
        return
      self._last_message = '(%s) %s:%i: %s' % (
          msg['params']['message']['level'],
          msg['params']['message']['url'],
          msg['params']['message']['line'],
          msg['params']['message']['text'])
      self._message_output_stream.write(
          '%s\n' % self._last_message)

    elif msg['method'] == 'Console.messageRepeatCountUpdated':
      if self._message_output_stream:
        self._message_output_stream.write('%s\n' % self._last_message)

  def GetCurrentConsoleOutputBuffer(self, timeout=10):
    self._message_output_stream = StringIO.StringIO()
    self._EnableConsoleOutputStream(timeout)
    try:
      self._inspector_websocket.DispatchNotifications(timeout)
      return self._message_output_stream.getvalue()
    except websocket.WebSocketTimeoutException:
      return self._message_output_stream.getvalue()
    finally:
      self._DisableConsoleOutputStream(timeout)
      self._message_output_stream.close()
      self._message_output_stream = None

  def _EnableConsoleOutputStream(self, timeout):
    self._inspector_websocket.SyncRequest({'method': 'Console.enable'}, timeout)

  def _DisableConsoleOutputStream(self, timeout):
    self._inspector_websocket.SyncRequest(
        {'method': 'Console.disable'}, timeout)
<commit_msg>[Telemetry] Remove dead code from console logging system.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import StringIO

from telemetry.internal.backends.chrome_inspector import websocket

class InspectorConsole(object):
  def __init__(self, inspector_websocket):
    self._inspector_websocket = inspector_websocket
    self._inspector_websocket.RegisterDomain('Console', self._OnNotification)
    self._message_output_stream = None
    self._last_message = None
    self._console_enabled = False

  def _OnNotification(self, msg):
    if msg['method'] == 'Console.messageAdded':
      assert self._message_output_stream
      if msg['params']['message']['url'] == 'chrome://newtab/':
        return
      self._last_message = '(%s) %s:%i: %s' % (
          msg['params']['message']['level'],
          msg['params']['message']['url'],
          msg['params']['message']['line'],
          msg['params']['message']['text'])
      self._message_output_stream.write(
          '%s\n' % self._last_message)

  def GetCurrentConsoleOutputBuffer(self, timeout=10):
    self._message_output_stream = StringIO.StringIO()
    self._EnableConsoleOutputStream(timeout)
    try:
      self._inspector_websocket.DispatchNotifications(timeout)
      return self._message_output_stream.getvalue()
    except websocket.WebSocketTimeoutException:
      return self._message_output_stream.getvalue()
    finally:
      self._DisableConsoleOutputStream(timeout)
      self._message_output_stream.close()
      self._message_output_stream = None

  def _EnableConsoleOutputStream(self, timeout):
    self._inspector_websocket.SyncRequest({'method': 'Console.enable'}, timeout)

  def _DisableConsoleOutputStream(self, timeout):
    self._inspector_websocket.SyncRequest(
        {'method': 'Console.disable'}, timeout)
<|endoftext|>


<commit_before>from argparse import ArgumentParser
from collections import Counter

import matplotlib.pyplot as plt

from graph import RGG, HypRG

def main():
    parser = ArgumentParser()
    parser.add_argument('-n', type=int, help='number of vertices', default=100)
    parser.add_argument('--alpha', type=float, help='alpha', default=1.)
    parser.add_argument('-C', type=float, help='C', default=0.)
    if False:
        rgg = RGG(
            t=100,
            #cluster_attachment='uniform'
        )
        print rgg.clusters
        print sorted(rgg.degrees())

    args = parser.parse_args()
    g = HypRG(args.n,alpha=args.alpha)
    v = g.vertices()
    e = g.edges()
    # vertices
    print(len(v))
    r, phi = zip(*v)
    ax = plt.subplot(111, projection='polar')
    ax.plot(phi, r, marker='o', linewidth=0)
    # edges
    print(len(e))
    for (v1, v2) in e:
        r1, phi1 = v1
        r2, phi2 = v2
        ax.plot((phi1, phi2), (r1, r2), color='g')

    plt.show()
    #deg = g.degrees()
    #print Counter(deg)

if __name__ == '__main__':
    main()
<commit_msg>visualization<commit_after>from argparse import ArgumentParser
from collections import Counter

import matplotlib.pyplot as plt

from graph import HypRG

def main():
    parser = ArgumentParser()
    parser.add_argument('-n', type=int, help='number of vertices', default=100)
    parser.add_argument('--alpha', type=float, help='alpha', default=1.)
    parser.add_argument('-C', type=float, help='C', default=0.)

    args = parser.parse_args()
    n = args.n
    alpha = args.alpha
    g = HypRG(n,alpha=alpha)
    v = g.vertices()
    e = g.edges()

    # vertices
    print "number of vertices: {}".format(len(v))
    r, phi = zip(*v)
    ax = plt.subplot(121, projection='polar')
    ax.plot(phi, r, marker='o', linewidth=0)

    # edges
    print "number of edges: {}".format(len(e))
    for (v1, v2) in e:
        r1, phi1 = v1
        r2, phi2 = v2
        ax.plot((phi1, phi2), (r1, r2), color='g')

    deg = g.degrees()
    n_isolated_vertices = deg[0]
    deg_dist = [i for i in Counter(deg).iteritems() if i[0] > 0]
    degs, counts = zip(*deg_dist)
    freqs = [float(c) / n for c in counts]
    ax_deg = plt.subplot(122)
    ax_deg.set_xscale("log")
    ax_deg.set_yscale("log")
    ax_deg.errorbar(degs, freqs)

    print "size of giant component: {}".format(n - n_isolated_vertices)
    plt.show()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from simphony.core.cuba import CUBA

_CUBA_MEMBERS = CUBA.__members__
_CUBA_KEYS = set(CUBA)

class DataContainer(dict):
    """ A DataContainer instance

    The DataContainer object is implemented as a python dictionary whose keys
    are restricted to be members of the CUBA enum class.

    The data container can be initialized like a typical python dict
    using the mapping and iterables where the keys are CUBA enum members.

    For convenience keywords can be passed as capitalized CUBA enum members::

        >>> DataContainer(ACCELERATION=234)  # CUBA.ACCELERATION is 19
        {<CUBA.ACCELERATION: 19>: 234}

    """

    # Memory usage optimization.
    __slots__ = ()

    def __init__(self, *args, **kwards):
        """ Constructor.

        Initialization follows the behaviour of the python dict class.

        """
        self._check_arguments(args, kwards)
        if len(args) == 1 and not hasattr(args[0], 'keys'):
            super(DataContainer, self).__init__()
            for key, value in args[0]:
                self.__setitem__(key, value)
        elif len(args) == 1:
            mapping = args[0]
            if not isinstance(mapping, DataContainer):
                if any(not isinstance(key, CUBA) for key in mapping):
                    non_cuba_keys = [
                        key for key in mapping if not isinstance(key, CUBA)]
                    message = \
                        "Key(s) {!r} are not in the approved CUBA keywords"
                    raise ValueError(message.format(non_cuba_keys))
            super(DataContainer, self).__init__(mapping)
        super(DataContainer, self).update(
            {CUBA[kward]: value for kward, value in kwards.viewitems()})

    def __setitem__(self, key, value):
        """ Set/Update the key value only when the key is a CUBA key.

        """
        if isinstance(key, CUBA):
            super(DataContainer, self).__setitem__(key, value)
        else:
            message = "Key {!r} is not in the approved CUBA keywords"
            raise ValueError(message.format(key))

    def update(self, *args, **kwards):
        self._check_arguments(args, kwards)
        if len(args) == 1 and not hasattr(args[0], 'keys'):
            for key, value in args[0]:
                self.__setitem__(key, value)
        elif len(args) == 1:
            mapping = args[0]
            if not isinstance(mapping, DataContainer):
                if any(not isinstance(key, CUBA) for key in mapping):
                    non_cuba_keys = [
                        key for key in mapping if not isinstance(key, CUBA)]
                    message = \
                        "Key(s) {!r} are not in the approved CUBA keywords"
                    raise ValueError(message.format(non_cuba_keys))
            super(DataContainer, self).update(mapping)
        super(DataContainer, self).update(
            {CUBA[kward]: value for kward, value in kwards.viewitems()})

    def _check_arguments(self, args, kwards):
        """ Check for the right arguments.

        """
        # See if there are any non CUBA keys in the keyword arguments
        if any(key not in _CUBA_MEMBERS for key in kwards):
            non_cuba_keys = kwards.viewkeys() - _CUBA_MEMBERS.viewkeys()
            message = "Key(s) {!r} are not in the approved CUBA keywords"
            raise ValueError(message.format(non_cuba_keys))
        # Only one positional argument is allowed.
        if len(args) > 1:
            message = 'DataContainer expected at most 1 arguments, got {}'
            raise TypeError(message.format(len(args)))
<commit_msg>Update data_container.py<commit_after>from simphony.core.cuba import CUBA

_CUBA_MEMBERS = CUBA.__members__

class DataContainer(dict):
    """ A DataContainer instance

    The DataContainer object is implemented as a python dictionary whose keys
    are restricted to be members of the CUBA enum class.

    The data container can be initialized like a typical python dict
    using the mapping and iterables where the keys are CUBA enum members.

    For convenience keywords can be passed as capitalized CUBA enum members::

        >>> DataContainer(ACCELERATION=234)  # CUBA.ACCELERATION is 19
        {<CUBA.ACCELERATION: 19>: 234}

    """

    # Memory usage optimization.
    __slots__ = ()

    def __init__(self, *args, **kwargs):
        """ Constructor.

        Initialization follows the behaviour of the python dict class.

        """
        self._check_arguments(args, kwargs)
        if len(args) == 1 and not hasattr(args[0], 'keys'):
            super(DataContainer, self).__init__()
            for key, value in args[0]:
                self.__setitem__(key, value)
        elif len(args) == 1:
            mapping = args[0]
            if not isinstance(mapping, DataContainer):
                if any(not isinstance(key, CUBA) for key in mapping):
                    non_cuba_keys = [
                        key for key in mapping if not isinstance(key, CUBA)]
                    message = \
                        "Key(s) {!r} are not in the approved CUBA keywords"
                    raise ValueError(message.format(non_cuba_keys))
            super(DataContainer, self).__init__(mapping)
        super(DataContainer, self).update(
            {CUBA[kwarg]: value for kwarg, value in kwargs.viewitems()})

    def __setitem__(self, key, value):
        """ Set/Update the key value only when the key is a CUBA key.

        """
        if isinstance(key, CUBA):
            super(DataContainer, self).__setitem__(key, value)
        else:
            message = "Key {!r} is not in the approved CUBA keywords"
            raise ValueError(message.format(key))

    def update(self, *args, **kwargs):
        self._check_arguments(args, kwargs)
        if len(args) == 1 and not hasattr(args[0], 'keys'):
            for key, value in args[0]:
                self.__setitem__(key, value)
        elif len(args) == 1:
            mapping = args[0]
            if not isinstance(mapping, DataContainer):
                if any(not isinstance(key, CUBA) for key in mapping):
                    non_cuba_keys = [
                        key for key in mapping if not isinstance(key, CUBA)]
                    message = \
                        "Key(s) {!r} are not in the approved CUBA keywords"
                    raise ValueError(message.format(non_cuba_keys))
            super(DataContainer, self).update(mapping)
        super(DataContainer, self).update(
            {CUBA[kwarg]: value for kwarg, value in kwards.viewitems()})

    def _check_arguments(self, args, kwards):
        """ Check for the right arguments.

        """
        # See if there are any non CUBA keys in the keyword arguments
        if any(key not in _CUBA_MEMBERS for key in kwargs):
            non_cuba_keys = kwargs.viewkeys() - _CUBA_MEMBERS.viewkeys()
            message = "Key(s) {!r} are not in the approved CUBA keywords"
            raise ValueError(message.format(non_cuba_keys))
        # Only one positional argument is allowed.
        if len(args) > 1:
            message = 'DataContainer expected at most 1 arguments, got {}'
            raise TypeError(message.format(len(args)))
<|endoftext|>