<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8

'''
A python wrapper of HTCondor allowing to apply pure python functions

* The method map(...) is similar to the standard map function but spawns the
  function execution across condor

* the method imap_unordered is similar to the function itertools.imap
  it works like map(...) but returns an iterator immediately
  the returned data may be unordered

Example:
    def f(x):
        return x**2

    p = CondorPool()
    results = p.map(f, range(5))

If the function has several arguments:
    def g(x,y):
        return x+y

    p = CondorPool()
    results = p.map(g, range(5), range(5,10))

WARNING:
    The mapped function *must* be safely importable: if your main mapping code
    and the target function are contained in the same file, you should make
    sure to enclose your main code (in which the CondorPool class is used) in a
    "if __name__ == '__main__':" section.

HOW IT WORKS:
    A pyro4 server is started to share the inputs/outputs across the machines
    The N inputs are stored in the server, alons with the functions name and
    the file containing it.
    A condor script is written: this module (__main__ in condor.py) acts as the
    worker, and is executed N times by condor - the pyro server address is passed as an
    argument.
    Each worker then connects to the server, gets the inputs and the function
    to run, runs it and stores the results in the server.
    When all jobs are done, the main script returns the values.
'''

import os
import imp
import sys
import socket
import getpass
from multiprocessing import Queue
from time import sleep
from os.path import dirname, basename
from multiprocessing import Process
from os import system
from tmpfiles import Tmp
from sys import argv
import inspect
from string import join as sjoin
from bisect import bisect
from datetime import datetime, timedelta
import Pyro4
import warnings
warnings.filterwarnings('ignore', category=UserWarning, module='Pyro4') # ignore warning "HMAC_KEY not set, protocol data may not be secure"

Pyro4.config.SERVERTYPE = "multiplex"

# the pickle serializer is less safe, but works for all objects
# the default and safe serializer for Pyro4 is 'serpent'
Pyro4.config.SERIALIZER = "pickle"
Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')

#
# Progress bar
#
try:
    from progressbar import ProgressBar, Percentage, Bar, ETA, Counter, Widget
except:
    #
    # define a basic ProgressBar (text progressbar)
    # http://code.google.com/p/python-progressbar/
    # if the module is not available
    #
    from datetime import datetime
    class ProgressBar(object):
        def __init__(self, *args, **kwargs):
            self.__total = kwargs['maxval']
            self.__previous = None
        def start(self): pass
        def update(self, N):
            if N != self.__previous:
                print '[{}] {}/{} results have been received'.format(
                        datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                        N, self.__total)
                self.__previous = N
            pass
        def finish(self):
            self.update(self.__total)
    def Percentage(): pass
    def Bar(): pass
    def ETA(): pass
    def Counter(x): pass
    class Widget(): pass

class Custom(Widget):
    '''
    A custom ProgressBar widget to display arbitrary text
    '''
    def update(self, bar):
        try: return self.__text
        except: return ''
    def set(self, text):
        self.__text = text

class Jobs(object):
    '''
    A class to manage the jobs inputs/outputs
    '''

    # status of jobs
    status_waiting = 0     # waiting  - job has not started
    status_running = 1     # running  - job is running
    status_stored  = 2     # stored   - job has been stored in results queue
    status_fetched  = 3    # fetched  - job has been dequeued

    def __init__(self):
        self.inputs = []
        self.outputs = Queue() # (id, value) pairs
        self.__totaltime = timedelta(0)
        self.__status = []

    def putJob(self, job):

        # the job starts with 'waiting'
        self.__status.append(self.status_waiting)

        self.inputs.append(job)

    def getJob(self, job_id):

        # the job becomes 'running'
        self.__status[job_id] = self.status_running

        return self.inputs[job_id]

    def putResult(self, TUPLE):

        # the job becomes 'stored'
        job_id = TUPLE[0]
        self.__status[job_id] = self.status_stored

        self.outputs.put(TUPLE)

    def getResult(self):

        (job_id, v, t) = self.outputs.get()

        # the job becomes 'fetched'
        self.__status[job_id] = self.status_fetched

        self.__totaltime += t
        return v # return one value

    def getResults_sorted(self):

        results = []
        keys = []
        for _ in xrange(self.nstored()):
            (job_id, v, t) = self.outputs.get()
            self.__status[job_id] = self.status_fetched
            index = bisect(keys, job_id)
            keys.insert(index, job_id)
            results.insert(index, v)
            self.__totaltime += t

        return results

    def nstored(self):
        return self.__status.count(self.status_stored)

    def nfetched(self):
        return self.__status.count(self.status_fetched)

    def nrunning(self):
        return self.__status.count(self.status_running)

    def ndone(self):
        ''' number of finished jobs (stored or fetched) '''
        return self.nstored() + self.nfetched()

    def finished(self, mode):
        '''
        returns whether all the jobs are finished
        'map' mode:  all jobs must be 'stored'
                     (they will be fetched afterwards)
        'imap' mode: all jobs must be 'fetched'
        '''
        if mode == 'map':
            return self.nstored() == self.total()

        elif mode == 'imap':
            # imap: all jobs must be 'fetched'
            return self.nfetched() == self.total()
        else:
            raise Exception('jobs.finished: mode should be either "map" or "imap"')

    def total(self):
        ''' total number of jobs '''
        return len(self.inputs)

    def totaltime(self):
        return self.__totaltime

def pyro_server(jobs, uri_q):

    # initialize the pyro4 daemon
    ip = socket.gethostbyname(socket.gethostname())
    if ip == '127.0.0.1':
        print 'Error retrieving local ip, exiting...'
        exit(1)
    daemon = Pyro4.Daemon(host=ip)

    uri = daemon.register(jobs)
    uri_q.put(uri)

    print 'starting pyro4 daemon at', uri
    daemon.requestLoop()

class Pool(object):
    '''
    This is a generic base class
    '''

    def __init__(self, progressbar=True):
        self.__progressbar = progressbar
        self.__server = None

    def map(self, function, *iterables):

        if len(iterables[0]) == 0:
            return []

        jobs = self._map_async(function, *iterables)

        #
        # wait for the jobs to finish
        #
        t0 = datetime.now()
        try:
            if self.__progressbar:
                custom = Custom()
                pbar = ProgressBar(widgets=[custom,Percentage(),Counter(',%d/'+str(jobs.total())),Bar(),' ',ETA()],
                        maxval=jobs.total())
                pbar.start()
            while not jobs.finished('map'):
                if self.__progressbar:
                    custom.set('[%d running] ' % (jobs.nrunning()))
                    pbar.update(jobs.ndone())
                sleep(2)
            if self.__progressbar:
                custom.set('')
                pbar.finish()
        except KeyboardInterrupt:
            self.__server.terminate()
            print 'interrupted!'
            raise

        #
        # store the results
        #
        results = jobs.getResults_sorted()

        # display total time
        if self.__progressbar:
            totaltime = datetime.now() - t0
            print 'Total time:', totaltime
            print 'Total CPU time:', jobs.totaltime()
            print 'Ratio is %.2f' % (jobs.totaltime().total_seconds()/totaltime.total_seconds())

        #
        # terminate the pyro daemon
        #
        self.__server.terminate()

        # return only the results
        return results

    def imap_unordered(self, function, *iterables):

        if len(iterables[0]) == 0:
            return
            yield

        jobs = self._map_async(function, *iterables)

        if self.__progressbar:
            custom = Custom()
            pbar = ProgressBar(widgets=[custom,Percentage(),Counter(',%d/'+str(jobs.total())),Bar(),' ',ETA()],
                    maxval=jobs.total())
            pbar.start()

        t0 = datetime.now()
        while not jobs.finished('imap'):

            if self.__progressbar:
                custom.set('[%d running] ' % (jobs.nrunning()))
                pbar.update(jobs.ndone())

            if jobs.nstored() > 0:
                yield jobs.getResult()
            else:
                sleep(2)

        # display total time
        if self.__progressbar:
            custom.set('')
            pbar.finish()
            totaltime = datetime.now() - t0
            print 'Total time:', totaltime
            print 'Total CPU time:', jobs.totaltime()
            print 'Ratio is %.2f' % (jobs.totaltime().total_seconds()/totaltime.total_seconds())

        #
        # terminate the pyro daemon
        #
        self.__server.terminate()

    def _map_async(self, function, *iterables):

        if function.func_name == '<lambda>':
            raise Exception('Can not run lambda functions using condor.py')

        #
        # start the pyro daemon in a thread
        #
        uri_q = Queue()
        self.__server = Process(target=pyro_server, args=(Jobs(), uri_q))
        self.__server.start()
        sleep(1)
        uri = uri_q.get()
        uri_q.close()
        jobs = Pyro4.Proxy(uri)

        #
        # initializations
        #

        filename = os.path.abspath(inspect.getfile(function))
        if filename.endswith('.pyc'):
            filename = filename[:-1]
            print 'DEBUG PYC'
        function_str = function.__name__
        print 'Map function "{}" in "{}" with executable "{}"'.format(function_str, filename, sys.executable)

        for args in zip(*iterables):
            jobs.putJob([filename, function_str, args])

        #
        # submit the jobs
        #
        self.submit(jobs, uri)

        return jobs

    def terminate_server(self):
        self.__server.terminate()

    def submit(self, jobs, uri):
        self.terminate_server()
        raise Exception('The Pool class is a base class, not intended to run jobs. Please use a subclass. ')

class CondorPool(Pool):
    '''
    This is a pool using HTCondor system
    '''

    def __init__(self, log='/tmp/condor-log-{}'.format(getpass.getuser()),
            loadavg = 0.8,
            memory = 2000,
            progressbar = True):

        Pool.__init__(self, progressbar=progressbar)

        self.__log = log
        self.__loadavg = loadavg
        self.__memory = memory

    def submit(self, jobs, uri):

        print 'Using condor'
        print 'Log directory is "{}"'.format(self.__log)

        #
        # create log directory if necessary
        #
        if not os.path.exists(self.__log):
            os.mkdir(self.__log)

        condor_header = '''
universe = vanilla
notification = Error
executable = /usr/bin/env
log = {dirlog}/$(Cluster).log
output = {dirlog}/$(Cluster).$(Process).out
error = {dirlog}/$(Cluster).$(Process).error
environment = "LD_LIBRARY_PATH={ld_library_path} PYTHONPATH={pythonpath} PATH={path}"
requirements = (OpSys == "LINUX") && (LoadAvg < {loadavg})
request_memory = {memory}
request_cpus = 1
'''

        condor_job = '''
arguments = "sh -c '{python_exec} -m {worker} {pyro_uri} {job_id}'"
queue
'''

        #
        # create the condor script
        #
        condor_script = Tmp('condor.run')
        fp = open(condor_script, 'w')
        fp.write(condor_header.format(
            pythonpath = sjoin(sys.path,':'),
            path = os.environ.get("PATH", failobj=""),
            ld_library_path = os.environ.get("LD_LIBRARY_PATH", failobj=""),
            dirlog = self.__log,
            memory = self.__memory,
            loadavg = self.__loadavg))
        for i in xrange(jobs.total()):
            fp.write(condor_job.format(
                worker=__name__,
                python_exec = sys.executable,
                pyro_uri=uri,
                job_id=i,
                ))
        fp.close()

        #
        # submit the jobs to condor
        #
        command = 'condor_submit {}'.format(condor_script)
        ret = system(command)
        if ret != 0:
            self.terminate_server()
            condor_script.clean()
            raise Exception('Could not run %s' % (command))

        sleep(3)
        condor_script.clean()

def worker(argv):

    pyro_uri = argv[1]
    job_id = int(argv[2])

    t0 = datetime.now()

    # write a small header at the start of stdout and stderr logs
    msg = '### {} log on {} ###\n'
    sys.stdout.write(msg.format('Output', socket.gethostname()))
    sys.stdout.flush()
    sys.stderr.write(msg.format('Error', socket.gethostname()))
    sys.stderr.flush()

    #
    # connect to the daemon
    #
    jobs = Pyro4.Proxy(pyro_uri)
    filename, function, args = jobs.getJob(job_id)

    #
    # for safety,"cd" to the directory containing the target function
    #
    os.chdir(os.path.dirname(filename))

    #
    # load the target function
    #
    modname = basename(filename)
    if modname.endswith('.py'):
        modname = modname[:-3]

    module = imp.find_module(modname, [dirname(filename)])

    mod = imp.load_module(modname, *module)
    f = getattr(mod, function)

    try:
        result = f(*args)
    except:
        jobs.putResult((job_id, None, datetime.now() - t0))
        raise

    jobs.putResult((job_id, result, datetime.now() - t0))

if __name__ == '__main__':

    worker(argv)

<commit_msg>QsubPool class added<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8

'''
A python wrapper of HTCondor allowing to apply pure python functions

* The method map(...) is similar to the standard map function but spawns the
  function execution across condor

* the method imap_unordered is similar to the function itertools.imap
  it works like map(...) but returns an iterator immediately
  the returned data may be unordered

Example:
    def f(x):
        return x**2

    p = CondorPool()
    results = p.map(f, range(5))

If the function has several arguments:
    def g(x,y):
        return x+y

    p = CondorPool()
    results = p.map(g, range(5), range(5,10))

WARNING:
    The mapped function *must* be safely importable: if your main mapping code
    and the target function are contained in the same file, you should make
    sure to enclose your main code (in which the CondorPool class is used) in a
    "if __name__ == '__main__':" section.

HOW IT WORKS:
    A pyro4 server is started to share the inputs/outputs across the machines
    The N inputs are stored in the server, alons with the functions name and
    the file containing it.
    A condor script is written: this module (__main__ in condor.py) acts as the
    worker, and is executed N times by condor - the pyro server address is passed as an
    argument.
    Each worker then connects to the server, gets the inputs and the function
    to run, runs it and stores the results in the server.
    When all jobs are done, the main script returns the values.
'''

import os
import imp
import sys
import socket
import getpass
from multiprocessing import Queue
from time import sleep
from os.path import dirname, basename
from multiprocessing import Process
from os import system
from tmpfiles import Tmp
from sys import argv
import inspect
from string import join as sjoin
from bisect import bisect
from datetime import datetime, timedelta
import Pyro4
import warnings
warnings.filterwarnings('ignore', category=UserWarning, module='Pyro4') # ignore warning "HMAC_KEY not set, protocol data may not be secure"

Pyro4.config.SERVERTYPE = "multiplex"

# the pickle serializer is less safe, but works for all objects
# the default and safe serializer for Pyro4 is 'serpent'
Pyro4.config.SERIALIZER = "pickle"
Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')

#
# Progress bar
#
try:
    from progressbar import ProgressBar, Percentage, Bar, ETA, Counter, Widget
except:
    #
    # define a basic ProgressBar (text progressbar)
    # http://code.google.com/p/python-progressbar/
    # if the module is not available
    #
    from datetime import datetime
    class ProgressBar(object):
        def __init__(self, *args, **kwargs):
            self.__total = kwargs['maxval']
            self.__previous = None
        def start(self): pass
        def update(self, N):
            if N != self.__previous:
                print '[{}] {}/{} results have been received'.format(
                        datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                        N, self.__total)
                self.__previous = N
            pass
        def finish(self):
            self.update(self.__total)
    def Percentage(): pass
    def Bar(): pass
    def ETA(): pass
    def Counter(x): pass
    class Widget(): pass

class Custom(Widget):
    '''
    A custom ProgressBar widget to display arbitrary text
    '''
    def update(self, bar):
        try: return self.__text
        except: return ''
    def set(self, text):
        self.__text = text

class Jobs(object):
    '''
    A class to manage the jobs inputs/outputs
    '''

    # status of jobs
    status_waiting = 0     # waiting  - job has not started
    status_running = 1     # running  - job is running
    status_stored  = 2     # stored   - job has been stored in results queue
    status_fetched  = 3    # fetched  - job has been dequeued

    def __init__(self):
        self.inputs = []
        self.outputs = Queue() # (id, value) pairs
        self.__totaltime = timedelta(0)
        self.__status = []

    def putJob(self, job):

        # the job starts with 'waiting'
        self.__status.append(self.status_waiting)

        self.inputs.append(job)

    def getJob(self, job_id):

        # the job becomes 'running'
        self.__status[job_id] = self.status_running

        return self.inputs[job_id]

    def putResult(self, TUPLE):

        # the job becomes 'stored'
        job_id = TUPLE[0]
        self.__status[job_id] = self.status_stored

        self.outputs.put(TUPLE)

    def getResult(self):

        (job_id, v, t) = self.outputs.get()

        # the job becomes 'fetched'
        self.__status[job_id] = self.status_fetched

        self.__totaltime += t
        return v # return one value

    def getResults_sorted(self):

        results = []
        keys = []
        for _ in xrange(self.nstored()):
            (job_id, v, t) = self.outputs.get()
            self.__status[job_id] = self.status_fetched
            index = bisect(keys, job_id)
            keys.insert(index, job_id)
            results.insert(index, v)
            self.__totaltime += t

        return results

    def nstored(self):
        return self.__status.count(self.status_stored)

    def nfetched(self):
        return self.__status.count(self.status_fetched)

    def nrunning(self):
        return self.__status.count(self.status_running)

    def ndone(self):
        ''' number of finished jobs (stored or fetched) '''
        return self.nstored() + self.nfetched()

    def finished(self, mode):
        '''
        returns whether all the jobs are finished
        'map' mode:  all jobs must be 'stored'
                     (they will be fetched afterwards)
        'imap' mode: all jobs must be 'fetched'
        '''
        if mode == 'map':
            return self.nstored() == self.total()

        elif mode == 'imap':
            # imap: all jobs must be 'fetched'
            return self.nfetched() == self.total()
        else:
            raise Exception('jobs.finished: mode should be either "map" or "imap"')

    def total(self):
        ''' total number of jobs '''
        return len(self.inputs)

    def totaltime(self):
        return self.__totaltime

def pyro_server(jobs, uri_q):

    # initialize the pyro4 daemon
    ip = socket.gethostbyname(socket.gethostname())
    if ip == '127.0.0.1':
        print 'Error retrieving local ip, exiting...'
        exit(1)
    daemon = Pyro4.Daemon(host=ip)

    uri = daemon.register(jobs)
    uri_q.put(uri)

    print 'starting pyro4 daemon at', uri
    daemon.requestLoop()

class Pool(object):
    '''
    This is a generic base class
    '''

    def __init__(self, progressbar=True):
        self.__progressbar = progressbar
        self.__server = None

    def map(self, function, *iterables):

        if len(iterables[0]) == 0:
            return []

        jobs = self._map_async(function, *iterables)

        #
        # wait for the jobs to finish
        #
        t0 = datetime.now()
        try:
            if self.__progressbar:
                custom = Custom()
                pbar = ProgressBar(widgets=[custom,Percentage(),Counter(',%d/'+str(jobs.total())),Bar(),' ',ETA()],
                        maxval=jobs.total())
                pbar.start()
            while not jobs.finished('map'):
                if self.__progressbar:
                    custom.set('[%d running] ' % (jobs.nrunning()))
                    pbar.update(jobs.ndone())
                sleep(2)
            if self.__progressbar:
                custom.set('')
                pbar.finish()
        except KeyboardInterrupt:
            self.__server.terminate()
            print 'interrupted!'
            raise

        #
        # store the results
        #
        results = jobs.getResults_sorted()

        # display total time
        if self.__progressbar:
            totaltime = datetime.now() - t0
            print 'Total time:', totaltime
            print 'Total CPU time:', jobs.totaltime()
            print 'Ratio is %.2f' % (jobs.totaltime().total_seconds()/totaltime.total_seconds())

        #
        # terminate the pyro daemon
        #
        self.__server.terminate()

        # return only the results
        return results

    def imap_unordered(self, function, *iterables):

        if len(iterables[0]) == 0:
            return
            yield

        jobs = self._map_async(function, *iterables)

        if self.__progressbar:
            custom = Custom()
            pbar = ProgressBar(widgets=[custom,Percentage(),Counter(',%d/'+str(jobs.total())),Bar(),' ',ETA()],
                    maxval=jobs.total())
            pbar.start()

        t0 = datetime.now()
        while not jobs.finished('imap'):

            if self.__progressbar:
                custom.set('[%d running] ' % (jobs.nrunning()))
                pbar.update(jobs.ndone())

            if jobs.nstored() > 0:
                yield jobs.getResult()
            else:
                sleep(2)

        # display total time
        if self.__progressbar:
            custom.set('')
            pbar.finish()
            totaltime = datetime.now() - t0
            print 'Total time:', totaltime
            print 'Total CPU time:', jobs.totaltime()
            print 'Ratio is %.2f' % (jobs.totaltime().total_seconds()/totaltime.total_seconds())

        #
        # terminate the pyro daemon
        #
        self.__server.terminate()

    def _map_async(self, function, *iterables):

        if function.func_name == '<lambda>':
            raise Exception('Can not run lambda functions using condor.py')

        #
        # start the pyro daemon in a thread
        #
        uri_q = Queue()
        self.__server = Process(target=pyro_server, args=(Jobs(), uri_q))
        self.__server.start()
        sleep(1)
        uri = uri_q.get()
        uri_q.close()
        jobs = Pyro4.Proxy(uri)

        #
        # initializations
        #

        filename = os.path.abspath(inspect.getfile(function))
        if filename.endswith('.pyc'):
            filename = filename[:-1]
            print 'DEBUG PYC'
        function_str = function.__name__
        print 'Map function "{}" in "{}" with executable "{}"'.format(function_str, filename, sys.executable)

        for args in zip(*iterables):
            jobs.putJob([filename, function_str, args])

        #
        # submit the jobs
        #
        self.submit(jobs, uri)

        return jobs

    def terminate_server(self):
        self.__server.terminate()

    def submit(self, jobs, uri):
        self.terminate_server()
        raise Exception('The Pool class is a base class, not intended to run jobs. Please use a subclass. ')

class CondorPool(Pool):
    '''
    This is a pool using HTCondor system
    '''

    def __init__(self, log='/tmp/condor-log-{}'.format(getpass.getuser()),
            loadavg = 0.8,
            memory = 2000,
            progressbar = True):

        Pool.__init__(self, progressbar=progressbar)

        self.__log = log
        self.__loadavg = loadavg
        self.__memory = memory

    def submit(self, jobs, uri):

        print 'Using condor'
        print 'Log directory is "{}"'.format(self.__log)

        #
        # create log directory if necessary
        #
        if not os.path.exists(self.__log):
            os.mkdir(self.__log)

        condor_header = '''
universe = vanilla
notification = Error
executable = /usr/bin/env
log = {dirlog}/$(Cluster).log
output = {dirlog}/$(Cluster).$(Process).out
error = {dirlog}/$(Cluster).$(Process).error
environment = "LD_LIBRARY_PATH={ld_library_path} PYTHONPATH={pythonpath} PATH={path}"
requirements = (OpSys == "LINUX") && (LoadAvg < {loadavg})
request_memory = {memory}
request_cpus = 1
'''

        condor_job = '''
arguments = "sh -c '{python_exec} -m {worker} {pyro_uri} {job_id}'"
queue
'''

        #
        # create the condor script
        #
        condor_script = Tmp('condor.run')
        fp = open(condor_script, 'w')
        fp.write(condor_header.format(
            pythonpath = sjoin(sys.path,':'),
            path = os.environ.get("PATH", failobj=""),
            ld_library_path = os.environ.get("LD_LIBRARY_PATH", failobj=""),
            dirlog = self.__log,
            memory = self.__memory,
            loadavg = self.__loadavg))
        for i in xrange(jobs.total()):
            fp.write(condor_job.format(
                worker=__name__,
                python_exec = sys.executable,
                pyro_uri=uri,
                job_id=i,
                ))
        fp.close()

        #
        # submit the jobs to condor
        #
        command = 'condor_submit {}'.format(condor_script)
        ret = system(command)
        if ret != 0:
            self.terminate_server()
            condor_script.clean()
            raise Exception('Could not run %s' % (command))

        sleep(3)
        condor_script.clean()

class QsubPool(Pool):
    '''
    This is a pool using QSUB (SGE) system
    '''

    def __init__(self, log='/tmp/qsub-log-{}'.format(getpass.getuser()),
                 loadavg = 0.8,
                 memory = 2000,
                 progressbar = True):

        Pool.__init__(self, progressbar=progressbar)

        self.__log = log
        self.__loadavg = loadavg
        self.__memory = memory

    def submit(self, jobs, uri):

        print 'Using QSUB'
        print 'Log directory is "{}"'.format(self.__log)

        #
        # create log directory if necessary
        #
        if not os.path.exists(self.__log):
            os.mkdir(self.__log)

        qsub_job = '''
#PBS -S /bin/bash
#PBS -o /home/compiegn/tmp/out
#PBS -e /home/compiegn/tmp/err
# PBS_ENVIRONMENT {ld_library_path}:{pythonpath}:{path}
export LD_LIBRARY_PATH={ld_library_path}
export PYTHONPATH={pythonpath}
export PATH={path}
sh -c '{python_exec} -m {worker} {pyro_uri} {job_id}'
'''

        for i in xrange(jobs.total()):
            #
            # create the qsub script
            #
            with Tmp('qsub.pbs') as qsub_script:
                fp = open(qsub_script, 'w')
                fp.write(qsub_job.format(
                        pythonpath = sjoin(sys.path,':'),
                        path = os.environ.get("PATH", failobj=""),
                        ld_library_path = os.environ.get("LD_LIBRARY_PATH", failobj=""),
                        dirlog = self.__log,
                        memory = self.__memory,
                        loadavg = self.__loadavg,
                        worker=__name__,
                        python_exec = sys.executable,
                        pyro_uri=uri,
                        job_id=i,))

                fp.close()

                #
                # submit the jobs to qsub
                #
                command = 'qsub {}'.format(qsub_script)
                ret = system(command)
                if ret != 0:
                    self.terminate_server()
                    raise Exception('Could not run %s' % (command))

def worker(argv):

    pyro_uri = argv[1]
    job_id = int(argv[2])

    t0 = datetime.now()

    # write a small header at the start of stdout and stderr logs
    msg = '### {} log on {} ###\n'
    sys.stdout.write(msg.format('Output', socket.gethostname()))
    sys.stdout.flush()
    sys.stderr.write(msg.format('Error', socket.gethostname()))
    sys.stderr.flush()

    #
    # connect to the daemon
    #
    jobs = Pyro4.Proxy(pyro_uri)
    filename, function, args = jobs.getJob(job_id)

    #
    # for safety,"cd" to the directory containing the target function
    #
    os.chdir(os.path.dirname(filename))

    #
    # load the target function
    #
    modname = basename(filename)
    if modname.endswith('.py'):
        modname = modname[:-3]

    module = imp.find_module(modname, [dirname(filename)])

    mod = imp.load_module(modname, *module)
    f = getattr(mod, function)

    try:
        result = f(*args)
    except:
        jobs.putResult((job_id, None, datetime.now() - t0))
        raise

    jobs.putResult((job_id, result, datetime.now() - t0))

if __name__ == '__main__':

    worker(argv)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8

'''
A python wrapper of HTCondor allowing to apply pure python functions

* The method map(...) is similar to the standard map function but spawns the
* function execution across condor

* the method imap_unordered is similar to the function itertools.imap
  it works like map(...) but returns an iterator immediately
  the returned data may be unordered
  this is useful in case of a large processing using a reduce function

Example:
    def f(x):
        return x**2

    p = CondorPool()
    results = p.map(f, range(5))

If the function has several arguments:
    def g(x,y):
        return x+y

    p = CondorPool()
    results = p.map(g, range(5), range(5,10))

How it works:
    A pyro4 server is started to share the inputs/outputs across the machines
    The N inputs are stored in the server, alons with the functions name and
    the file containing it.
    A condor script is written: this module (__main__ in condor.py) acts as the
    worker, and is executed N times by condor - the pyro server address is passed as an
    argument.
    Each worker then connects to the server, gets the inputs and the function
    to run, runs it and stores the results in the server.
    When all jobs are done, the main script returns the values.
'''

import os
import imp
import sys
import socket
from multiprocessing import Queue
from time import sleep
from os.path import dirname, basename
from multiprocessing import Process
from os import system
from tmpfiles import Tmp
from sys import argv
import inspect
from string import join as sjoin
import Pyro4
Pyro4.config.SERVERTYPE = "multiplex"

try:
    from progressbar import ProgressBar
    progressbar_available = True
except:
    progressbar_available = False

condor_header = '''
universe = vanilla
notification = Error
executable = {python_exec}
log = {dirlog}/log
output = {dirlog}/$(Process).out
error = {dirlog}/$(Process).error
environment = "PYTHONPATH={pythonpath}"
requirements = (Memory >= {memory}) && (OpSys == "LINUX") && (LoadAvg < {loadavg})
'''

condor_job = '''
arguments = -m {worker} {pyro_uri}
queue
'''

class Jobs(object):

    def __init__(self):
        self.inputs = Queue()
        self.outputs = Queue()
        self.jobsleft = Queue()

    def putJob(self, job):
        self.jobsleft.put(0)
        self.inputs.put(job)

    def getJob(self):
        return self.inputs.get()

    def putResult(self, value):
        self.jobsleft.get()
        self.outputs.put(value)

    def getResult(self):
        return self.outputs.get()

    def resultsLeft(self):
        return self.outputs.qsize()

    def jobsLeft(self):
        return self.jobsleft.qsize()

    def totalLeft(self):
        return self.jobsLeft() + self.resultsLeft()

def pyro_server(jobs, uri_q):

    # initialize the pyro4 daemon
    ip = socket.gethostbyname(socket.gethostname())
    if ip == '127.0.0.1':
        print 'Error retrieving local ip, exiting...'
        exit(1)
    daemon = Pyro4.Daemon(host=ip)

    uri = daemon.register(jobs)
    uri_q.put(uri)

    print 'starting pyro4 daemon at', uri
    daemon.requestLoop()

class CondorPool(object):

    def __init__(self, log='condor-log', loadavg = 0.8, memory = 2000):

        self.__log = log
        self.__loadavg = loadavg
        self.__memory = memory

    def map(self, function, *iterables):

        server, jobs, njobs = self._condor_map_async(function, *iterables)

        #
        # wait for the jobs to finish
        #
        if progressbar_available:
            pbar = ProgressBar(maxval = njobs)
            pbar.start()
        while jobs.jobsLeft() != 0:
            if progressbar_available:
                pbar.update(njobs - jobs.jobsLeft())
            else:
                print '{}/{} results have been received'.format(
                        njobs - jobs.jobsLeft(),
                        njobs)
            sleep(1)
        if progressbar_available:
            pbar.finish()

        #
        # store the results
        #
        results = []
        while jobs.resultsLeft() != 0:
            # get (count, result)
            results.append(jobs.getResult())

        # sort by count
        results.sort(key=lambda x: x[0])

        #
        # terminate the pyro daemon
        #
        server.terminate()

        # return only the results
        return map(lambda x: x[1], results)

    def imap_unordered(self, function, *iterables):

        server, jobs, njobs = self._condor_map_async(function, *iterables)

        while (jobs.totalLeft() != 0):
            result = jobs.getResult()
            yield result[1]

        #
        # terminate the pyro daemon
        #
        server.terminate()

    def _condor_map_async(self, function, *iterables):

        #
        # init jobs object
        #
        jobs = Jobs()

        #
        # start the pyro daemon in a thread
        #
        uri_q = Queue()
        server = Process(target=pyro_server, args=(jobs, uri_q))
        server.start()
        sleep(1)
        uri = uri_q.get()

        #
        # initializations
        #

        filename = os.path.abspath(inspect.getfile(function))
        if filename.endswith('.pyc'):
            filename = filename[:-1]
            print 'DEBUG PYC'
        function_str = function.__name__
        print 'Condor map function "{}"'.format(function_str), 'in', filename, 'with executable', sys.executable

        count = 0
        for args in zip(*iterables):
            jobs.putJob([count, filename, function_str, args])
            count += 1

        njobs = len(iterables[0])

        #
        # create log directory if necessary
        #
        if not os.path.exists(self.__log):
            os.mkdir(self.__log)

        #
        # create the condor script
        #
        condor_script = Tmp('condor.run')
        fp = open(condor_script, 'w')
        fp.write(condor_header.format(
            python_exec = sys.executable,
            pythonpath = sjoin(sys.path,':'),
            dirlog = self.__log,
            memory = self.__memory,
            loadavg = self.__loadavg))
        for i in xrange(njobs):
            fp.write(condor_job.format(
                worker=__name__,
                pyro_uri=uri,
                ))
        fp.close()

        #
        # submit the jobs to condor
        #
        command = 'condor_submit {}'.format(condor_script)
        system(command)
        sleep(3)
        condor_script.clean()

        return server, jobs, njobs

def worker(argv):

    pyro_uri = argv[1]

    #
    # connect to the daemon
    #
    jobs = Pyro4.Proxy(pyro_uri)
    count, filename, function, args = jobs.getJob()

    modname = basename(filename)
    if modname.endswith('.py'):
        modname = modname[:-3]

    module = imp.find_module(modname, [dirname(filename)])

    mod = imp.load_module(modname, *module)
    f = getattr(mod, function)

    try:
        result = f(*args)
    except:
        jobs.putResult((count, None))
        raise

    jobs.putResult((count, result))

if __name__ == '__main__':

    worker(argv)

<commit_msg>Improvement: "cd" to the directory containing the target function.<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8

'''
A python wrapper of HTCondor allowing to apply pure python functions

* The method map(...) is similar to the standard map function but spawns the
  function execution across condor

* the method imap_unordered is similar to the function itertools.imap
  it works like map(...) but returns an iterator immediately
  the returned data may be unordered

Example:
    def f(x):
        return x**2

    p = CondorPool()
    results = p.map(f, range(5))

If the function has several arguments:
    def g(x,y):
        return x+y

    p = CondorPool()
    results = p.map(g, range(5), range(5,10))

WARNING:
    The mapped function *must* be safely importable: if your main mapping code
    and the target function are contained in the same file, you should make
    sure to enclose your main code (in which the CondorPool class is used) in a
    "if __name__ == '__main__':" section.

HOW IT WORKS:
    A pyro4 server is started to share the inputs/outputs across the machines
    The N inputs are stored in the server, alons with the functions name and
    the file containing it.
    A condor script is written: this module (__main__ in condor.py) acts as the
    worker, and is executed N times by condor - the pyro server address is passed as an
    argument.
    Each worker then connects to the server, gets the inputs and the function
    to run, runs it and stores the results in the server.
    When all jobs are done, the main script returns the values.
'''

import os
import imp
import sys
import socket
from multiprocessing import Queue
from time import sleep
from os.path import dirname, basename
from multiprocessing import Process
from os import system
from tmpfiles import Tmp
from sys import argv
import inspect
from string import join as sjoin
import Pyro4
Pyro4.config.SERVERTYPE = "multiplex"

try:
    from progressbar import ProgressBar
    progressbar_available = True
except:
    progressbar_available = False

condor_header = '''
universe = vanilla
notification = Error
executable = {python_exec}
log = {dirlog}/log
output = {dirlog}/$(Process).out
error = {dirlog}/$(Process).error
environment = "PYTHONPATH={pythonpath}"
requirements = (Memory >= {memory}) && (OpSys == "LINUX") && (LoadAvg < {loadavg})
'''

condor_job = '''
arguments = -m {worker} {pyro_uri}
queue
'''

class Jobs(object):

    def __init__(self):
        self.inputs = Queue()
        self.outputs = Queue()
        self.jobsleft = Queue()

    def putJob(self, job):
        self.jobsleft.put(0)
        self.inputs.put(job)

    def getJob(self):
        return self.inputs.get()

    def putResult(self, value):
        self.jobsleft.get()
        self.outputs.put(value)

    def getResult(self):
        return self.outputs.get()

    def resultsLeft(self):
        return self.outputs.qsize()

    def jobsLeft(self):
        return self.jobsleft.qsize()

    def totalLeft(self):
        return self.jobsLeft() + self.resultsLeft()

def pyro_server(jobs, uri_q):

    # initialize the pyro4 daemon
    ip = socket.gethostbyname(socket.gethostname())
    if ip == '127.0.0.1':
        print 'Error retrieving local ip, exiting...'
        exit(1)
    daemon = Pyro4.Daemon(host=ip)

    uri = daemon.register(jobs)
    uri_q.put(uri)

    print 'starting pyro4 daemon at', uri
    daemon.requestLoop()

class CondorPool(object):

    def __init__(self, log='condor-log', loadavg = 0.8, memory = 2000):

        self.__log = log
        self.__loadavg = loadavg
        self.__memory = memory

    def map(self, function, *iterables):

        server, jobs, njobs = self._condor_map_async(function, *iterables)

        #
        # wait for the jobs to finish
        #
        if progressbar_available:
            pbar = ProgressBar(maxval = njobs)
            pbar.start()
        while jobs.jobsLeft() != 0:
            if progressbar_available:
                pbar.update(njobs - jobs.jobsLeft())
            else:
                print '{}/{} results have been received'.format(
                        njobs - jobs.jobsLeft(),
                        njobs)
            sleep(1)
        if progressbar_available:
            pbar.finish()

        #
        # store the results
        #
        results = []
        while jobs.resultsLeft() != 0:
            # get (count, result)
            results.append(jobs.getResult())

        # sort by count
        results.sort(key=lambda x: x[0])

        #
        # terminate the pyro daemon
        #
        server.terminate()

        # return only the results
        return map(lambda x: x[1], results)

    def imap_unordered(self, function, *iterables):

        server, jobs, njobs = self._condor_map_async(function, *iterables)

        while (jobs.totalLeft() != 0):
            result = jobs.getResult()
            yield result[1]

        #
        # terminate the pyro daemon
        #
        server.terminate()

    def _condor_map_async(self, function, *iterables):

        #
        # init jobs object
        #
        jobs = Jobs()

        #
        # start the pyro daemon in a thread
        #
        uri_q = Queue()
        server = Process(target=pyro_server, args=(jobs, uri_q))
        server.start()
        sleep(1)
        uri = uri_q.get()

        #
        # initializations
        #

        filename = os.path.abspath(inspect.getfile(function))
        if filename.endswith('.pyc'):
            filename = filename[:-1]
            print 'DEBUG PYC'
        function_str = function.__name__
        print 'Condor map function "{}"'.format(function_str), 'in', filename, 'with executable', sys.executable

        count = 0
        for args in zip(*iterables):
            jobs.putJob([count, filename, function_str, args])
            count += 1

        njobs = len(iterables[0])

        #
        # create log directory if necessary
        #
        if not os.path.exists(self.__log):
            os.mkdir(self.__log)

        #
        # create the condor script
        #
        condor_script = Tmp('condor.run')
        fp = open(condor_script, 'w')
        fp.write(condor_header.format(
            python_exec = sys.executable,
            pythonpath = sjoin(sys.path,':'),
            dirlog = self.__log,
            memory = self.__memory,
            loadavg = self.__loadavg))
        for i in xrange(njobs):
            fp.write(condor_job.format(
                worker=__name__,
                pyro_uri=uri,
                ))
        fp.close()

        #
        # submit the jobs to condor
        #
        command = 'condor_submit {}'.format(condor_script)
        system(command)
        sleep(3)
        condor_script.clean()

        return server, jobs, njobs

def worker(argv):

    pyro_uri = argv[1]

    #
    # connect to the daemon
    #
    jobs = Pyro4.Proxy(pyro_uri)
    count, filename, function, args = jobs.getJob()

    #
    # for safety,"cd" to the directory containing the target function
    #
    os.chdir(os.path.dirname(filename))

    #
    # load the target function
    #
    modname = basename(filename)
    if modname.endswith('.py'):
        modname = modname[:-3]

    module = imp.find_module(modname, [dirname(filename)])

    mod = imp.load_module(modname, *module)
    f = getattr(mod, function)

    try:
        result = f(*args)
    except:
        jobs.putResult((count, None))
        raise

    jobs.putResult((count, result))

if __name__ == '__main__':

    worker(argv)

<|endoftext|>


<commit_before>import shortuuid
from django.db import models
from django.shortcuts import reverse
from meta.views import Meta
from stencila.schema.util import node_type as schema_node_type

from projects.models.projects import Project
from users.models import User, get_name

def generate_node_key():
    """
    Generate a unique, and very difficult to guess, key to access the node.
    """
    return shortuuid.ShortUUID().random(length=32)

# Applications known to create nodes
# For these provide further details in HTML views of nodes
APPS = {
    "api": ("API", "https://hub.stenci.la/api"),
    "encoda": ("Encoda", "https://github.com/stencila/encoda#readme"),
    "gsuita": (
        "Google Docs",
        "https://workspace.google.com/marketplace/app/stencila/110435422451",
    ),
}

class Node(models.Model):
    """
    A Stencila Schema node.

    Could be any type of node e.g. a `CodeChunk`, `CreativeWork`, `Number` within
    some larger document, or a complete `Article` or `Review`.

    Each node has a unique `key` generated at the time of creation. This
    is the only way to retreive a node.

    Each node can be associated with a `project`. This is for authorization.
    Although the `key` is a secret, project based authorization adds an additional
    layer of security e.g. in case of accidental leakage of a node URL.
    This field does not use cascading delete because node URLs
    should be permananent. The `project` is not required. This allows
    apps to create nodes in documents (e.g. GSuita) or to or to convert documents
    (e.g. Encoda) without having to associate them with a project.

    Each node is created by an `app`. This string is primarily used when generating
    HTML representations of the node to provide links back to that app.

    A node is usually created within a `host`. This is a URL that is primarily used
    when generating HTML representations of the node to provide links back to the
    document.

    The `json` of the node is also immutable. It is returned to requests with
    `Accept: application/json` (if authorized).
    """

    class Meta:
        unique_together = (
            "project",
            "key",
        )

    creator = models.ForeignKey(
        User,
        null=True,  # Should only be null if the creator is deleted
        blank=True,
        on_delete=models.SET_NULL,
        related_name="nodes_created",
        help_text="User who created the project.",
    )

    created = models.DateTimeField(
        auto_now_add=True, help_text="When the project was created."
    )

    project = models.ForeignKey(
        Project,
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="nodes",
        help_text="The project this node is associated with.",
    )

    app = models.TextField(
        null=True,
        blank=True,
        help_text="An identifier for the app that created the node.",
    )

    host = models.URLField(
        null=True,
        blank=True,
        help_text="URL of the host document within which the node was created.",
    )

    key = models.CharField(
        unique=True,
        default=generate_node_key,
        max_length=64,
        help_text="A unique, and very difficult to guess, key to access this node if it is not public.",
    )

    json = models.JSONField(help_text="The JSON content of node.")

    def get_absolute_url(self):
        """Get the URL of this node."""
        return reverse("api-nodes-detail", kwargs={"key": self.key})

    def get_app(self):
        """Get name and URL of the creator app."""
        return APPS.get(self.app, (self.app, None))

    def get_meta(self) -> Meta:
        """Get the metadata to include in the head of the node's page."""
        json = self.json
        node_type = schema_node_type(json)
        if node_type in ("CodeChunk", "CodeExpression"):
            lang = json.get("programmingLanguage", "")
            kind = "code chunk" if node_type == "CodeChunk" else "code expression"
            title = f"{lang.title()} {kind}"
        elif node_type in ("MathBlock", "MathFragment"):
            lang = json.get("mathLanguage", "")
            kind = "math block" if node_type == "MathBlock" else "math fragment"
            title = f"{lang.title()} {kind}"
        else:
            title = node_type + " node"

        description = "Created"
        if self.creator:
            description += f" by { get_name(self.creator) }"

        app_name, app_url = self.get_app()
        if app_name:
            description += f" using { app_name }"

        if self.created:
            description += f" at { self.created } UTC"
        return Meta(object_type="article", title=title, description=description,)
<commit_msg>fix(Nodes): Match using lowercase; change GSuita label<commit_after>import shortuuid
from django.db import models
from django.shortcuts import reverse
from meta.views import Meta
from stencila.schema.util import node_type as schema_node_type

from projects.models.projects import Project
from users.models import User, get_name

def generate_node_key():
    """
    Generate a unique, and very difficult to guess, key to access the node.
    """
    return shortuuid.ShortUUID().random(length=32)

# Applications known to create nodes
# For these provide further details in HTML views of nodes
APPS = {
    "api": ("API", "https://hub.stenci.la/api"),
    "encoda": ("Encoda", "https://github.com/stencila/encoda#readme"),
    "gsuita": (
        "Google Workspace",
        "https://workspace.google.com/marketplace/app/stencila/110435422451",
    ),
}

class Node(models.Model):
    """
    A Stencila Schema node.

    Could be any type of node e.g. a `CodeChunk`, `CreativeWork`, `Number` within
    some larger document, or a complete `Article` or `Review`.

    Each node has a unique `key` generated at the time of creation. This
    is the only way to retreive a node.

    Each node can be associated with a `project`. This is for authorization.
    Although the `key` is a secret, project based authorization adds an additional
    layer of security e.g. in case of accidental leakage of a node URL.
    This field does not use cascading delete because node URLs
    should be permananent. The `project` is not required. This allows
    apps to create nodes in documents (e.g. GSuita) or to or to convert documents
    (e.g. Encoda) without having to associate them with a project.

    Each node is created by an `app`. This string is primarily used when generating
    HTML representations of the node to provide links back to that app.

    A node is usually created within a `host`. This is a URL that is primarily used
    when generating HTML representations of the node to provide links back to the
    document.

    The `json` of the node is also immutable. It is returned to requests with
    `Accept: application/json` (if authorized).
    """

    class Meta:
        unique_together = (
            "project",
            "key",
        )

    creator = models.ForeignKey(
        User,
        null=True,  # Should only be null if the creator is deleted
        blank=True,
        on_delete=models.SET_NULL,
        related_name="nodes_created",
        help_text="User who created the project.",
    )

    created = models.DateTimeField(
        auto_now_add=True, help_text="When the project was created."
    )

    project = models.ForeignKey(
        Project,
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="nodes",
        help_text="The project this node is associated with.",
    )

    app = models.TextField(
        null=True,
        blank=True,
        help_text="An identifier for the app that created the node.",
    )

    host = models.URLField(
        null=True,
        blank=True,
        help_text="URL of the host document within which the node was created.",
    )

    key = models.CharField(
        unique=True,
        default=generate_node_key,
        max_length=64,
        help_text="A unique, and very difficult to guess, key to access this node if it is not public.",
    )

    json = models.JSONField(help_text="The JSON content of node.")

    def get_absolute_url(self):
        """Get the URL of this node."""
        return reverse("api-nodes-detail", kwargs={"key": self.key})

    def get_app(self):
        """Get name and URL of the creator app."""
        return APPS.get(self.app.lower(), (self.app, None))

    def get_meta(self) -> Meta:
        """Get the metadata to include in the head of the node's page."""
        json = self.json
        node_type = schema_node_type(json)
        if node_type in ("CodeChunk", "CodeExpression"):
            lang = json.get("programmingLanguage", "")
            kind = "code chunk" if node_type == "CodeChunk" else "code expression"
            title = f"{lang.title()} {kind}"
        elif node_type in ("MathBlock", "MathFragment"):
            lang = json.get("mathLanguage", "")
            kind = "math block" if node_type == "MathBlock" else "math fragment"
            title = f"{lang.title()} {kind}"
        else:
            title = node_type + " node"

        description = "Created"
        if self.creator:
            description += f" by { get_name(self.creator) }"

        app_name, app_url = self.get_app()
        if app_name:
            description += f" using { app_name }"

        if self.created:
            description += f" at { self.created } UTC"
        return Meta(object_type="article", title=title, description=description,)
<|endoftext|>


<commit_before>import utilities
import subprocess

def create_mangrove_tiles(tile_id):

    print "Getting bounding coordinates for tile", tile_id
    xmin, xmax, ymin, ymax = utilities.coords(tile_id)
    print "ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin

    print "Warping tile"
    out_tile = '{0}_{1}.tif'.format(utilities.mangrove_tile_out, tile_id)
    # cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te',
    #        xmin, ymin, xmax, ymax, '-dstnodata', '-9999', utilities.mangrove_vrt, out_tile]
    # subprocess.check_call(cmd)
    out = '{}_biomass.tif'.format(tile_id)
    warp = ['gdalwarp', '-t_srs', 'EPSG:4326', '-tr', '0.00025', '0.00025', '-te', xmin, ymin, xmax, ymax, 'mangrove_biomass.vrt', out]
    subprocess.check_call(warp)

    print "Tile warped"

    utilities.upload_final(utilities.mangrove_tile_out, utilities.out_dir, tile_id)

<commit_msg>Prepping Loly Fatoyibo's mangrove biomass tifs into Hansen tiles<commit_after>import utilities
import subprocess

def create_mangrove_tiles(tile_id):

    print "Getting bounding coordinates for tile", tile_id
    xmin, xmax, ymin, ymax = utilities.coords(tile_id)
    print "ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin

    print "Warping tile"
    out_tile = '{0}_{1}.tif'.format(utilities.mangrove_tile_out, tile_id)
    # cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te',
    #        xmin, ymin, xmax, ymax, '-dstnodata', '-9999', utilities.mangrove_vrt, out_tile]
    # subprocess.check_call(cmd)
    out = '{}_biomass.tif'.format(tile_id)
    warp = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te',
            xmin, ymin, xmax, ymax, '-dstnodata', '-9999', utilities.mangrove_vrt, out]
    subprocess.check_call(warp)

    print "Tile warped"

    utilities.upload_final(utilities.mangrove_tile_out, utilities.out_dir, tile_id)

<|endoftext|>


<commit_before>from collections import Iterable

import numpy as np

from menpo.shape import PointCloud
from .base import Image

def create_patches_image(patches, patch_centers, patches_indices=None,
                         offset_index=None):
    # Parse inputs
    if offset_index is None:
        offset_index = 0
    if patches_indices is None:
        patches_indices = np.arange(patches.shape[0])
    elif not isinstance(patches_indices, Iterable):
        patches_indices = [patches_indices]

    # Compute patches image's shape
    n_channels = patches.shape[2]
    patch_shape0 = patches.shape[3]
    patch_shape1 = patches.shape[4]
    top, left = np.min(patch_centers.points, 0)
    bottom, right = np.max(patch_centers.points, 0)
    min_0 = np.floor(top - patch_shape0)
    min_1 = np.floor(left - patch_shape1)
    max_0 = np.ceil(bottom + patch_shape0)
    max_1 = np.ceil(right + patch_shape1)
    height = max_0 - min_0 + 1
    width = max_1 - min_1 + 1

    # Translate the patch centers to fit in the new image
    new_patch_centers = patch_centers.copy()
    new_patch_centers.points = patch_centers.points - np.array([[min_0, min_1]])

    # Create temporary pointcloud with the selected patch centers
    tmp_centers = PointCloud(new_patch_centers.points[patches_indices])

    # Create black new image and attach the corrected patch centers
    patches_image = Image.init_blank((height, width), n_channels)
    patches_image.landmarks['all_patch_centers'] = new_patch_centers
    patches_image.landmarks['selected_patch_centers'] = tmp_centers

    # Set the patches
    patches_image.set_patches_around_landmarks(patches[patches_indices],
                                               group='selected_patch_centers',
                                               offset_index=offset_index)

    return patches_image<commit_msg>adds render_rectangles_around_patches<commit_after>from collections import Iterable

import numpy as np

from menpo.shape import PointCloud
from .base import Image

def create_patches_image(patches, patch_centers, patches_indices=None,
                         offset_index=None):
    # Parse inputs
    if offset_index is None:
        offset_index = 0
    if patches_indices is None:
        patches_indices = np.arange(patches.shape[0])
    elif not isinstance(patches_indices, Iterable):
        patches_indices = [patches_indices]

    # Compute patches image's shape
    n_channels = patches.shape[2]
    patch_shape0 = patches.shape[3]
    patch_shape1 = patches.shape[4]
    top, left = np.min(patch_centers.points, 0)
    bottom, right = np.max(patch_centers.points, 0)
    min_0 = np.floor(top - patch_shape0)
    min_1 = np.floor(left - patch_shape1)
    max_0 = np.ceil(bottom + patch_shape0)
    max_1 = np.ceil(right + patch_shape1)
    height = max_0 - min_0 + 1
    width = max_1 - min_1 + 1

    # Translate the patch centers to fit in the new image
    new_patch_centers = patch_centers.copy()
    new_patch_centers.points = patch_centers.points - np.array([[min_0, min_1]])

    # Create temporary pointcloud with the selected patch centers
    tmp_centers = PointCloud(new_patch_centers.points[patches_indices])

    # Create black new image and attach the corrected patch centers
    patches_image = Image.init_blank((height, width), n_channels)
    patches_image.landmarks['all_patch_centers'] = new_patch_centers
    patches_image.landmarks['selected_patch_centers'] = tmp_centers

    # Set the patches
    patches_image.set_patches_around_landmarks(patches[patches_indices],
                                               group='selected_patch_centers',
                                               offset_index=offset_index)

    return patches_image

def render_rectangles_around_patches(centers, patch_shape, axes=None, image_view=True,
                                     line_colour='r', line_style='-', line_width=1):
    import matplotlib.pyplot as plt
    from matplotlib.patches import Rectangle

    # Dictionary with the line styles
    line_style_dict = {'-': 'solid', '--': 'dashed', '-.': 'dashdot',
                       '.': 'dotted'}

    # Get axes object
    if axes is None:
        axes = plt.gca()

    # Need those in order to compute the lower left corner of the rectangle
    half_patch_shape = [patch_shape[0] / 2,
                        patch_shape[1] / 2]

    # Set the view mode
    if image_view:
        xi = 1
        yi = 0
    else:
        xi = 0
        yi = 1

    # Render rectangles
    for p in range(centers.shape[0]):
        xc = np.intp(centers[p, xi] - half_patch_shape[xi]) - 0.5
        yc = np.intp(centers[p, yi] - half_patch_shape[yi]) - 0.5
        axes.add_patch(Rectangle((xc, yc), patch_shape[xi], patch_shape[yi],
                                 fill=False, edgecolor=line_colour,
                                 linewidth=line_width,
                                 linestyle=line_style_dict[line_style]))
<|endoftext|>


<commit_before>import os
from subprocess import CalledProcessError
from pathlib import Path
import shutil
from scipy.io import loadmat
from menpobench import configure_matlab_bin_path
from menpobench.config import (is_linux, is_osx, is_windows,
                               resolve_config_path)
from menpobench.method.base import predefined_method_dir, save_images_to_dir, save_landmarks_to_dir, images_to_mat
from menpobench.utils import invoke_process, TempDirectory

_POTENTIAL_RELEASES = ['2015a', '2014b', '2014a', '2013b', '2013a', '2012b',
                       '2012a']
_DEFAULT_OSX_PATHS = ['/Applications/MATLAB_R{}.app/bin/matlab']
_DEFAULT_WINDOWS_PATHS = [r'C:\Program Files\MATLAB\R{}\bin\matlab.exe',
                          r'C:\Program Files (x86)\MATLAB\R{}\bin\matlab.exe']
_DEFAULT_LINUX_PATHS = ['/usr/local/MATLAB/R{}/bin/matlab']

_matlab_bin_path = None

def matlab_functions_dir():
    return Path(os.path.abspath(__file__)).parent / 'matlab'

def _which_matlab():
    try:
        return invoke_process(['which', 'matlab'])
    except CalledProcessError:
        # Swallow non-zero returncode of which when not found
        pass

def _check_if_matlab_bin_exists(base_paths):
    # Prefer most recent release over older releases.
    for pr in _POTENTIAL_RELEASES:
        for base_path in base_paths:
            p = Path(base_path.format(pr))
            if p.exists():
                return p
    else:
        return ""

def find_matlab_binary():
    if is_linux():
        matlab_path = _which_matlab()
        if not matlab_path:
            matlab_path = _check_if_matlab_bin_exists(_DEFAULT_LINUX_PATHS)
    elif is_osx():
        matlab_path = _which_matlab()
        if not matlab_path:
            matlab_path = _check_if_matlab_bin_exists(_DEFAULT_OSX_PATHS)
    elif is_windows():
        matlab_path = _check_if_matlab_bin_exists(_DEFAULT_WINDOWS_PATHS)
    else:
        # Unknown operating system, just give up - user will have to set
        matlab_path = ""

    return matlab_path

def resolve_matlab_bin_path(verbose=False):
    global _matlab_bin_path
    if _matlab_bin_path is not None:
        return _matlab_bin_path

    from menpobench.utils import load_yaml
    config_path = resolve_config_path()

    try:
        # Will throw key error if matlab_bin_path does not exist
        matlab_bin_path = Path(load_yaml(config_path)['matlab_bin_path'])
    except KeyError:
        # OK, there's no known matlab path, lets be proactive and try and
        # find it ourselves
        matlab_bin_path = find_matlab_binary()
        # If we still failed, re-throw the key-error
        if not matlab_bin_path:
            raise KeyError('matlab_bin_path')
        else:  # Save the found matlab path
            if verbose:
                print('Automatically found Matlab path: {}'.format(matlab_bin_path))
            configure_matlab_bin_path(matlab_bin_path)

    if verbose:
        print('Matlab path: {}'.format(matlab_bin_path))
    # Cache the result so we don't keep querying the rc file
    _matlab_bin_path = matlab_bin_path
    return _matlab_bin_path

def invoke_matlab(command):
    matlab_bin_path = resolve_matlab_bin_path()
    invoke_process(['{}'.format(matlab_bin_path),
                    '-nosplash', '-nodesktop', '-nojvm', '-r',
                    '{}'.format(command)])

def load_matlab_results(results_path):
    return loadmat(str(results_path / 'menpobench_test_results.mat'))

class MatlabWrapper(object):

    def __init__(self, method_path):
        self.method_path = method_path

    def __call__(self, img_generator):
        test_path = TempDirectory.create_new() / 'menpobench_test_images'
        # Save images down to mat file
        images_to_mat(img_generator, test_path)

        # Call matlab bridge to test file - will drop out a result mat
        invoke_matlab("addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
            matlab_functions_dir(), self.method_path, test_path))

        return load_matlab_results(test_path)

def train_matlab_method(method_path, matlab_train_filename,
                        training_images_path):
    # Get absolute path to the train method and copy across to the method
    # folder
    matlab_train_path = predefined_method_dir() / matlab_train_filename
    shutil.copyfile(str(matlab_train_path),
                    str(method_path / 'menpobench_namespace.m'))

    # Call matlab bridge to train file - will drop out a model file
    invoke_matlab("addpath('{}'); menpobench_matlab_train('{}', '{}');".format(
        matlab_functions_dir(), method_path, training_images_path))
<commit_msg>Amend path finding from previous commit<commit_after>import os
from subprocess import CalledProcessError
from pathlib import Path
import shutil
from scipy.io import loadmat
from menpobench import configure_matlab_bin_path
from menpobench.config import (is_linux, is_osx, is_windows,
                               resolve_config_path)
from menpobench.method.base import predefined_method_dir, images_to_mat
from menpobench.utils import invoke_process, TempDirectory

_POTENTIAL_RELEASES = ['2015a', '2014b', '2014a', '2013b', '2013a', '2012b',
                       '2012a']
_DEFAULT_OSX_PATHS = ['/Applications/MATLAB_R{}.app/bin/matlab']
_DEFAULT_WINDOWS_PATHS = [r'C:\Program Files\MATLAB\R{}\bin\matlab.exe',
                          r'C:\Program Files (x86)\MATLAB\R{}\bin\matlab.exe']
_DEFAULT_LINUX_PATHS = ['/usr/local/MATLAB/R{}/bin/matlab']

_matlab_bin_path = None

def matlab_functions_dir():
    return Path(os.path.abspath(__file__)).parent

def _which_matlab():
    try:
        return invoke_process(['which', 'matlab'])
    except CalledProcessError:
        # Swallow non-zero returncode of which when not found
        pass

def _check_if_matlab_bin_exists(base_paths):
    # Prefer most recent release over older releases.
    for pr in _POTENTIAL_RELEASES:
        for base_path in base_paths:
            p = Path(base_path.format(pr))
            if p.exists():
                return p
    else:
        return ""

def find_matlab_binary():
    if is_linux():
        matlab_path = _which_matlab()
        if not matlab_path:
            matlab_path = _check_if_matlab_bin_exists(_DEFAULT_LINUX_PATHS)
    elif is_osx():
        matlab_path = _which_matlab()
        if not matlab_path:
            matlab_path = _check_if_matlab_bin_exists(_DEFAULT_OSX_PATHS)
    elif is_windows():
        matlab_path = _check_if_matlab_bin_exists(_DEFAULT_WINDOWS_PATHS)
    else:
        # Unknown operating system, just give up - user will have to set
        matlab_path = ""

    return matlab_path

def resolve_matlab_bin_path(verbose=False):
    global _matlab_bin_path
    if _matlab_bin_path is not None:
        return _matlab_bin_path

    from menpobench.utils import load_yaml
    config_path = resolve_config_path()

    try:
        # Will throw key error if matlab_bin_path does not exist
        matlab_bin_path = Path(load_yaml(config_path)['matlab_bin_path'])
    except KeyError:
        # OK, there's no known matlab path, lets be proactive and try and
        # find it ourselves
        matlab_bin_path = find_matlab_binary()
        # If we still failed, re-throw the key-error
        if not matlab_bin_path:
            raise KeyError('matlab_bin_path')
        else:  # Save the found matlab path
            if verbose:
                print('Automatically found Matlab path: {}'.format(matlab_bin_path))
            configure_matlab_bin_path(matlab_bin_path)

    if verbose:
        print('Matlab path: {}'.format(matlab_bin_path))
    # Cache the result so we don't keep querying the rc file
    _matlab_bin_path = matlab_bin_path
    return _matlab_bin_path

def invoke_matlab(command):
    matlab_bin_path = resolve_matlab_bin_path()
    invoke_process(['{}'.format(matlab_bin_path),
                    '-nosplash', '-nodesktop', '-nojvm', '-r',
                    '{}'.format(command)])

def load_matlab_results(results_path):
    return loadmat(str(results_path / 'menpobench_test_results.mat'))

class MatlabWrapper(object):

    def __init__(self, method_path):
        self.method_path = method_path

    def __call__(self, img_generator):
        test_path = TempDirectory.create_new() / 'menpobench_test_images'
        # Save images down to mat file
        images_to_mat(img_generator, test_path)

        # Call matlab bridge to test file - will drop out a result mat
        invoke_matlab("addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
            matlab_functions_dir(), self.method_path, test_path))

        return load_matlab_results(test_path)

def train_matlab_method(method_path, matlab_train_filename,
                        training_images_path):
    # Get absolute path to the train method and copy across to the method
    # folder
    matlab_train_path = predefined_method_dir() / matlab_train_filename
    shutil.copyfile(str(matlab_train_path),
                    str(method_path / 'menpobench_namespace.m'))

    # Call matlab bridge to train file - will drop out a model file
    invoke_matlab("addpath('{}'); menpobench_matlab_train('{}', '{}');".format(
        matlab_functions_dir(), method_path, training_images_path))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import platform
import sys

import pytest

ARGS = [
    #'--pdb',
    #'--exitfirst',
]

if platform.system() == 'Windows':
    if 'idlelib' in sys.modules:
        ARGS.extend(['--capture=sys', '--color=no'])

sys.exit(pytest.main(ARGS + sys.argv[1:]))
<commit_msg>make test runner python3 only<commit_after>#!/usr/bin/env python3

import platform
import sys

import pytest

ARGS = [
    #'--pdb',
    #'--exitfirst',
]

if platform.system() == 'Windows':
    if 'idlelib' in sys.modules:
        ARGS.extend(['--capture=sys', '--color=no'])

sys.exit(pytest.main(ARGS + sys.argv[1:]))
<|endoftext|>


<commit_before>import cantera as ct
import sys

def equivalence_ratio(gas,eqRatio,fuel,oxidizer,completeProducts,additionalSpecies,):
    num_H_fuel = 0
    num_C_fuel = 0
    num_O_fuel = 0
    num_H_oxid = 0
    num_C_oxid = 0
    num_O_oxid = 0
    num_H_cprod = 0
    num_C_cprod = 0
    num_O_cprod = 0
    reactants = ''
    #fuel_tot = sum(fuel.values())
    for species, fuel_amt in fuel.items():
        num_H_fuel += gas.n_atoms(species,'H')*fuel_amt
        num_C_fuel += gas.n_atoms(species,'C')*fuel_amt
        num_O_fuel += gas.n_atoms(species,'O')*fuel_amt

    #oxid_tot = sum(oxidizer.values())
    for species, oxid_amt in oxidizer.items():
        num_H_oxid += gas.n_atoms(species,'H')*oxid_amt
        num_C_oxid += gas.n_atoms(species,'C')*oxid_amt
        num_O_oxid += gas.n_atoms(species,'O')*oxid_amt

    num_H_req = num_H_fuel + num_H_oxid
    num_C_req = num_C_fuel + num_C_oxid

    for species in completeProducts:
        num_H_cprod += gas.n_atoms(species,'H')
        num_C_cprod += gas.n_atoms(species,'C')

    if num_H_cprod > 0:
        if num_H_req == 0:
            print('Error: All elements specified in the Complete Products must be in the Fuel or Oxidizer')
            sys.exit(1)

        H_multiplier = num_H_req/num_H_cprod
    else:
        H_multiplier = 0

    if num_C_cprod > 0:
        if num_C_req == 0:
            print('Error: All elements specified in the Complete Products must be in the Fuel or Oxidizer')
            sys.exit(1)

        C_multiplier = num_C_req/num_C_cprod
    else:
        C_multiplier = 0

    for species in completeProducts:
        num_C = gas.n_atoms(species,'C')
        num_H = gas.n_atoms(species,'H')
        num_O = gas.n_atoms(species,'O')
        if num_C > 0:
            num_O_cprod += num_O * C_multiplier
        elif num_H > 0:
            num_O_cprod += num_O * H_multiplier

    O_mult = (num_O_cprod - num_O_fuel)/num_O_oxid

    totalOxidMoles = sum([O_mult * amt for amt in oxidizer.values()])
    totalFuelMoles = sum([eqRatio * amt for amt in fuel.values()])
    totalReactantMoles = totalOxidMoles + totalFuelMoles

    if additionalSpecies:
        totalAdditionalSpecies = sum(additionalSpecies.values())
        if totalAdditionalSpecies >= 1:
            print('Error: Additional species must sum to less than 1')
        remain = 1 - totalAdditionalSpecies
        for species, molefrac in additionalSpecies.items():
            qwer = ':'.join([species,str(molefrac)])
            reactants = ','.join([reactants,qwer])
    else:
        remain = 1
    for species,ox_amt in oxidizer.items():
        molefrac = ox_amt * O_mult/totalReactantMoles * remain
        qwer = ':'.join([species,str(molefrac)])
        reactants = ','.join([reactants,qwer])

    for species, fuel_amt in fuel.items():
        molefrac = fuel_amt * eqRatio /totalReactantMoles * remain
        qwer = ':'.join([species,str(molefrac)])
        reactants = ','.join([reactants,qwer])

    #Take off the first character, which is a comma
    reactants = reactants[1:]

    return reactants,

def run_case(mechFilename,saveFilename,keywords):
    gas = ct.Solution(mechFilename)
    initialTemp = keywords['temperature']
    initialPres = keywords['pressure']*ct.one_atm
    if 'eqRatio' in keywords:
        reactants, = equivalence_ratio(gas,keywords['eqRatio'],keywords['fuel'],
                                      keywords['oxidizer'],
                                      keywords['completeProducts'],
                                      keywords['additionalSpecies'],
                                      )
    else:
        reactants = ','.join(keywords['reactants'])
    gas.TPX = initialTemp, initialPres, reactants

    if keywords['problemType'] == 1:
        reac = ct.Reactor(gas)
    elif keyworkds['problemType'] == 2:
        reac = ct.ConstPressureReactor(gas)

    netw = ct.ReactorNet([reac])
    if 'abstol' in keywords:
        netw.atol = keywords['abstol']
    if 'reltol' in keywords:
        netw.rtol = keywords['reltol']
    tend = keywords['endTime']
    time = 0
    if 'tempLimit' in keywords:
        tempLimit = keywords['tempLimit']
    else:
        tempLimit = keywords['tempThresh'] + keywords['temperature']
    if 'prntTimeInt' in keywords and 'saveTimeInt' in keywords:
        timeStep = min(keywords['prntTimeInt'],keywords['saveTimeInt'])
        timeStepPrint = keywords['prntTimeInt']
        timeStepSave = keywords['saveTimeInt']
    elif 'prntTimeInt' in keywords and 'saveTimeInt' not in keywords:
        timeStep = keywords['prntTimeInt']
        timeStepPrint = keywords['prntTimeInt']
        timeStepSave = 0
    elif 'saveTimeInt' in keywords and 'prntTimeInt' not in keywords:
        timeStep = keywords['saveTimeInt']
        timeStepPrint = tend/100
        timeStepSave = keywords['saveTimeInt']
    else:
        timeStep = tend/100
        timeStepPrint = tend/100
        timeStepSave = tend/100

    #while time < tend:
    #    nextTime = time + timeStep
    #    netw.advance(nextTime)
    #    time = netw.time
    #    print(time,reac.T,reac.thermo.P)
    #    if reac.T > tempLimit:
    #        print(time,reac.T,reac.thermo.P)
    #        break
    while time < tend:
        time = netw.step(tend)
        if reac.T > tempLimit:
            print(time,reac.T,reac.thermo.P)
            break<commit_msg>Fix typo in problem type chooser statement<commit_after>import cantera as ct
import sys

def equivalence_ratio(gas,eqRatio,fuel,oxidizer,completeProducts,additionalSpecies,):
    num_H_fuel = 0
    num_C_fuel = 0
    num_O_fuel = 0
    num_H_oxid = 0
    num_C_oxid = 0
    num_O_oxid = 0
    num_H_cprod = 0
    num_C_cprod = 0
    num_O_cprod = 0
    reactants = ''
    #fuel_tot = sum(fuel.values())
    for species, fuel_amt in fuel.items():
        num_H_fuel += gas.n_atoms(species,'H')*fuel_amt
        num_C_fuel += gas.n_atoms(species,'C')*fuel_amt
        num_O_fuel += gas.n_atoms(species,'O')*fuel_amt

    #oxid_tot = sum(oxidizer.values())
    for species, oxid_amt in oxidizer.items():
        num_H_oxid += gas.n_atoms(species,'H')*oxid_amt
        num_C_oxid += gas.n_atoms(species,'C')*oxid_amt
        num_O_oxid += gas.n_atoms(species,'O')*oxid_amt

    num_H_req = num_H_fuel + num_H_oxid
    num_C_req = num_C_fuel + num_C_oxid

    for species in completeProducts:
        num_H_cprod += gas.n_atoms(species,'H')
        num_C_cprod += gas.n_atoms(species,'C')

    if num_H_cprod > 0:
        if num_H_req == 0:
            print('Error: All elements specified in the Complete Products must be in the Fuel or Oxidizer')
            sys.exit(1)

        H_multiplier = num_H_req/num_H_cprod
    else:
        H_multiplier = 0

    if num_C_cprod > 0:
        if num_C_req == 0:
            print('Error: All elements specified in the Complete Products must be in the Fuel or Oxidizer')
            sys.exit(1)

        C_multiplier = num_C_req/num_C_cprod
    else:
        C_multiplier = 0

    for species in completeProducts:
        num_C = gas.n_atoms(species,'C')
        num_H = gas.n_atoms(species,'H')
        num_O = gas.n_atoms(species,'O')
        if num_C > 0:
            num_O_cprod += num_O * C_multiplier
        elif num_H > 0:
            num_O_cprod += num_O * H_multiplier

    O_mult = (num_O_cprod - num_O_fuel)/num_O_oxid

    totalOxidMoles = sum([O_mult * amt for amt in oxidizer.values()])
    totalFuelMoles = sum([eqRatio * amt for amt in fuel.values()])
    totalReactantMoles = totalOxidMoles + totalFuelMoles

    if additionalSpecies:
        totalAdditionalSpecies = sum(additionalSpecies.values())
        if totalAdditionalSpecies >= 1:
            print('Error: Additional species must sum to less than 1')
        remain = 1 - totalAdditionalSpecies
        for species, molefrac in additionalSpecies.items():
            qwer = ':'.join([species,str(molefrac)])
            reactants = ','.join([reactants,qwer])
    else:
        remain = 1
    for species,ox_amt in oxidizer.items():
        molefrac = ox_amt * O_mult/totalReactantMoles * remain
        qwer = ':'.join([species,str(molefrac)])
        reactants = ','.join([reactants,qwer])

    for species, fuel_amt in fuel.items():
        molefrac = fuel_amt * eqRatio /totalReactantMoles * remain
        qwer = ':'.join([species,str(molefrac)])
        reactants = ','.join([reactants,qwer])

    #Take off the first character, which is a comma
    reactants = reactants[1:]

    return reactants,

def run_case(mechFilename,saveFilename,keywords):
    gas = ct.Solution(mechFilename)
    initialTemp = keywords['temperature']
    initialPres = keywords['pressure']*ct.one_atm
    if 'eqRatio' in keywords:
        reactants, = equivalence_ratio(gas,keywords['eqRatio'],keywords['fuel'],
                                      keywords['oxidizer'],
                                      keywords['completeProducts'],
                                      keywords['additionalSpecies'],
                                      )
    else:
        reactants = ','.join(keywords['reactants'])
    gas.TPX = initialTemp, initialPres, reactants

    if keywords['problemType'] == 1:
        reac = ct.Reactor(gas)
    elif keywords['problemType'] == 2:
        reac = ct.ConstPressureReactor(gas)

    netw = ct.ReactorNet([reac])
    if 'abstol' in keywords:
        netw.atol = keywords['abstol']
    if 'reltol' in keywords:
        netw.rtol = keywords['reltol']
    tend = keywords['endTime']
    time = 0
    if 'tempLimit' in keywords:
        tempLimit = keywords['tempLimit']
    else:
        tempLimit = keywords['tempThresh'] + keywords['temperature']
    if 'prntTimeInt' in keywords and 'saveTimeInt' in keywords:
        timeStep = min(keywords['prntTimeInt'],keywords['saveTimeInt'])
        timeStepPrint = keywords['prntTimeInt']
        timeStepSave = keywords['saveTimeInt']
    elif 'prntTimeInt' in keywords and 'saveTimeInt' not in keywords:
        timeStep = keywords['prntTimeInt']
        timeStepPrint = keywords['prntTimeInt']
        timeStepSave = 0
    elif 'saveTimeInt' in keywords and 'prntTimeInt' not in keywords:
        timeStep = keywords['saveTimeInt']
        timeStepPrint = tend/100
        timeStepSave = keywords['saveTimeInt']
    else:
        timeStep = tend/100
        timeStepPrint = tend/100
        timeStepSave = tend/100

    #while time < tend:
    #    nextTime = time + timeStep
    #    netw.advance(nextTime)
    #    time = netw.time
    #    print(time,reac.T,reac.thermo.P)
    #    if reac.T > tempLimit:
    #        print(time,reac.T,reac.thermo.P)
    #        break
    while time < tend:
        time = netw.step(tend)
        if reac.T > tempLimit:
            print(time,reac.T,reac.thermo.P)
            break<|endoftext|>