<commit_before>"""
High-level user-level commands controlling the entire simulation.
$Id$
"""
__version__='$Revision$'

import cPickle as pickle
import gnosis.xml.pickle
from xml.parsers.expat import ExpatError

import time
import string
import re
import os

import __main__

# gzip module might not have been built (if zlib could not be found when building)
try:
    import gzip
except ImportError:
    pass

from .. import param
from ..param.parameterized import PicklableClassAttributes

import topo
from topo.base.functionfamilies import OutputFn
from topo.base.sheet import Sheet
from topo.base.cf import CFSheet
from topo.base.projection import Projection, ProjectionSheet
from topo.sheet.generator import GeneratorSheet
from topo.misc.utils import ExtraPickler
from topo.misc.filepaths import normalize_path
from topo.misc import filepaths

def save_input_generators():
    """Save a copy of the active_sim's current input_generators for all GeneratorSheets."""
    # ensure EPs get started (if save_input_generators is called before the simulation is run())
    topo.sim.run(0.0)

    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.push_input_generator()

def restore_input_generators():
    """Restore previously saved input_generators for all of topo.sim's GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.pop_input_generator()

def pattern_present(inputs={},duration=1.0,plastic=False,overwrite_previous=False,apply_output_fn=True):
    """
    Present the specified test patterns for the specified duration.

    Given a set of input patterns (dictionary of
    GeneratorSheetName:PatternGenerator pairs), installs them into the
    specified GeneratorSheets, runs the simulation for the specified
    length of time, then restores the original patterns and the
    original simulation time.  Thus this input is not considered part
    of the regular simulation, and is usually for testing purposes.

    If a simulation is not provided, the active simulation, if one
    exists, is requested.

    If this process is interrupted by the user, the temporary patterns
    may still be installed on the retina.

    If overwrite_previous is true, the given inputs overwrite those
    previously defined.

    If plastic is False, overwrites the existing values of Sheet.plastic
    to disable plasticity, then reenables plasticity.
    """
    # ensure EPs get started (if pattern_present is called before the simulation is run())
    topo.sim.run(0.0)

    if not overwrite_previous:
        save_input_generators()

    if not plastic:
        # turn off plasticity everywhere
        for sheet in topo.sim.objects(Sheet).values():
             sheet.override_plasticity_state(new_plasticity_state=False)

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
             each.apply_output_fn = False

    gen_eps_list = topo.sim.objects(GeneratorSheet)

    # Register the inputs on each input sheet
    for each in inputs.keys():
        if gen_eps_list.has_key(each):
            gen_eps_list[each].set_input_generator(inputs[each])
        else:
            param.Parameterized().warning('%s not a valid Sheet Name.' % each)

    topo.sim.event_push()
    topo.sim.run(duration)
    topo.sim.event_pop()

    # turn sheets' plasticity and output_fn plasticity back on if we turned it off before

    if not plastic:
        for sheet in topo.sim.objects(Sheet).values():
            sheet.restore_plasticity_state()

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
            each.apply_output_fn = True

    if not overwrite_previous:
        restore_input_generators()

def save_snapshot(snapshot_name=None,xml=False):
    """
    Save a snapshot of the network's current state.

    The snapshot is saved as a gzip-compressed Python binary pickle.

    (xml snapshots are currently experimental, and will not be useful
    for most users.)

    As this function uses Python's 'pickle' module, it is subject to
    the same limitations (see the pickle module's documentation) -
    with the notable exception of class attributes. Python does not
    pickle class attributes, but this function stores class attributes
    of any Parameterized class that is declared within the topo
    package. See the topo.base.PicklableClassAttributes class for more
    information.
    """
    if not snapshot_name:
        snapshot_name = topo.sim.basename() + ".typ"

    # For now we just search topo, but could do same for other packages.
    topoPOclassattrs = PicklableClassAttributes(topo,exclusions=('plotting','tests','tkgui'),
                                                startup_commands=topo.sim.startup_commands)

    topo.sim.RELEASE=topo.release
    topo.sim.VERSION=topo.version

    # CEBHACKALERT: is a tuple guaranteed to be unpacked in order?
    # If not, then startup commands are not necessarily executed before
    # the simulation is unpickled
    to_save = (topoPOclassattrs,ExtraPickler(),topo.sim)

    if not xml:
        try:
            snapshot_file=gzip.open(normalize_path(snapshot_name),'w',compresslevel=5)
        except NameError:
            snapshot_file=open(normalize_path(snapshot_name),'w')

        pickle.dump(to_save,snapshot_file,2)
    else:
        snapshot_file=open(normalize_path(snapshot_name),'w')
        gnosis.xml.pickle.dump(to_save,snapshot_file,2,allow_rawpickles=True)

    snapshot_file.close()

def _load_pickle(snapshot):
    # If it's not xml, open as a normal pickle.
    try:
        gnosis.xml.pickle.load(snapshot,allow_rawpickles=True,class_search=gnosis.xml.pickle.SEARCH_ALL)
    except ExpatError:
        snapshot.seek(0)
        pickle.load(snapshot)

def load_snapshot(snapshot_name):
    """
    Load the simulation stored in snapshot_name.
    """
    # unpickling the PicklableClassAttributes() executes startup_commands and
    # sets PO class parameters.

    # If it's not gzipped, open as a normal file.
    try:
        snapshot = gzip.open(snapshot_name,'r')
        snapshot.read(1)
        snapshot.seek(0)
    except (IOError,NameError):
        snapshot = open(snapshot_name,'r')

    try:
        _load_pickle(snapshot)
    except: # what?
        p = param.Parameterized(name="load_snapshot")
        p.message("snapshot '%s' couldn't be loaded; installing legacy support"%snapshot_name)
        import topo.misc.legacy as L
        L.SnapshotSupport.install()
        try:
            _load_pickle(snapshot)
            p.message("snapshot loaded successfully with legacy support")
        except Exception, inst: # what?
            m = "snapshot could not be loaded. If you make a copy " + \
                "of the snapshot available to Topographica's developers, support " + \
                "can probably be added to Topographica; please email " + \
                "developers[at]topographica.org, or file a bug report at " + \
                "www.sf.net/projects/topographica.\n" + \
                "Error:"
            p.warning(m)
            raise

def save_script_repr(script_name=None):
    """
    Save the current simulation as a Topographica script.

    Generates a script that, if run, would generate a simulation with
    the same architecture as the one currently in memory.  This can be
    useful when defining networks in place, so that the same general
    configuration can be recreated later.  It also helps when
    comparing two similar networks generated with different scripts,
    so that the corresponding items can be matched rigorously.

    Note that the result of this operation is usually just a starting
    point for further editing, because it will not usually be runnable
    as-is (for instance, some parameters may not have runnable
    representations).  Even so, this is usually a good start.
    """
    if not script_name:
        script_name = topo.sim.basename() + "_script_repr.ty"

    header = ("# Generated by Topographica %s on %s\n\n" %
              (topo.release,time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())))
    script = header+topo.sim.script_repr()

    script_file = open(normalize_path(script_name),'w')
    script_file.write(script)

# Used only by default_analysis_function
# Should be in order they are needed; e.g. Activity after map measurement,
# in case Activity plot includes map subplots
default_analysis_plotgroups=["Orientation Preference","Activity"]

def default_analysis_function():
    """
    Basic example of an analysis command for run_batch; users are
    likely to need something similar but highly customized.
    """
    import topo
    from topo.command.analysis import save_plotgroup
    from topo.base.projection import ProjectionSheet
    from topo.sheet.generator import GeneratorSheet

    # Build a list of all sheets worth measuring
    f = lambda x: hasattr(x,'measure_maps') and x.measure_maps
    measured_sheets = filter(f,topo.sim.objects(ProjectionSheet).values())
    input_sheets = topo.sim.objects(GeneratorSheet).values()

    # Set potentially reasonable defaults; not necessarily useful
    topo.command.analysis.coordinate=(0.0,0.0)
    if input_sheets:    topo.command.analysis.input_sheet_name=input_sheets[0].name
    if measured_sheets: topo.command.analysis.sheet_name=measured_sheets[0].name

    # Save all plotgroups listed in default_analysis_plotgroups
    for pg in default_analysis_plotgroups:
        save_plotgroup(pg)

    # Save at least one projection plot
    if measured_sheets:
        for p in measured_sheets[0].in_connections:
            save_plotgroup("Projection",projection=p)

    # Test response to a standardized pattern
    from topo.pattern.basic import Gaussian
    from math import pi
    pattern_present(inputs={"Retina":Gaussian(orientation=pi/4,aspect_ratio=4.7)})
    save_plotgroup("Activity",saver_params={"filename_suffix":"_45d"})

# JAB: Should also have some sort of time scaling, so that sims with
# different lengths don't need an entirely new set of analysis times.
# Should encode the cvs state somehow in the output directory, in a
# form that could be used to recreate the CVS version of each file
# used.  Should at least copy the script file into the output directory,
# in any case.
def run_batch(script_file,output_directory="Output",
              analysis_fn = default_analysis_function,
              times = [50,100,500,1000,2000,3000,4000,5000,10000],
              **params):
    """
    Run a Topographica simulation in batch mode.

    Features:

      - Generates a unique, well-defined name for each 'experiment'
        (i.e. simulation run) based on the date, script file, and
        parameter settings

      - Allows parameters to be varied on the command-line,
        to allow comparing various settings

      - Saves a script capturing the simulation state periodically,
        to preserve parameter values from old experiments and to allow
        them to be reproduced exactly later

      - Can perform user-specified analysis routines periodically,
        to monitor the simulation as it progresses.

      - Stores commandline output (stdout) in the output directory

    A typical use of this function is for remote execution of a large
    number of simulations with different parameters, often on remote
    machines (such as clusters).

    The script_file parameter defines the .ty script we want to run in
    batch mode. The output_directory defines the root directory in
    which a unique individual directory will be created for this
    particular run.  The optional analysis_fn can be any python
    function to be called at each of the simulation iterations defined
    in the analysis times list.  This function should perform whatever
    analysis of the simulation you want to perform, such as plotting
    or calculating some statistics.  The analysis_fn should avoid
    using any GUI functions (i.e., should not import anything from
    topo.tkgui), and it should save all of its results into files.

    Any other optional parameters supplied will be set in the main
    namespace before any scripts are run.  They will also be used to
    construct a unique topo.sim.name for the file, and they will be
    encoded into the simulation directory name, to make it clear how
    each simulation differs from the others.
    """
    import sys # CEBALERT: why I have to import this again? (Also done elsewhere below.)

    from topo.misc.commandline import auto_import_commands
    auto_import_commands()

    command_used_to_start = string.join(sys.argv)

    starttime=time.time()
    startnote = "Batch run started at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                                           time.gmtime())
    print startnote

    # Ensure that saved state includes all parameter values
    from topo.command.basic import save_script_repr
    from topo.param import parameterized as parameterizedobject
    parameterizedobject.script_repr_suppress_defaults=False

    # Make sure pylab plots are saved to disk

    # Construct simulation name
    scriptbase= re.sub('.ty$','',os.path.basename(script_file))
    prefix = ""
    prefix += time.strftime("%Y%m%d%H%M")
    prefix += "_" + scriptbase

    simname = prefix

    # Construct parameter-value portion of filename; should do more filtering
    for a in params.keys():
        val=params[a]

        # Special case to give reasonable filenames for lists
        valstr= ("_".join([str(i) for i in val]) if isinstance(val,list)
                 else str(params[a]))
        prefix += "," + a + "=" + valstr

    # Set provided parameter values in main namespace
    for a in params.keys():
        __main__.__dict__[a] = params[a]

    # Create output directories
    if not os.path.isdir(normalize_path(output_directory)):
        os.mkdir(normalize_path(output_directory))

    filepaths.output_path = normalize_path(os.path.join(output_directory,prefix))

    if os.path.isdir(filepaths.output_path):
	print "Batch run: Warning -- directory: " +  \
              filepaths.output_path + \
              " already exists! Run aborted; rename directory or wait one minute before trying again."
        import sys
        sys.exit(-1)
    else:
	os.mkdir(filepaths.output_path)
        print "Batch run output will be in " + filepaths.output_path

    ##################################
    # capture stdout
    #
    import StringIO
    stdout = StringIO.StringIO()
    sys.stdout = stdout
    ##################################

    # Run script in main
    try:
        execfile(script_file,__main__.__dict__)

        topo.sim.name=simname

        # Run each segment, doing the analysis and saving the script state each time
        for run_to in times:
            topo.sim.run(run_to - topo.sim.time())
            analysis_fn()
            save_script_repr()
            elapsedtime=time.time()-starttime
            param.Parameterized(name="run_batch").message(
                "Elapsed real time %02d:%02d." % (int(elapsedtime/60),int(elapsedtime%60)))
    except:
        import traceback
        traceback.print_exc(file=sys.stdout)
        sys.stderr.write("Warning -- Error detected: execution halted.\n")

    endnote = "Batch run completed at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                                           time.gmtime())

    ##################################
    # Write stdout to output file and restore original stdout
    stdout_file = open(normalize_path(simname+".out"),'w')
    stdout_file.write(command_used_to_start+"\n")
    stdout_file.write(startnote+"\n")
    stdout_file.write(stdout.getvalue())
    stdout_file.write(endnote+"\n")
    stdout.close()
    sys.stdout = sys.__stdout__
    ##################################

    # ALERT: Need to count number of errors and warnings and put that on stdout
    # and at the end of the .out file, so that they will be sure to be noticed.

    print endnote

def wipe_out_activity():
    """
    Resets activity of all Sheets and their connections to zero.
    """
    # ALERT: this works for now, but it may need to be implemented
    # recursively using methods implemented separately on each class,
    # if there are often new types of objects created that store an
    # activity value.
    for s in topo.sim.objects(Sheet).values():
        s.activity*=0.0
        for c in s.in_connections:
            if hasattr(c,'activity'):
                c.activity*=0.0

# maybe an explicit list would be better?
import types
__all__ = list(set([k for k,v in locals().items()
                    if isinstance(v,types.FunctionType)
                    and not v.__name__.startswith('_')]))
<commit_msg>No visible change (removed code accidentally added in previous checkin.<commit_after>"""
High-level user-level commands controlling the entire simulation.
$Id$
"""
__version__='$Revision$'

import cPickle as pickle
import gnosis.xml.pickle
from xml.parsers.expat import ExpatError

import time
import string
import re
import os

import __main__

# gzip module might not have been built (if zlib could not be found when building)
try:
    import gzip
except ImportError:
    pass

from .. import param
from ..param.parameterized import PicklableClassAttributes

import topo
from topo.base.functionfamilies import OutputFn
from topo.base.sheet import Sheet
from topo.base.cf import CFSheet
from topo.base.projection import Projection, ProjectionSheet
from topo.sheet.generator import GeneratorSheet
from topo.misc.utils import ExtraPickler
from topo.misc.filepaths import normalize_path
from topo.misc import filepaths

def save_input_generators():
    """Save a copy of the active_sim's current input_generators for all GeneratorSheets."""
    # ensure EPs get started (if save_input_generators is called before the simulation is run())
    topo.sim.run(0.0)

    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.push_input_generator()

def restore_input_generators():
    """Restore previously saved input_generators for all of topo.sim's GeneratorSheets."""
    generator_sheets = topo.sim.objects(GeneratorSheet).values()
    for sheet in generator_sheets:
        sheet.pop_input_generator()

def pattern_present(inputs={},duration=1.0,plastic=False,overwrite_previous=False,apply_output_fn=True):
    """
    Present the specified test patterns for the specified duration.

    Given a set of input patterns (dictionary of
    GeneratorSheetName:PatternGenerator pairs), installs them into the
    specified GeneratorSheets, runs the simulation for the specified
    length of time, then restores the original patterns and the
    original simulation time.  Thus this input is not considered part
    of the regular simulation, and is usually for testing purposes.

    If a simulation is not provided, the active simulation, if one
    exists, is requested.

    If this process is interrupted by the user, the temporary patterns
    may still be installed on the retina.

    If overwrite_previous is true, the given inputs overwrite those
    previously defined.

    If plastic is False, overwrites the existing values of Sheet.plastic
    to disable plasticity, then reenables plasticity.
    """
    # ensure EPs get started (if pattern_present is called before the simulation is run())
    topo.sim.run(0.0)

    if not overwrite_previous:
        save_input_generators()

    if not plastic:
        # turn off plasticity everywhere
        for sheet in topo.sim.objects(Sheet).values():
             sheet.override_plasticity_state(new_plasticity_state=False)

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
             each.apply_output_fn = False

    gen_eps_list = topo.sim.objects(GeneratorSheet)

    # Register the inputs on each input sheet
    for each in inputs.keys():
        if gen_eps_list.has_key(each):
            gen_eps_list[each].set_input_generator(inputs[each])
        else:
            param.Parameterized().warning('%s not a valid Sheet Name.' % each)

    topo.sim.event_push()
    topo.sim.run(duration)
    topo.sim.event_pop()

    # turn sheets' plasticity and output_fn plasticity back on if we turned it off before

    if not plastic:
        for sheet in topo.sim.objects(Sheet).values():
            sheet.restore_plasticity_state()

    if not apply_output_fn:
        for each in topo.sim.objects(Sheet).values():
            each.apply_output_fn = True

    if not overwrite_previous:
        restore_input_generators()

def save_snapshot(snapshot_name=None,xml=False):
    """
    Save a snapshot of the network's current state.

    The snapshot is saved as a gzip-compressed Python binary pickle.

    (xml snapshots are currently experimental, and will not be useful
    for most users.)

    As this function uses Python's 'pickle' module, it is subject to
    the same limitations (see the pickle module's documentation) -
    with the notable exception of class attributes. Python does not
    pickle class attributes, but this function stores class attributes
    of any Parameterized class that is declared within the topo
    package. See the topo.base.PicklableClassAttributes class for more
    information.
    """
    if not snapshot_name:
        snapshot_name = topo.sim.basename() + ".typ"

    # For now we just search topo, but could do same for other packages.
    topoPOclassattrs = PicklableClassAttributes(topo,exclusions=('plotting','tests','tkgui'),
                                                startup_commands=topo.sim.startup_commands)

    topo.sim.RELEASE=topo.release
    topo.sim.VERSION=topo.version

    # CEBHACKALERT: is a tuple guaranteed to be unpacked in order?
    # If not, then startup commands are not necessarily executed before
    # the simulation is unpickled
    to_save = (topoPOclassattrs,ExtraPickler(),topo.sim)

    if not xml:
        try:
            snapshot_file=gzip.open(normalize_path(snapshot_name),'w',compresslevel=5)
        except NameError:
            snapshot_file=open(normalize_path(snapshot_name),'w')

        pickle.dump(to_save,snapshot_file,2)
    else:
        snapshot_file=open(normalize_path(snapshot_name),'w')
        gnosis.xml.pickle.dump(to_save,snapshot_file,2,allow_rawpickles=True)

    snapshot_file.close()

def _load_pickle(snapshot):
    # If it's not xml, open as a normal pickle.
    try:
        gnosis.xml.pickle.load(snapshot,allow_rawpickles=True,class_search=gnosis.xml.pickle.SEARCH_ALL)
    except ExpatError:
        snapshot.seek(0)
        pickle.load(snapshot)

def load_snapshot(snapshot_name):
    """
    Load the simulation stored in snapshot_name.
    """
    # unpickling the PicklableClassAttributes() executes startup_commands and
    # sets PO class parameters.

    # If it's not gzipped, open as a normal file.
    try:
        snapshot = gzip.open(snapshot_name,'r')
        snapshot.read(1)
        snapshot.seek(0)
    except (IOError,NameError):
        snapshot = open(snapshot_name,'r')

    try:
        _load_pickle(snapshot)
    except: # what?
        p = param.Parameterized(name="load_snapshot")
        p.message("snapshot '%s' couldn't be loaded; installing legacy support"%snapshot_name)
        import topo.misc.legacy as L
        L.SnapshotSupport.install()
        try:
            _load_pickle(snapshot)
            p.message("snapshot loaded successfully with legacy support")
        except: # what?
            m = "snapshot could not be loaded. If you make a copy " + \
                "of the snapshot available to Topographica's developers, support " + \
                "can probably be added to Topographica; please email " + \
                "developers[at]topographica.org, or file a bug report at " + \
                "www.sf.net/projects/topographica.\n" + \
                "Error:"
            p.warning(m)
            raise

def save_script_repr(script_name=None):
    """
    Save the current simulation as a Topographica script.

    Generates a script that, if run, would generate a simulation with
    the same architecture as the one currently in memory.  This can be
    useful when defining networks in place, so that the same general
    configuration can be recreated later.  It also helps when
    comparing two similar networks generated with different scripts,
    so that the corresponding items can be matched rigorously.

    Note that the result of this operation is usually just a starting
    point for further editing, because it will not usually be runnable
    as-is (for instance, some parameters may not have runnable
    representations).  Even so, this is usually a good start.
    """
    if not script_name:
        script_name = topo.sim.basename() + "_script_repr.ty"

    header = ("# Generated by Topographica %s on %s\n\n" %
              (topo.release,time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())))
    script = header+topo.sim.script_repr()

    script_file = open(normalize_path(script_name),'w')
    script_file.write(script)

# Used only by default_analysis_function
# Should be in order they are needed; e.g. Activity after map measurement,
# in case Activity plot includes map subplots
default_analysis_plotgroups=["Orientation Preference","Activity"]

def default_analysis_function():
    """
    Basic example of an analysis command for run_batch; users are
    likely to need something similar but highly customized.
    """
    import topo
    from topo.command.analysis import save_plotgroup
    from topo.base.projection import ProjectionSheet
    from topo.sheet.generator import GeneratorSheet

    # Build a list of all sheets worth measuring
    f = lambda x: hasattr(x,'measure_maps') and x.measure_maps
    measured_sheets = filter(f,topo.sim.objects(ProjectionSheet).values())
    input_sheets = topo.sim.objects(GeneratorSheet).values()

    # Set potentially reasonable defaults; not necessarily useful
    topo.command.analysis.coordinate=(0.0,0.0)
    if input_sheets:    topo.command.analysis.input_sheet_name=input_sheets[0].name
    if measured_sheets: topo.command.analysis.sheet_name=measured_sheets[0].name

    # Save all plotgroups listed in default_analysis_plotgroups
    for pg in default_analysis_plotgroups:
        save_plotgroup(pg)

    # Save at least one projection plot
    if measured_sheets:
        for p in measured_sheets[0].in_connections:
            save_plotgroup("Projection",projection=p)

    # Test response to a standardized pattern
    from topo.pattern.basic import Gaussian
    from math import pi
    pattern_present(inputs={"Retina":Gaussian(orientation=pi/4,aspect_ratio=4.7)})
    save_plotgroup("Activity",saver_params={"filename_suffix":"_45d"})

# JAB: Should also have some sort of time scaling, so that sims with
# different lengths don't need an entirely new set of analysis times.
# Should encode the cvs state somehow in the output directory, in a
# form that could be used to recreate the CVS version of each file
# used.  Should at least copy the script file into the output directory,
# in any case.
def run_batch(script_file,output_directory="Output",
              analysis_fn = default_analysis_function,
              times = [50,100,500,1000,2000,3000,4000,5000,10000],
              **params):
    """
    Run a Topographica simulation in batch mode.

    Features:

      - Generates a unique, well-defined name for each 'experiment'
        (i.e. simulation run) based on the date, script file, and
        parameter settings

      - Allows parameters to be varied on the command-line,
        to allow comparing various settings

      - Saves a script capturing the simulation state periodically,
        to preserve parameter values from old experiments and to allow
        them to be reproduced exactly later

      - Can perform user-specified analysis routines periodically,
        to monitor the simulation as it progresses.

      - Stores commandline output (stdout) in the output directory

    A typical use of this function is for remote execution of a large
    number of simulations with different parameters, often on remote
    machines (such as clusters).

    The script_file parameter defines the .ty script we want to run in
    batch mode. The output_directory defines the root directory in
    which a unique individual directory will be created for this
    particular run.  The optional analysis_fn can be any python
    function to be called at each of the simulation iterations defined
    in the analysis times list.  This function should perform whatever
    analysis of the simulation you want to perform, such as plotting
    or calculating some statistics.  The analysis_fn should avoid
    using any GUI functions (i.e., should not import anything from
    topo.tkgui), and it should save all of its results into files.

    Any other optional parameters supplied will be set in the main
    namespace before any scripts are run.  They will also be used to
    construct a unique topo.sim.name for the file, and they will be
    encoded into the simulation directory name, to make it clear how
    each simulation differs from the others.
    """
    import sys # CEBALERT: why I have to import this again? (Also done elsewhere below.)

    from topo.misc.commandline import auto_import_commands
    auto_import_commands()

    command_used_to_start = string.join(sys.argv)

    starttime=time.time()
    startnote = "Batch run started at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                                           time.gmtime())
    print startnote

    # Ensure that saved state includes all parameter values
    from topo.command.basic import save_script_repr
    from topo.param import parameterized as parameterizedobject
    parameterizedobject.script_repr_suppress_defaults=False

    # Make sure pylab plots are saved to disk

    # Construct simulation name
    scriptbase= re.sub('.ty$','',os.path.basename(script_file))
    prefix = ""
    prefix += time.strftime("%Y%m%d%H%M")
    prefix += "_" + scriptbase

    simname = prefix

    # Construct parameter-value portion of filename; should do more filtering
    for a in params.keys():
        val=params[a]

        # Special case to give reasonable filenames for lists
        valstr= ("_".join([str(i) for i in val]) if isinstance(val,list)
                 else str(params[a]))
        prefix += "," + a + "=" + valstr

    # Set provided parameter values in main namespace
    for a in params.keys():
        __main__.__dict__[a] = params[a]

    # Create output directories
    if not os.path.isdir(normalize_path(output_directory)):
        os.mkdir(normalize_path(output_directory))

    filepaths.output_path = normalize_path(os.path.join(output_directory,prefix))

    if os.path.isdir(filepaths.output_path):
	print "Batch run: Warning -- directory: " +  \
              filepaths.output_path + \
              " already exists! Run aborted; rename directory or wait one minute before trying again."
        import sys
        sys.exit(-1)
    else:
	os.mkdir(filepaths.output_path)
        print "Batch run output will be in " + filepaths.output_path

    ##################################
    # capture stdout
    #
    import StringIO
    stdout = StringIO.StringIO()
    sys.stdout = stdout
    ##################################

    # Run script in main
    try:
        execfile(script_file,__main__.__dict__)

        topo.sim.name=simname

        # Run each segment, doing the analysis and saving the script state each time
        for run_to in times:
            topo.sim.run(run_to - topo.sim.time())
            analysis_fn()
            save_script_repr()
            elapsedtime=time.time()-starttime
            param.Parameterized(name="run_batch").message(
                "Elapsed real time %02d:%02d." % (int(elapsedtime/60),int(elapsedtime%60)))
    except:
        import traceback
        traceback.print_exc(file=sys.stdout)
        sys.stderr.write("Warning -- Error detected: execution halted.\n")

    endnote = "Batch run completed at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                                           time.gmtime())

    ##################################
    # Write stdout to output file and restore original stdout
    stdout_file = open(normalize_path(simname+".out"),'w')
    stdout_file.write(command_used_to_start+"\n")
    stdout_file.write(startnote+"\n")
    stdout_file.write(stdout.getvalue())
    stdout_file.write(endnote+"\n")
    stdout.close()
    sys.stdout = sys.__stdout__
    ##################################

    # ALERT: Need to count number of errors and warnings and put that on stdout
    # and at the end of the .out file, so that they will be sure to be noticed.

    print endnote

def wipe_out_activity():
    """
    Resets activity of all Sheets and their connections to zero.
    """
    # ALERT: this works for now, but it may need to be implemented
    # recursively using methods implemented separately on each class,
    # if there are often new types of objects created that store an
    # activity value.
    for s in topo.sim.objects(Sheet).values():
        s.activity*=0.0
        for c in s.in_connections:
            if hasattr(c,'activity'):
                c.activity*=0.0

# maybe an explicit list would be better?
import types
__all__ = list(set([k for k,v in locals().items()
                    if isinstance(v,types.FunctionType)
                    and not v.__name__.startswith('_')]))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from django.contrib import admin
from django.db import models

from pagedown.widgets import AdminPagedownWidget

from .models import Promotion, Student, StudentApplication

from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponseRedirect

class StudentAdmin(admin.ModelAdmin):
    list_display = ('artist', 'number', 'promotion', 'graduate')
    search_fields = ('number', 'user__first_name', 'user__last_name')

    formfield_overrides = {
        models.TextField: {'widget': AdminPagedownWidget },
    }

def output_excel(modeladmin, request, queryset):
    # do something
    selected = request.POST.getlist(admin.ACTION_CHECKBOX_NAME)
    ct = ContentType.objects.get_for_model(queryset.model)
    return HttpResponseRedirect("/export/?ct=%s&ids=%s" % (ct.pk, ",".join(selected)))

output_excel.short_description = "Sortie Excel"

class StudentApplicationAdmin(admin.ModelAdmin):

    def _get_name(self,obj):
        return obj.artist.user.get_full_name()

    _get_name.short_description = "Nom"
    list_display = ('_get_name','current_year_application_count','created_on','selected_for_interview')

admin.site.register(Promotion)
admin.site.register(StudentApplication, StudentApplicationAdmin )
admin.site.register(Student, StudentAdmin)
<commit_msg>change admin view application model<commit_after># -*- encoding: utf-8 -*-
from django.contrib import admin
from django.db import models

from pagedown.widgets import AdminPagedownWidget

from .models import Promotion, Student, StudentApplication

from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponseRedirect

class StudentAdmin(admin.ModelAdmin):
    list_display = ('artist', 'number', 'promotion', 'graduate')
    search_fields = ('number', 'user__first_name', 'user__last_name')

    formfield_overrides = {
        models.TextField: {'widget': AdminPagedownWidget },
    }

def output_excel(modeladmin, request, queryset):
    # do something
    selected = request.POST.getlist(admin.ACTION_CHECKBOX_NAME)
    ct = ContentType.objects.get_for_model(queryset.model)
    return HttpResponseRedirect("/export/?ct=%s&ids=%s" % (ct.pk, ",".join(selected)))

output_excel.short_description = "Sortie Excel"

class StudentApplicationAdmin(admin.ModelAdmin):

    def _get_name(self,obj):
        return obj.artist.user.get_full_name()

    _get_name.short_description = "Nom"
    list_display = ('_get_name','current_year_application_count','created_on','selected_for_interview', 'asynchronous_element', 'application_complete', 'remark')

admin.site.register(Promotion)
admin.site.register(StudentApplication, StudentApplicationAdmin )
admin.site.register(Student, StudentAdmin)
<|endoftext|>


<commit_before>"""
    pygments.lexers._postgres_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Self-updating data files for PostgreSQL lexer.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re
import urllib

# One man's constant is another man's variable.
SOURCE_URL = 'https://github.com/postgres/postgres/raw/master'
KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'

def update_myself():
    data_file = list(fetch(DATATYPES_URL))
    datatypes = parse_datatypes(data_file)
    pseudos = parse_pseudos(data_file)

    keywords = parse_keywords(fetch(KEYWORDS_URL))
    update_consts(__file__, 'DATATYPES', datatypes)
    update_consts(__file__, 'PSEUDO_TYPES', pseudos)
    update_consts(__file__, 'KEYWORDS', keywords)

def parse_keywords(f):
    kw = []
    for m in re.finditer(
            r'\s*<entry><token>([^<]+)</token></entry>\s*'
            r'<entry>([^<]+)</entry>', f.read()):
        kw.append(m.group(1))

    if not kw:
        raise ValueError('no keyword found')

    kw.sort()
    return kw

def parse_datatypes(f):
    dt = set()
    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
    for line in f:
        if '<sect1' in line:
            break
        if '<entry><type>' not in line:
            continue

        # Parse a string such as
        # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
        # into types "time" and "without time zone"

        # remove all the tags
        line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
        line = re.sub("<[^>]+>", "", line)

        # Drop the parts containing braces
        for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
            for t in tmp.split(','):
                t = t.strip()
                if not t: continue
                dt.add(" ".join(t.split()))

    dt = list(dt)
    dt.sort()
    return dt

def parse_pseudos(f):
    dt = []
    re_start = re.compile(r'\s*
')
    re_entry = re.compile(r'\s*<entry><type>([^<]+)</></entry>')
    re_end = re.compile(r'\s*
')

    f = iter(f)
    for line in f:
        if re_start.match(line) is not None:
            break
    else:
        raise ValueError('pseudo datatypes table not found')

    for line in f:
        m = re_entry.match(line)
        if m is not None:
            dt.append(m.group(1))

        if re_end.match(line) is not None:
            break
    else:
        raise ValueError('end of pseudo datatypes table not found')

    if not dt:
        raise ValueError('pseudo datatypes not found')

    return dt

def fetch(url):
    return urllib.urlopen(url)

def update_consts(filename, constname, content):
    f = open(filename)
    lines = f.readlines()
    f.close()

    # Line to start/end inserting
    re_start = re.compile(r'^%s\s*=\s*\[\s*$' % constname)
    re_end = re.compile(r'^\s*\]\s*$')
    start = [ n for n, l in enumerate(lines) if re_start.match(l) ]
    if not start:
        raise ValueError("couldn't find line containing '%s = ['" % constname)
    if len(start) > 1:
        raise ValueError("too many lines containing '%s = ['" % constname)
    start = start[0] + 1

    end = [ n for n, l in enumerate(lines) if n >= start and re_end.match(l) ]
    if not end:
        raise ValueError("couldn't find line containing ']' after %s " % constname)
    end = end[0]

    # Pack the new content in lines not too long
    content = [repr(item) for item in content ]
    new_lines = [[]]
    for item in content:
        if sum(map(len, new_lines[-1])) + 2 * len(new_lines[-1]) + len(item) + 4 > 75:
            new_lines.append([])
        new_lines[-1].append(item)

    lines[start:end] = [ "    %s,\n" % ", ".join(items) for items in new_lines ]

    f = open(filename, 'w')
    f.write(''.join(lines))
    f.close()

# Autogenerated: please edit them if you like wasting your time.

KEYWORDS = [
    'ABORT', 'ABSOLUTE', 'ACCESS', 'ACTION', 'ADD', 'ADMIN', 'AFTER',
    'AGGREGATE', 'ALL', 'ALSO', 'ALTER', 'ALWAYS', 'ANALYSE', 'ANALYZE',
    'AND', 'ANY', 'ARRAY', 'AS', 'ASC', 'ASSERTION', 'ASSIGNMENT',
    'ASYMMETRIC', 'AT', 'ATTRIBUTE', 'AUTHORIZATION', 'BACKWARD', 'BEFORE',
    'BEGIN', 'BETWEEN', 'BIGINT', 'BINARY', 'BIT', 'BOOLEAN', 'BOTH', 'BY',
    'CACHE', 'CALLED', 'CASCADE', 'CASCADED', 'CASE', 'CAST', 'CATALOG',
    'CHAIN', 'CHAR', 'CHARACTER', 'CHARACTERISTICS', 'CHECK', 'CHECKPOINT',
    'CLASS', 'CLOSE', 'CLUSTER', 'COALESCE', 'COLLATE', 'COLLATION',
    'COLUMN', 'COMMENT', 'COMMENTS', 'COMMIT', 'COMMITTED', 'CONCURRENTLY',
    'CONFIGURATION', 'CONNECTION', 'CONSTRAINT', 'CONSTRAINTS', 'CONTENT',
    'CONTINUE', 'CONVERSION', 'COPY', 'COST', 'CREATE', 'CROSS', 'CSV',
    'CURRENT', 'CURRENT_CATALOG', 'CURRENT_DATE', 'CURRENT_ROLE',
    'CURRENT_SCHEMA', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER',
    'CURSOR', 'CYCLE', 'DATA', 'DATABASE', 'DAY', 'DEALLOCATE', 'DEC',
    'DECIMAL', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', 'DEFERRED',
    'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', 'DESC', 'DICTIONARY',
    'DISABLE', 'DISCARD', 'DISTINCT', 'DO', 'DOCUMENT', 'DOMAIN', 'DOUBLE',
    'DROP', 'EACH', 'ELSE', 'ENABLE', 'ENCODING', 'ENCRYPTED', 'END',
    'ENUM', 'ESCAPE', 'EXCEPT', 'EXCLUDE', 'EXCLUDING', 'EXCLUSIVE',
    'EXECUTE', 'EXISTS', 'EXPLAIN', 'EXTENSION', 'EXTERNAL', 'EXTRACT',
    'FALSE', 'FAMILY', 'FETCH', 'FIRST', 'FLOAT', 'FOLLOWING', 'FOR',
    'FORCE', 'FOREIGN', 'FORWARD', 'FREEZE', 'FROM', 'FULL', 'FUNCTION',
    'FUNCTIONS', 'GLOBAL', 'GRANT', 'GRANTED', 'GREATEST', 'GROUP',
    'HANDLER', 'HAVING', 'HEADER', 'HOLD', 'HOUR', 'IDENTITY', 'IF',
    'ILIKE', 'IMMEDIATE', 'IMMUTABLE', 'IMPLICIT', 'IN', 'INCLUDING',
    'INCREMENT', 'INDEX', 'INDEXES', 'INHERIT', 'INHERITS', 'INITIALLY',
    'INLINE', 'INNER', 'INOUT', 'INPUT', 'INSENSITIVE', 'INSERT', 'INSTEAD',
    'INT', 'INTEGER', 'INTERSECT', 'INTERVAL', 'INTO', 'INVOKER', 'IS',
    'ISNULL', 'ISOLATION', 'JOIN', 'KEY', 'LABEL', 'LANGUAGE', 'LARGE',
    'LAST', 'LC_COLLATE', 'LC_CTYPE', 'LEADING', 'LEAST', 'LEFT', 'LEVEL',
    'LIKE', 'LIMIT', 'LISTEN', 'LOAD', 'LOCAL', 'LOCALTIME',
    'LOCALTIMESTAMP', 'LOCATION', 'LOCK', 'MAPPING', 'MATCH', 'MAXVALUE',
    'MINUTE', 'MINVALUE', 'MODE', 'MONTH', 'MOVE', 'NAME', 'NAMES',
    'NATIONAL', 'NATURAL', 'NCHAR', 'NEXT', 'NO', 'NONE', 'NOT', 'NOTHING',
    'NOTIFY', 'NOTNULL', 'NOWAIT', 'NULL', 'NULLIF', 'NULLS', 'NUMERIC',
    'OBJECT', 'OF', 'OFF', 'OFFSET', 'OIDS', 'ON', 'ONLY', 'OPERATOR',
    'OPTION', 'OPTIONS', 'OR', 'ORDER', 'OUT', 'OUTER', 'OVER', 'OVERLAPS',
    'OVERLAY', 'OWNED', 'OWNER', 'PARSER', 'PARTIAL', 'PARTITION',
    'PASSING', 'PASSWORD', 'PLACING', 'PLANS', 'POSITION', 'PRECEDING',
    'PRECISION', 'PREPARE', 'PREPARED', 'PRESERVE', 'PRIMARY', 'PRIOR',
    'PRIVILEGES', 'PROCEDURAL', 'PROCEDURE', 'QUOTE', 'RANGE', 'READ',
    'REAL', 'REASSIGN', 'RECHECK', 'RECURSIVE', 'REF', 'REFERENCES',
    'REINDEX', 'RELATIVE', 'RELEASE', 'RENAME', 'REPEATABLE', 'REPLACE',
    'REPLICA', 'RESET', 'RESTART', 'RESTRICT', 'RETURNING', 'RETURNS',
    'REVOKE', 'RIGHT', 'ROLE', 'ROLLBACK', 'ROW', 'ROWS', 'RULE',
    'SAVEPOINT', 'SCHEMA', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY',
    'SELECT', 'SEQUENCE', 'SEQUENCES', 'SERIALIZABLE', 'SERVER', 'SESSION',
    'SESSION_USER', 'SET', 'SETOF', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE',
    'SMALLINT', 'SOME', 'STABLE', 'STANDALONE', 'START', 'STATEMENT',
    'STATISTICS', 'STDIN', 'STDOUT', 'STORAGE', 'STRICT', 'STRIP',
    'SUBSTRING', 'SYMMETRIC', 'SYSID', 'SYSTEM', 'TABLE', 'TABLES',
    'TABLESPACE', 'TEMP', 'TEMPLATE', 'TEMPORARY', 'TEXT', 'THEN', 'TIME',
    'TIMESTAMP', 'TO', 'TRAILING', 'TRANSACTION', 'TREAT', 'TRIGGER',
    'TRIM', 'TRUE', 'TRUNCATE', 'TRUSTED', 'TYPE', 'UNBOUNDED',
    'UNCOMMITTED', 'UNENCRYPTED', 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN',
    'UNLOGGED', 'UNTIL', 'UPDATE', 'USER', 'USING', 'VACUUM', 'VALID',
    'VALIDATE', 'VALIDATOR', 'VALUE', 'VALUES', 'VARCHAR', 'VARIADIC',
    'VARYING', 'VERBOSE', 'VERSION', 'VIEW', 'VOLATILE', 'WHEN', 'WHERE',
    'WHITESPACE', 'WINDOW', 'WITH', 'WITHOUT', 'WORK', 'WRAPPER', 'WRITE',
    'XML', 'XMLATTRIBUTES', 'XMLCONCAT', 'XMLELEMENT', 'XMLEXISTS',
    'XMLFOREST', 'XMLPARSE', 'XMLPI', 'XMLROOT', 'XMLSERIALIZE', 'YEAR',
    'YES', 'ZONE',
    ]

DATATYPES = [
    'bigint', 'bigserial', 'bit', 'bit varying', 'bool', 'boolean', 'box',
    'bytea', 'char', 'character', 'character varying', 'cidr', 'circle',
    'date', 'decimal', 'double precision', 'float4', 'float8', 'inet',
    'int', 'int2', 'int4', 'int8', 'integer', 'interval', 'line', 'lseg',
    'macaddr', 'money', 'numeric', 'path', 'point', 'polygon', 'real',
    'serial', 'serial4', 'serial8', 'smallint', 'text', 'time', 'timestamp',
    'timestamptz', 'timetz', 'tsquery', 'tsvector', 'txid_snapshot', 'uuid',
    'varbit', 'varchar', 'with time zone', 'without time zone', 'xml',
    ]

PSEUDO_TYPES = [
    'any', 'anyarray', 'anyelement', 'anyenum', 'anynonarray', 'cstring',
    'internal', 'language_handler', 'fdw_handler', 'record', 'trigger',
    'void', 'opaque',
    ]

# Remove 'trigger' from types
PSEUDO_TYPES = sorted(set(PSEUDO_TYPES) - set(map(str.lower, KEYWORDS)))

PLPGSQL_KEYWORDS = [
    'ALIAS', 'CONSTANT', 'DIAGNOSTICS', 'ELSIF', 'EXCEPTION', 'EXIT',
    'FOREACH', 'GET', 'LOOP', 'NOTICE', 'OPEN', 'PERFORM', 'QUERY', 'RAISE',
    'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
    ]

if __name__ == '__main__':
    update_myself()

<commit_msg>Added a few new PostgreSQL 9.2 types<commit_after>"""
    pygments.lexers._postgres_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Self-updating data files for PostgreSQL lexer.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re
import urllib

# One man's constant is another man's variable.
SOURCE_URL = 'https://github.com/postgres/postgres/raw/master'
KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'

def update_myself():
    data_file = list(fetch(DATATYPES_URL))
    datatypes = parse_datatypes(data_file)
    pseudos = parse_pseudos(data_file)

    keywords = parse_keywords(fetch(KEYWORDS_URL))
    update_consts(__file__, 'DATATYPES', datatypes)
    update_consts(__file__, 'PSEUDO_TYPES', pseudos)
    update_consts(__file__, 'KEYWORDS', keywords)

def parse_keywords(f):
    kw = []
    for m in re.finditer(
            r'\s*<entry><token>([^<]+)</token></entry>\s*'
            r'<entry>([^<]+)</entry>', f.read()):
        kw.append(m.group(1))

    if not kw:
        raise ValueError('no keyword found')

    kw.sort()
    return kw

def parse_datatypes(f):
    dt = set()
    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
    for line in f:
        if '<sect1' in line:
            break
        if '<entry><type>' not in line:
            continue

        # Parse a string such as
        # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
        # into types "time" and "without time zone"

        # remove all the tags
        line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
        line = re.sub("<[^>]+>", "", line)

        # Drop the parts containing braces
        for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
            for t in tmp.split(','):
                t = t.strip()
                if not t: continue
                dt.add(" ".join(t.split()))

    dt = list(dt)
    dt.sort()
    return dt

def parse_pseudos(f):
    dt = []
    re_start = re.compile(r'\s*
')
    re_entry = re.compile(r'\s*<entry><type>([^<]+)</></entry>')
    re_end = re.compile(r'\s*
')

    f = iter(f)
    for line in f:
        if re_start.match(line) is not None:
            break
    else:
        raise ValueError('pseudo datatypes table not found')

    for line in f:
        m = re_entry.match(line)
        if m is not None:
            dt.append(m.group(1))

        if re_end.match(line) is not None:
            break
    else:
        raise ValueError('end of pseudo datatypes table not found')

    if not dt:
        raise ValueError('pseudo datatypes not found')

    return dt

def fetch(url):
    return urllib.urlopen(url)

def update_consts(filename, constname, content):
    f = open(filename)
    lines = f.readlines()
    f.close()

    # Line to start/end inserting
    re_start = re.compile(r'^%s\s*=\s*\[\s*$' % constname)
    re_end = re.compile(r'^\s*\]\s*$')
    start = [ n for n, l in enumerate(lines) if re_start.match(l) ]
    if not start:
        raise ValueError("couldn't find line containing '%s = ['" % constname)
    if len(start) > 1:
        raise ValueError("too many lines containing '%s = ['" % constname)
    start = start[0] + 1

    end = [ n for n, l in enumerate(lines) if n >= start and re_end.match(l) ]
    if not end:
        raise ValueError("couldn't find line containing ']' after %s " % constname)
    end = end[0]

    # Pack the new content in lines not too long
    content = [repr(item) for item in content ]
    new_lines = [[]]
    for item in content:
        if sum(map(len, new_lines[-1])) + 2 * len(new_lines[-1]) + len(item) + 4 > 75:
            new_lines.append([])
        new_lines[-1].append(item)

    lines[start:end] = [ "    %s,\n" % ", ".join(items) for items in new_lines ]

    f = open(filename, 'w')
    f.write(''.join(lines))
    f.close()

# Autogenerated: please edit them if you like wasting your time.

KEYWORDS = [
    'ABORT', 'ABSOLUTE', 'ACCESS', 'ACTION', 'ADD', 'ADMIN', 'AFTER',
    'AGGREGATE', 'ALL', 'ALSO', 'ALTER', 'ALWAYS', 'ANALYSE', 'ANALYZE',
    'AND', 'ANY', 'ARRAY', 'AS', 'ASC', 'ASSERTION', 'ASSIGNMENT',
    'ASYMMETRIC', 'AT', 'ATTRIBUTE', 'AUTHORIZATION', 'BACKWARD', 'BEFORE',
    'BEGIN', 'BETWEEN', 'BIGINT', 'BINARY', 'BIT', 'BOOLEAN', 'BOTH', 'BY',
    'CACHE', 'CALLED', 'CASCADE', 'CASCADED', 'CASE', 'CAST', 'CATALOG',
    'CHAIN', 'CHAR', 'CHARACTER', 'CHARACTERISTICS', 'CHECK', 'CHECKPOINT',
    'CLASS', 'CLOSE', 'CLUSTER', 'COALESCE', 'COLLATE', 'COLLATION',
    'COLUMN', 'COMMENT', 'COMMENTS', 'COMMIT', 'COMMITTED', 'CONCURRENTLY',
    'CONFIGURATION', 'CONNECTION', 'CONSTRAINT', 'CONSTRAINTS', 'CONTENT',
    'CONTINUE', 'CONVERSION', 'COPY', 'COST', 'CREATE', 'CROSS', 'CSV',
    'CURRENT', 'CURRENT_CATALOG', 'CURRENT_DATE', 'CURRENT_ROLE',
    'CURRENT_SCHEMA', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER',
    'CURSOR', 'CYCLE', 'DATA', 'DATABASE', 'DAY', 'DEALLOCATE', 'DEC',
    'DECIMAL', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', 'DEFERRED',
    'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', 'DESC', 'DICTIONARY',
    'DISABLE', 'DISCARD', 'DISTINCT', 'DO', 'DOCUMENT', 'DOMAIN', 'DOUBLE',
    'DROP', 'EACH', 'ELSE', 'ENABLE', 'ENCODING', 'ENCRYPTED', 'END',
    'ENUM', 'ESCAPE', 'EXCEPT', 'EXCLUDE', 'EXCLUDING', 'EXCLUSIVE',
    'EXECUTE', 'EXISTS', 'EXPLAIN', 'EXTENSION', 'EXTERNAL', 'EXTRACT',
    'FALSE', 'FAMILY', 'FETCH', 'FIRST', 'FLOAT', 'FOLLOWING', 'FOR',
    'FORCE', 'FOREIGN', 'FORWARD', 'FREEZE', 'FROM', 'FULL', 'FUNCTION',
    'FUNCTIONS', 'GLOBAL', 'GRANT', 'GRANTED', 'GREATEST', 'GROUP',
    'HANDLER', 'HAVING', 'HEADER', 'HOLD', 'HOUR', 'IDENTITY', 'IF',
    'ILIKE', 'IMMEDIATE', 'IMMUTABLE', 'IMPLICIT', 'IN', 'INCLUDING',
    'INCREMENT', 'INDEX', 'INDEXES', 'INHERIT', 'INHERITS', 'INITIALLY',
    'INLINE', 'INNER', 'INOUT', 'INPUT', 'INSENSITIVE', 'INSERT', 'INSTEAD',
    'INT', 'INTEGER', 'INTERSECT', 'INTERVAL', 'INTO', 'INVOKER', 'IS',
    'ISNULL', 'ISOLATION', 'JOIN', 'KEY', 'LABEL', 'LANGUAGE', 'LARGE',
    'LAST', 'LC_COLLATE', 'LC_CTYPE', 'LEADING', 'LEAST', 'LEFT', 'LEVEL',
    'LIKE', 'LIMIT', 'LISTEN', 'LOAD', 'LOCAL', 'LOCALTIME',
    'LOCALTIMESTAMP', 'LOCATION', 'LOCK', 'MAPPING', 'MATCH', 'MAXVALUE',
    'MINUTE', 'MINVALUE', 'MODE', 'MONTH', 'MOVE', 'NAME', 'NAMES',
    'NATIONAL', 'NATURAL', 'NCHAR', 'NEXT', 'NO', 'NONE', 'NOT', 'NOTHING',
    'NOTIFY', 'NOTNULL', 'NOWAIT', 'NULL', 'NULLIF', 'NULLS', 'NUMERIC',
    'OBJECT', 'OF', 'OFF', 'OFFSET', 'OIDS', 'ON', 'ONLY', 'OPERATOR',
    'OPTION', 'OPTIONS', 'OR', 'ORDER', 'OUT', 'OUTER', 'OVER', 'OVERLAPS',
    'OVERLAY', 'OWNED', 'OWNER', 'PARSER', 'PARTIAL', 'PARTITION',
    'PASSING', 'PASSWORD', 'PLACING', 'PLANS', 'POSITION', 'PRECEDING',
    'PRECISION', 'PREPARE', 'PREPARED', 'PRESERVE', 'PRIMARY', 'PRIOR',
    'PRIVILEGES', 'PROCEDURAL', 'PROCEDURE', 'QUOTE', 'RANGE', 'READ',
    'REAL', 'REASSIGN', 'RECHECK', 'RECURSIVE', 'REF', 'REFERENCES',
    'REINDEX', 'RELATIVE', 'RELEASE', 'RENAME', 'REPEATABLE', 'REPLACE',
    'REPLICA', 'RESET', 'RESTART', 'RESTRICT', 'RETURNING', 'RETURNS',
    'REVOKE', 'RIGHT', 'ROLE', 'ROLLBACK', 'ROW', 'ROWS', 'RULE',
    'SAVEPOINT', 'SCHEMA', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY',
    'SELECT', 'SEQUENCE', 'SEQUENCES', 'SERIALIZABLE', 'SERVER', 'SESSION',
    'SESSION_USER', 'SET', 'SETOF', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE',
    'SMALLINT', 'SOME', 'STABLE', 'STANDALONE', 'START', 'STATEMENT',
    'STATISTICS', 'STDIN', 'STDOUT', 'STORAGE', 'STRICT', 'STRIP',
    'SUBSTRING', 'SYMMETRIC', 'SYSID', 'SYSTEM', 'TABLE', 'TABLES',
    'TABLESPACE', 'TEMP', 'TEMPLATE', 'TEMPORARY', 'TEXT', 'THEN', 'TIME',
    'TIMESTAMP', 'TO', 'TRAILING', 'TRANSACTION', 'TREAT', 'TRIGGER',
    'TRIM', 'TRUE', 'TRUNCATE', 'TRUSTED', 'TYPE', 'UNBOUNDED',
    'UNCOMMITTED', 'UNENCRYPTED', 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN',
    'UNLOGGED', 'UNTIL', 'UPDATE', 'USER', 'USING', 'VACUUM', 'VALID',
    'VALIDATE', 'VALIDATOR', 'VALUE', 'VALUES', 'VARCHAR', 'VARIADIC',
    'VARYING', 'VERBOSE', 'VERSION', 'VIEW', 'VOLATILE', 'WHEN', 'WHERE',
    'WHITESPACE', 'WINDOW', 'WITH', 'WITHOUT', 'WORK', 'WRAPPER', 'WRITE',
    'XML', 'XMLATTRIBUTES', 'XMLCONCAT', 'XMLELEMENT', 'XMLEXISTS',
    'XMLFOREST', 'XMLPARSE', 'XMLPI', 'XMLROOT', 'XMLSERIALIZE', 'YEAR',
    'YES', 'ZONE',
    ]

DATATYPES = [
    'bigint', 'bigserial', 'bit', 'bit varying', 'bool', 'boolean', 'box',
    'bytea', 'char', 'character', 'character varying', 'cidr', 'circle',
    'date', 'decimal', 'double precision', 'float4', 'float8', 'inet',
    'int', 'int2', 'int4', 'int8', 'integer', 'interval', 'json', 'line',
    'lseg', 'macaddr', 'money', 'numeric', 'path', 'point', 'polygon',
    'real', 'serial', 'serial2', 'serial4', 'serial8', 'smallint',
    'smallserial', 'text', 'time', 'timestamp', 'timestamptz', 'timetz',
    'tsquery', 'tsvector', 'txid_snapshot', 'uuid', 'varbit', 'varchar',
    'with time zone', 'without time zone', 'xml',
    ]

PSEUDO_TYPES = [
    'any', 'anyelement', 'anyarray', 'anynonarray', 'anyenum', 'anyrange',
    'cstring', 'internal', 'language_handler', 'fdw_handler', 'record',
    'trigger', 'void', 'opaque',
    ]

# Remove 'trigger' from types
PSEUDO_TYPES = sorted(set(PSEUDO_TYPES) - set(map(str.lower, KEYWORDS)))

PLPGSQL_KEYWORDS = [
    'ALIAS', 'CONSTANT', 'DIAGNOSTICS', 'ELSIF', 'EXCEPTION', 'EXIT',
    'FOREACH', 'GET', 'LOOP', 'NOTICE', 'OPEN', 'PERFORM', 'QUERY', 'RAISE',
    'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
    ]

if __name__ == '__main__':
    update_myself()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# core.py
#
# Core class and function definitions

import os
import sys
import datetime
import socket
import pickle
import time
import subprocess
import shlex
import copy

TIME_OUT = 5.0
"""float: how long the socket waits before failing when sending data
"""

USER_DATA_DIR = '~/.neronet' # Remember to os.path.expanduser
USER_DATA_DIR_ABS = os.path.expanduser(USER_DATA_DIR)

MANDATORY_FIELDS = set(['run_command_prefix', 'main_code_file', 'parameters',
                        'parameters_format'])
OPTIONAL_FIELDS = set(['outputs', 'collection', 'required_files',
                        'conditions', 'sbatch_args'])
AUTOMATIC_FIELDS = set(['path', 'time_created', 'time_modified', 'state',
                        'cluster_id', 'warnings'])

class Cluster(object):
    """
    Attributes:
        cid (str): The unique ID (name) of the cluster
        ctype (str): Type of the cluster. Either slurm or unmanaged
        ssh_address (str): SSH address or config hostname corresponding to
            the cluster.
        sbatch_args (str): Slurm SBATCH arguments.
    """

    class Type:
        unmanaged = 'unmanaged'
        slurm = 'slurm'
        _members = set(['slurm', 'unmanaged'])
        @classmethod
        def is_member(cls, arg):
            return arg in cls._members

    def __init__(self, cid, ctype, ssh_address, sbatch_args=None):
        self.cid = cid
        self.ctype = ctype
        self.ssh_address = ssh_address
        self.sbatch_args = sbatch_args
        self.dir = USER_DATA_DIR

    def __str__(self):
        return '%s (%s, %s)' % (self.cid, self.ctype, self.ssh_address)

    def sshrun(self, cmd, inp=None):
        """Execute a shell command via SSH on the remote Neronet cluster."""
        # Ask SSH to execute a command that starts by changing the working
        # directory to 'self.dir' at the machine served at the specified
        # address and port
        scmd = 'ssh %s "cd %s;' % (self.ssh_address, self.dir)
        # Potentially include initialization commands depending on cluster
        # type
        if self.ctype == self.Type.unmanaged:
            pass
        elif self.ctype == self.Type.slurm:
            # Load the python 2.7 module to gain access to the interpreter
            scmd += ' module load python/2.7.4;'
        # Run the given command with the PATH and PYTHONPATH environment
        # variables defined to include the neronet executables and modules
        scmd += ' PATH="%s/neronet:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="%s" %s"' \
                % (self.dir, self.dir, cmd)
        # Actual execution
        res = osrunroe(scmd, inp=inp)
        if res.rv != 0:
            raise RuntimeError('Failed to run "%s" via SSH at cluster "%s"! Err: "%s", Out: "%s".' \
                % (cmd, self.cid, res.err, res.out))
        return res
        # PATH="$HOME/.neronet/neronet:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="$HOME/.neronet"

    def start_neromum(self):
        res = self.sshrun('neromum --start')
        print('Finished: %d, "%s", "%s"' % (res.rv, res.err, res.out))

    def clean_experiments(self, exceptions):
        data = {'action': 'clean_experiments', 'exceptions': exceptions}
        res = self.sshrun('neromum --input', inp=pickle.dumps(data, -1))
        print(res.out)
        if res.err:
            print('Error: %s\n' % (res.err))

    def yield_status(self):
        data = {'action': 'fetch', 'msg': 'I love honeybees!'}
        res = self.sshrun('neromum --input', inp=pickle.dumps(data, -1))
        yield 'Finished: %d, "%s", "%s"' % (res.rv, res.err, res.out)

class Experiment(object):
    """
    Attributes:
        experiment_id (str): Unique identifier to the experiment
        run_command_prefix (str): The run command of the experiment
        main_code_file (str): The code file to be run
        required_files (str): Other files required by the experiments
        logoutput (str): File where the experiment outputs information
        parameters (dict): The Experiment parameters
        parameters_format (str): The format of the experiment parameters
        collection (str): The collection the experiment is part of
        conditions (dict): Special condition for the experiment to do stuff
        state (list of tuples): The states of the experiment with timestamp
        cluster_id (str): The ID of the cluster where the experiment is run
        time_created (datetime): Timestamp of when the experiment was created
        time_modified (datetime): Timestamp of when the experiment was modified
        last
        path (str): Path to the experiment folder
    """

    class State:
        #none = 'none'
        defined = 'defined'
        submitted = 'submitted'
        submitted_to_kid = 'submitted_to_kid'
        lost = 'lost'
        terminated = 'terminated'
        running = 'running'
        finished = 'finished'

    def __init__(self, experiment_id, run_command_prefix, main_code_file,
                    parameters, parameters_format, path, required_files=None,
                    outputs="stdout", collection=None, conditions=None, sbatch_args=None):
        now = datetime.datetime.now()
        fields = {'run_command_prefix': run_command_prefix,
                    'main_code_file': main_code_file,
                    'required_files': required_files if required_files else [],
                    'outputs': outputs if isinstance(outputs, list) else
                                [outputs],
                    'parameters': parameters,
                    'parameters_format': parameters_format,
                    'collection': collection,
                    'conditions': conditions,
                    'sbatch_args': sbatch_args,
                    'path': path,
                    'time_created': now,
                    'time_modified': now,
                    'states_info': [(Experiment.State.defined, now)],
                    'cluster_id': None,
                    'warnings' : [] }
        #MAGIC: Creates the attributes for the experiment class
        self.__dict__['_fields'] = fields
        #super(Experiment, self).__setattr__('_fields', fields)
        super(Experiment, self).__setattr__('_experiment_id', experiment_id)

    def get_action(self, logrow):
        init_action = ('no action', '')
        try:
            for key in self._fields['conditions']:
                action = self._fields['conditions'][key].get_action(logrow)
                if action == 'kill':
                    return (action, key)
                elif action != 'no action':
                    init_action = (action, key)
        except TypeError:
            return init_action
        return init_action

    def set_warning(self, warning):
        self._fields['warnings'].append(str(datetime.datetime.now()) + ": The condition '" + warning + "' was met")

    def set_multiple_warnings(self, warnings):
        self._fields['warnings'] = warnings

    def has_warnings(self):
        if self._fields['warnings']:
            return 'WARNING'
        else:
            return ''

    def get_warnings(self):
        return self._fields['warnings']

    def __getattr__(self, attr):
        """Getter for the experiment class hides the inner dictionary"""
        #Gets the inner dictionary
        fields = super(Experiment, self).__getattribute__('_fields')
        if attr in fields:
            return fields[attr]
        #Gets hidden attributes by adding _
        if attr == 'id':
            attr = 'experiment_id'
        elif attr == 'state':
            return fields['states_info'][-1][0]
        elif attr == 'state_info':
            return fields['states_info'][-1]
        return super(Experiment, self).__getattribute__('_' + attr)

    def __setattr__(self, attr, value):
        """Setter for the experiment class

        Raises:
            AttributeError: if the attribute doesn't exist
        """
        #Gets the inner dictionary
        fields = super(Experiment, self).__getattribute__('_fields')
        if attr == 'id' or attr == 'experiment_id':
            super(Experiment, self).__setattr__('_experiment_id', value)
        elif attr in fields or attr in ('log_output', ):
            fields[attr] = value
        else:
            raise AttributeError('Experiment has no attribute named "%s"!' % attr)

    @property
    def callstring(self):
        rcmd = self._fields['run_command_prefix']
        code_file = self._fields['main_code_file']
        parameters = self._fields['parameters']
        param_format = self._fields['parameters_format']
        parameters_string = param_format.format(**parameters)
        callstring = ' '.join([rcmd, code_file, parameters_string])
        return callstring

    def update_state(self, state):
        """ Updates the state
        """
        if state == self.state: return
        self._fields['states_info'].append((state, datetime.datetime.now()))

    def as_dict(self):
        """ Returns the experiment as a dictionary
        """
        return {self._experiment_id: copy.deepcopy(self._fields)}

    def as_gen(self):
        """Creates a generate that generates info about the experiment
        Yields:
            str: A line of experiment status
        """
        yield "%s\n" % self._experiment_id
        yield "  Run command: %s\n" % self._fields['run_command_prefix']
        yield "  Main code file: %s\n" % self._fields['main_code_file']
        params = self._fields['parameters_format'].format( \
            **self._fields['parameters'])
        yield "  Parameters: %s\n" % params
        yield "  Parameters format: %s\n" % self._fields['parameters_format']
        if self._fields['collection']:
            yield "  Collection: %s\n" % self._fields['collection']
        yield "  State: %s\n" % self.state
        if self._fields['cluster_id']:
            yield "  Cluster: " + self._fields['cluster_id'] + '\n'
        yield "  Last modified: %s\n" % self._fields['time_modified']
        if self._fields['conditions']:
            conds = '  Conditions:\n'
            for condition in self._fields['conditions']:
                conds +=  '    ' + self._fields['conditions'][condition].name + ':\n'
                conds +=  '      variablename: ' + self._fields['conditions'][condition].varname + '\n'
                conds +=  '      killvalue: ' + str(self._fields['conditions'][condition].killvalue) + '\n'
                conds +=  '      comparator: ' + self._fields['conditions'][condition].comparator + '\n'
                conds +=  '      when: ' + self._fields['conditions'][condition].when + '\n'
                conds +=  '      action: ' + self._fields['conditions'][condition].action + '\n'
            yield conds
        if self._fields['warnings']:
            warns = '  Warnings:\n'
            for warn in self._fields['warnings']:
                warns += '    ' + warn + '\n'
            yield warns

    def __str__(self):
        return "%s %s" % (self._experiment_id, self._fields['state'][-1][0])

class Runresult: """A class for holding shell command execution results."""

def osrunroe(cmd, vrb=True, inp=None):
    """Execute a shell command and return the return code, stdout and -err.

    Args:
        vrb (boolean): Whether to print command or not.
        inp (str): Input string for the sub process.

    Returns
        Runresult: The result object of the executed command.
    """
    if vrb: print('> %s' % (cmd))
    if type(cmd) == str:
        cmd = shlex.split(cmd)
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    res = Runresult()
    res.cmd = cmd
    res.out, res.err = proc.communicate(inp)
    res.rv = proc.poll()
    return res

def osrun(cmd, vrb=True):
    res = osrunroe(cmd, vrb)
    if res.rv != 0:
        raise RuntimeError('osrun(%s) failed! Err: "%s", Out: "%s".' \
                % (res.cmd, res.err, res.out))
    return res

def osrunq(cmd):
    return osrun(cmd, vrb=False)

def get_hostname():
    return osrunq('hostname').out.strip()

def time_now():
    return datetime.datetime.now() #.strftime('%H:%M:%S %d-%m-%Y')

def write_file(filepath, data):
    text = str(data)
    with open(filepath, 'w') as f:
        f.write(text)

def read_file(filepath, default=None):
    result = default
    try:
        with open(filepath, 'r') as f:
            result = ''
            for line in f:
                result += line
    except IOError as e:
        pass
    return result

def create_config_template():
    # Creates an empty experiment config file with the required fields.
    if os.path.exists('template.yaml'):
        print('A config template already exists in this folder.')
        return
    with open('template.yaml', 'w') as f:
        f.write('###Mandatory fields###\n')
        for field in MANDATORY_FIELDS:
            f.write('%s: \n' % field)
        f.write('###Optional fields###\n')
        for field in OPTIONAL_FIELDS:
            f.write('%s: \n' % field)

WARNING_FIELDS = set(['variablename', 'killvalue', 'comparator', 'when', \
                        'action'])

class ExperimentWarning:

    def __init__(self, name, variablename, killvalue, comparator, when, action):
        self.name = name.strip()
        self.varname = variablename.strip()
        self.killvalue = killvalue
        self.comparator = comparator.strip()
        self.when = when.strip()
        self.action = action.strip()
        self.start_time = datetime.datetime.now()

    def get_action(self, logrow):
        logrow = logrow.strip()
        varlen = len(self.varname)
        check_condition = True
        if 'time' in self.when:
            time_when = float(self.when[4:].strip())
            time_passed = datetime.datetime.now() - self.start_time
            time_passed_sec = time_passed.days * 86400 + time_passed.seconds
            time_passed_min = time_passed_sec / 60
            if time_passed_min < time_when:
                check_condition = False
        if check_condition and logrow[:varlen].strip() == self.varname:
            varvalue = logrow[varlen:].strip()
            try:
                varvalue = float(varvalue)
            except:
                return 'no action'
            if any( [self.comparator == 'gt' and varvalue > self.killvalue,
                self.comparator == 'lt' and varvalue < self.killvalue,
                self.comparator == 'eq' and varvalue == self.killvalue,
                self.comparator == 'geq' and varvalue >= self.killvalue,
                self.comparator == 'leq' and varvalue <= self.killvalue] ):
                return self.action
        return 'no action'

    def __eq__(self, other):
        if not other:
            return False
        for value in ['name', 'varname', 'killvalue', 'comparator', 'when', 'action']:
            if getattr(self, value) != getattr(other, value): return False
        return True<commit_msg>Debugged experiment condition start time<commit_after># -*- coding: utf-8 -*-
# core.py
#
# Core class and function definitions

import os
import sys
import datetime
import socket
import pickle
import time
import subprocess
import shlex
import copy

TIME_OUT = 5.0
"""float: how long the socket waits before failing when sending data
"""

USER_DATA_DIR = '~/.neronet' # Remember to os.path.expanduser
USER_DATA_DIR_ABS = os.path.expanduser(USER_DATA_DIR)

MANDATORY_FIELDS = set(['run_command_prefix', 'main_code_file', 'parameters',
                        'parameters_format'])
OPTIONAL_FIELDS = set(['outputs', 'collection', 'required_files',
                        'conditions', 'sbatch_args'])
AUTOMATIC_FIELDS = set(['path', 'time_created', 'time_modified', 'state',
                        'cluster_id', 'warnings'])

class Cluster(object):
    """
    Attributes:
        cid (str): The unique ID (name) of the cluster
        ctype (str): Type of the cluster. Either slurm or unmanaged
        ssh_address (str): SSH address or config hostname corresponding to
            the cluster.
        sbatch_args (str): Slurm SBATCH arguments.
    """

    class Type:
        unmanaged = 'unmanaged'
        slurm = 'slurm'
        _members = set(['slurm', 'unmanaged'])
        @classmethod
        def is_member(cls, arg):
            return arg in cls._members

    def __init__(self, cid, ctype, ssh_address, sbatch_args=None):
        self.cid = cid
        self.ctype = ctype
        self.ssh_address = ssh_address
        self.sbatch_args = sbatch_args
        self.dir = USER_DATA_DIR

    def __str__(self):
        return '%s (%s, %s)' % (self.cid, self.ctype, self.ssh_address)

    def sshrun(self, cmd, inp=None):
        """Execute a shell command via SSH on the remote Neronet cluster."""
        # Ask SSH to execute a command that starts by changing the working
        # directory to 'self.dir' at the machine served at the specified
        # address and port
        scmd = 'ssh %s "cd %s;' % (self.ssh_address, self.dir)
        # Potentially include initialization commands depending on cluster
        # type
        if self.ctype == self.Type.unmanaged:
            pass
        elif self.ctype == self.Type.slurm:
            # Load the python 2.7 module to gain access to the interpreter
            scmd += ' module load python/2.7.4;'
        # Run the given command with the PATH and PYTHONPATH environment
        # variables defined to include the neronet executables and modules
        scmd += ' PATH="%s/neronet:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="%s" %s"' \
                % (self.dir, self.dir, cmd)
        # Actual execution
        res = osrunroe(scmd, inp=inp)
        if res.rv != 0:
            raise RuntimeError('Failed to run "%s" via SSH at cluster "%s"! Err: "%s", Out: "%s".' \
                % (cmd, self.cid, res.err, res.out))
        return res
        # PATH="$HOME/.neronet/neronet:/usr/local/bin:/usr/bin:/bin" PYTHONPATH="$HOME/.neronet"

    def start_neromum(self):
        res = self.sshrun('neromum --start')
        print('Finished: %d, "%s", "%s"' % (res.rv, res.err, res.out))

    def clean_experiments(self, exceptions):
        data = {'action': 'clean_experiments', 'exceptions': exceptions}
        res = self.sshrun('neromum --input', inp=pickle.dumps(data, -1))
        print(res.out)
        if res.err:
            print('Error: %s\n' % (res.err))

    def yield_status(self):
        data = {'action': 'fetch', 'msg': 'I love honeybees!'}
        res = self.sshrun('neromum --input', inp=pickle.dumps(data, -1))
        yield 'Finished: %d, "%s", "%s"' % (res.rv, res.err, res.out)

class Experiment(object):
    """
    Attributes:
        experiment_id (str): Unique identifier to the experiment
        run_command_prefix (str): The run command of the experiment
        main_code_file (str): The code file to be run
        required_files (str): Other files required by the experiments
        logoutput (str): File where the experiment outputs information
        parameters (dict): The Experiment parameters
        parameters_format (str): The format of the experiment parameters
        collection (str): The collection the experiment is part of
        conditions (dict): Special condition for the experiment to do stuff
        state (list of tuples): The states of the experiment with timestamp
        cluster_id (str): The ID of the cluster where the experiment is run
        time_created (datetime): Timestamp of when the experiment was created
        time_modified (datetime): Timestamp of when the experiment was modified
        last
        path (str): Path to the experiment folder
    """

    class State:
        #none = 'none'
        defined = 'defined'
        submitted = 'submitted'
        submitted_to_kid = 'submitted_to_kid'
        lost = 'lost'
        terminated = 'terminated'
        running = 'running'
        finished = 'finished'

    def __init__(self, experiment_id, run_command_prefix, main_code_file,
                    parameters, parameters_format, path, required_files=None,
                    outputs="stdout", collection=None, conditions=None, sbatch_args=None):
        now = datetime.datetime.now()
        fields = {'run_command_prefix': run_command_prefix,
                    'main_code_file': main_code_file,
                    'required_files': required_files if required_files else [],
                    'outputs': outputs if isinstance(outputs, list) else
                                [outputs],
                    'parameters': parameters,
                    'parameters_format': parameters_format,
                    'collection': collection,
                    'conditions': conditions,
                    'sbatch_args': sbatch_args,
                    'path': path,
                    'time_created': now,
                    'time_modified': now,
                    'states_info': [(Experiment.State.defined, now)],
                    'cluster_id': None,
                    'warnings' : [] }
        #MAGIC: Creates the attributes for the experiment class
        self.__dict__['_fields'] = fields
        #super(Experiment, self).__setattr__('_fields', fields)
        super(Experiment, self).__setattr__('_experiment_id', experiment_id)

    def get_action(self, logrow):
        init_action = ('no action', '')
        try:
            for key in self._fields['conditions']:
                action = self._fields['conditions'][key].get_action(logrow)
                if action == 'kill':
                    return (action, key)
                elif action != 'no action':
                    init_action = (action, key)
        except TypeError:
            return init_action
        return init_action

    def set_warning(self, warning):
        self._fields['warnings'].append(str(datetime.datetime.now()) + ": The condition '" + warning + "' was met")

    def set_multiple_warnings(self, warnings):
        self._fields['warnings'] = warnings

    def has_warnings(self):
        if self._fields['warnings']:
            return 'WARNING'
        else:
            return ''

    def get_warnings(self):
        return self._fields['warnings']

    def __getattr__(self, attr):
        """Getter for the experiment class hides the inner dictionary"""
        #Gets the inner dictionary
        fields = super(Experiment, self).__getattribute__('_fields')
        if attr in fields:
            return fields[attr]
        #Gets hidden attributes by adding _
        if attr == 'id':
            attr = 'experiment_id'
        elif attr == 'state':
            return fields['states_info'][-1][0]
        elif attr == 'state_info':
            return fields['states_info'][-1]
        return super(Experiment, self).__getattribute__('_' + attr)

    def __setattr__(self, attr, value):
        """Setter for the experiment class

        Raises:
            AttributeError: if the attribute doesn't exist
        """
        #Gets the inner dictionary
        fields = super(Experiment, self).__getattribute__('_fields')
        if attr == 'id' or attr == 'experiment_id':
            super(Experiment, self).__setattr__('_experiment_id', value)
        elif attr in fields or attr in ('log_output', ):
            fields[attr] = value
        else:
            raise AttributeError('Experiment has no attribute named "%s"!' % attr)

    @property
    def callstring(self):
        rcmd = self._fields['run_command_prefix']
        code_file = self._fields['main_code_file']
        parameters = self._fields['parameters']
        param_format = self._fields['parameters_format']
        parameters_string = param_format.format(**parameters)
        callstring = ' '.join([rcmd, code_file, parameters_string])
        return callstring

    def update_state(self, state):
        """ Updates the state
        """
        if state == self.state: return
        if state == 'running' and self._fields['conditions']:
            for c in self._fields['conditions']:
                self._fields['conditions'][c].start_time = datetime.datetime.now()
        self._fields['states_info'].append((state, datetime.datetime.now()))

    def as_dict(self):
        """ Returns the experiment as a dictionary
        """
        return {self._experiment_id: copy.deepcopy(self._fields)}

    def as_gen(self):
        """Creates a generate that generates info about the experiment
        Yields:
            str: A line of experiment status
        """
        yield "%s\n" % self._experiment_id
        yield "  Run command: %s\n" % self._fields['run_command_prefix']
        yield "  Main code file: %s\n" % self._fields['main_code_file']
        params = self._fields['parameters_format'].format( \
            **self._fields['parameters'])
        yield "  Parameters: %s\n" % params
        yield "  Parameters format: %s\n" % self._fields['parameters_format']
        if self._fields['collection']:
            yield "  Collection: %s\n" % self._fields['collection']
        yield "  State: %s\n" % self.state
        if self._fields['cluster_id']:
            yield "  Cluster: " + self._fields['cluster_id'] + '\n'
        yield "  Last modified: %s\n" % self._fields['time_modified']
        if self._fields['conditions']:
            conds = '  Conditions:\n'
            for condition in self._fields['conditions']:
                conds +=  '    ' + self._fields['conditions'][condition].name + ':\n'
                conds +=  '      variablename: ' + self._fields['conditions'][condition].varname + '\n'
                conds +=  '      killvalue: ' + str(self._fields['conditions'][condition].killvalue) + '\n'
                conds +=  '      comparator: ' + self._fields['conditions'][condition].comparator + '\n'
                conds +=  '      when: ' + self._fields['conditions'][condition].when + '\n'
                conds +=  '      action: ' + self._fields['conditions'][condition].action + '\n'
            yield conds
        if self._fields['warnings']:
            warns = '  Warnings:\n'
            for warn in self._fields['warnings']:
                warns += '    ' + warn + '\n'
            yield warns

    def __str__(self):
        return "%s %s" % (self._experiment_id, self._fields['state'][-1][0])

class Runresult: """A class for holding shell command execution results."""

def osrunroe(cmd, vrb=True, inp=None):
    """Execute a shell command and return the return code, stdout and -err.

    Args:
        vrb (boolean): Whether to print command or not.
        inp (str): Input string for the sub process.

    Returns
        Runresult: The result object of the executed command.
    """
    if vrb: print('> %s' % (cmd))
    if type(cmd) == str:
        cmd = shlex.split(cmd)
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    res = Runresult()
    res.cmd = cmd
    res.out, res.err = proc.communicate(inp)
    res.rv = proc.poll()
    return res

def osrun(cmd, vrb=True):
    res = osrunroe(cmd, vrb)
    if res.rv != 0:
        raise RuntimeError('osrun(%s) failed! Err: "%s", Out: "%s".' \
                % (res.cmd, res.err, res.out))
    return res

def osrunq(cmd):
    return osrun(cmd, vrb=False)

def get_hostname():
    return osrunq('hostname').out.strip()

def time_now():
    return datetime.datetime.now() #.strftime('%H:%M:%S %d-%m-%Y')

def write_file(filepath, data):
    text = str(data)
    with open(filepath, 'w') as f:
        f.write(text)

def read_file(filepath, default=None):
    result = default
    try:
        with open(filepath, 'r') as f:
            result = ''
            for line in f:
                result += line
    except IOError as e:
        pass
    return result

def create_config_template():
    # Creates an empty experiment config file with the required fields.
    if os.path.exists('template.yaml'):
        print('A config template already exists in this folder.')
        return
    with open('template.yaml', 'w') as f:
        f.write('###Mandatory fields###\n')
        for field in MANDATORY_FIELDS:
            f.write('%s: \n' % field)
        f.write('###Optional fields###\n')
        for field in OPTIONAL_FIELDS:
            f.write('%s: \n' % field)

WARNING_FIELDS = set(['variablename', 'killvalue', 'comparator', 'when', \
                        'action'])

class ExperimentWarning:

    def __init__(self, name, variablename, killvalue, comparator, when, action):
        self.name = name.strip()
        self.varname = variablename.strip()
        self.killvalue = killvalue
        self.comparator = comparator.strip()
        self.when = when.strip()
        self.action = action.strip()
        self.start_time = datetime.datetime.now()

    def get_action(self, logrow):
        logrow = logrow.strip()
        varlen = len(self.varname)
        check_condition = True
        if 'time' in self.when:
            time_when = float(self.when[4:].strip())
            time_passed = datetime.datetime.now() - self.start_time
            time_passed_sec = time_passed.days * 86400 + time_passed.seconds
            time_passed_min = time_passed_sec / 60
            if time_passed_min < time_when:
                check_condition = False
        if check_condition and logrow[:varlen].strip() == self.varname:
            varvalue = logrow[varlen:].strip()
            try:
                varvalue = float(varvalue)
            except:
                return 'no action'
            if any( [self.comparator == 'gt' and varvalue > self.killvalue,
                self.comparator == 'lt' and varvalue < self.killvalue,
                self.comparator == 'eq' and varvalue == self.killvalue,
                self.comparator == 'geq' and varvalue >= self.killvalue,
                self.comparator == 'leq' and varvalue <= self.killvalue] ):
                return self.action
        return 'no action'

    def __eq__(self, other):
        if not other:
            return False
        for value in ['name', 'varname', 'killvalue', 'comparator', 'when', 'action']:
            if getattr(self, value) != getattr(other, value): return False
        return True
<|endoftext|>


<commit_before>"""
Module containing functions to compile and differentiate Theano graphs. Part of
the pymanopt package.

Jamie Townsend December 2014
"""
import itertools

try:
    import theano
    import theano.tensor as T
    from theano.gradient import disconnected_grad
except ImportError:
    theano = None
    T = None

from ._backend import Backend, assert_backend_available
from .. import make_graph_backend_decorator
from ...tools import unpack_arguments, flatten_args, group_return_values

class TheanoBackend(Backend):
    def __str__(self):
        return "theano"

    @staticmethod
    def is_available():
        return theano is not None

    @assert_backend_available
    def is_compatible(self, func, argument):
        args = flatten_args(argument)
        return all([isinstance(arg, T.TensorVariable) for arg in args])

    @assert_backend_available
    def compile_function(self, func, argument):
        """
        Wrapper for theano.function(). Compiles a theano graph into a python
        function.
        """
        args = flatten_args(argument)
        f = theano.function(inputs=args, outputs=func)
        if len(args) == 1:
            return f
        return unpack_arguments(f)

    @assert_backend_available
    def compute_gradient(self, func, argument):
        """
        Returns a compiled function computing the gradient of 'func' with
        respect to 'argument'.
        """
        args = flatten_args(argument)
        # For cost functions expecting one argument, make sure we don't return
        # the gradient evaluated at some point as a singleton tuple.
        if len(args) == 1:
            (arg,) = args
            g = T.grad(func, arg)
            return theano.function(args, g)

        g = T.grad(func, args)
        grad = theano.function(args, g)
        return group_return_values(unpack_arguments(grad), argument)

    @staticmethod
    def _compute_unary_hvp(g, argument):
        """
        Returns a function accepting two arguments to compute a Hessian-vector
        product of a scalar-valued unary function.
        """
        u = argument.type()
        try:
            R = T.Rop(g, argument, u)
        except NotImplementedError:
            proj = T.sum(g * disconnected_grad(u))
            R = T.grad(proj, argument)
        return theano.function([argument, u], R, on_unused_input="warn")

    @staticmethod
    def _compute_nary_hvp(g, args):
        """
        Returns a function accepting 2 * len(args) arguments to compute a
        Hessian-vector product of a multivariate function defined on a product
        manifold.
        """
        u = [arg.type() for arg in args]
        try:
            R = T.Rop(g, args, u)
        except NotImplementedError:
            # TODO(nkoep): Write a test case for this path.
            # Implementation based on:
            #   tensorflow.python.ops.gradients_impl._hessian_vector_product
            proj = [T.sum(gi * disconnected_grad(ui)) for gi, ui in zip(g, u)]
            proj_grad = [T.grad(proj_elem, args,
                                disconnected_inputs="ignore",
                                return_disconnected="None")
                         for proj_elem in proj]
            proj_grad_transpose = map(list, zip(*proj_grad))
            proj_grad_stack = [
                T.stacklists([c for c in row if c is not None])
                for row in proj_grad_transpose]
            R = [T.sum(stack, axis=0) for stack in proj_grad_stack]
        return theano.function(list(itertools.chain(args, u)), R,
                               on_unused_input="warn")

    @assert_backend_available
    def compute_hessian(self, func, argument):
        """
        Computes the directional derivative of the gradient (which is equal to
        the Hessian multiplied by direction).
        """
        # See above.
        args = flatten_args(argument)
        if len(args) == 1:
            (arg,) = args
            g = T.grad(func, arg)
            return self._compute_unary_hvp(g, arg)

        g = T.grad(func, args)
        hess = self._compute_nary_hvp(g, args)
        def wrapper(x, u):
            return hess(*itertools.chain(flatten_args(x), flatten_args(u)))
        return group_return_values(wrapper, argument)

Theano = make_graph_backend_decorator(TheanoBackend)
<commit_msg>_theano.py: use more explicit variable names<commit_after>import itertools

try:
    import theano
    import theano.tensor as T
    from theano.gradient import disconnected_grad
except ImportError:
    theano = None
    T = None

from ._backend import Backend, assert_backend_available
from .. import make_graph_backend_decorator
from ...tools import unpack_arguments, flatten_args, group_return_values

class _TheanoBackend(Backend):
    def __str__(self):
        return "theano"

    @staticmethod
    def is_available():
        return theano is not None

    @assert_backend_available
    def is_compatible(self, func, argument):
        args = flatten_args(argument)
        return all([isinstance(arg, T.TensorVariable) for arg in args])

    @assert_backend_available
    def compile_function(self, function, arguments):
        """Compiles a Theano graph into a python function."""
        flattened_arguments = flatten_args(arguments)
        compiled_function = theano.function(inputs=flattened_arguments,
                                            outputs=function)
        if len(flattened_arguments) == 1:
            return compiled_function
        return unpack_arguments(compiled_function)

    @assert_backend_available
    def compute_gradient(self, function, arguments):
        """Returns a compiled function computing the gradient of 'function'
        with respect to 'arguments'.
        """
        flattened_arguments = flatten_args(arguments)
        # For cost functions expecting one argument, make sure we don't return
        # the gradient evaluated at some point as a singleton tuple.
        if len(flattened_arguments) == 1:
            (argument,) = flattened_arguments
            gradient = T.grad(function, argument)
            return theano.function(flattened_arguments, gradient)

        gradient = T.grad(function, flattened_arguments)
        compiled_gradient = theano.function(flattened_arguments, gradient)
        return group_return_values(
            unpack_arguments(compiled_gradient), arguments)

    @staticmethod
    def _compute_unary_hessian_vector_product(gradient, argument):
        """Returns a function accepting two arguments to compute a
        Hessian-vector product of a scalar-valued unary function.
        """
        argument_type = argument.type()
        try:
            Rop = T.Rop(gradient, argument, argument_type)
        except NotImplementedError:
            proj = T.sum(gradient * disconnected_grad(argument_type))
            Rop = T.grad(proj, argument)
        return theano.function([argument, argument_type], Rop,
                               on_unused_input="ignore")

    @staticmethod
    def _compute_nary_hessian_vector_product(gradients, arguments):
        """Returns a function accepting `2 * len(arguments)` arguments to
        compute a Hessian-vector product of a multivariate function.

        Notes
        -----
        The implementation is based on TensorFlow's '_hessian_vector_product'
        function in 'tensorflow.python.ops.gradients_impl'.
        """
        argument_types = [argument.type() for argument in arguments]
        try:
            Rop = T.Rop(gradients, arguments, argument_types)
        except NotImplementedError:
            proj = [T.sum(gradient * disconnected_grad(argument_type))
                    for gradient, argument_type in zip(gradients,
                                                       argument_types)]
            proj_grad = [T.grad(proj_elem, arguments,
                                disconnected_inputs="ignore",
                                return_disconnected="None")
                         for proj_elem in proj]
            proj_grad_transpose = map(list, zip(*proj_grad))
            proj_grad_stack = [
                T.stacklists([c for c in row if c is not None])
                for row in proj_grad_transpose]
            Rop = [T.sum(stack, axis=0) for stack in proj_grad_stack]
        return theano.function(
            list(itertools.chain(arguments, argument_types)), Rop,
            on_unused_input="warn")

    @assert_backend_available
    def compute_hessian(self, function, arguments):
        """Computes the directional derivative of the gradient, which is
        equivalent to computing a Hessian-vector product with the direction
        vector.
        """
        flattened_arguments = flatten_args(arguments)
        if len(flattened_arguments) == 1:
            (argument,) = flattened_arguments
            gradient = T.grad(function, argument)
            return self._compute_unary_hessian_vector_product(
                gradient, argument)

        gradients = T.grad(function, flattened_arguments)
        hessian_vector_product = self._compute_nary_hessian_vector_product(
            gradients, flattened_arguments)

        def wrapper(point, vector):
            return hessian_vector_product(
                *itertools.chain(flatten_args(point), flatten_args(vector)))

        return group_return_values(wrapper, arguments)

Theano = make_graph_backend_decorator(_TheanoBackend)
<|endoftext|>