<commit_before>"""

The config module stores the default values for various tunable parameters
in the TKP Python package.

The module also tries to read (in order):

- a :file:`.transientskp/tkp.cfg` file in the users home directory

- a :file:`.tkp.cfg` file in the users home directory,

where parameters can be overriden (such as the database login details).

There are three sections in the configuration:

- database

- source_association

- source_extraction

The parameters are stored in a dictionary, with each section as keyword.
Each value in itself is another dictionary, with (option, value) pairs.

The config module can be imported by the various subpackages of the
TKP package.  Since the :file:`.transientskp/tkp.cfg` or
:file:`.tkp.cfg` file is only read on first import, after which the
variable HAS_READ is set to False, there are no multiple reads of this
file.

"""

from __future__ import with_statement
import ConfigParser
import os
import os.path
import logging

_TO_DO = """\

To do:

- avoid the HAS_READ / NameError trick below,
  possibly by use of a singleton

- (optional) use class instead of dictionary to store options

"""

TKPCFGFILE = ('~/.transientskp/tkp.cfg', '~/.tkp.cfg')

# Avoid eval
# This is very simple, and likely may be fooled by incorrect
# input
def double_list_from_string(text, contenttype=str):
    """Helper function to parse a double list from a string"""

    origtext = text[:]
    if contenttype not in (str, int, float, bool):
        raise ValueError("unknown or not allowed contenttype")
    text = origtext.strip()
    if text[0] != '[':
        raise ValueError("%s does not start with a list" % origtext)
    text = text[1:].strip()
    if text[0] != '[':
        raise ValueError("%s is not a valid double list" % origtext)
    text = text[1:].strip()
    elements = []
    while True:
        i = text.find(']')
        part = text[:i]
        #elements.append(map(contenttype, part.split(',')))
        elements.append([contenttype(var) for var in part.split(',')])
        try:
            text = text[i+1:].strip()
        except IndexError:
            raise ValueError("%s is incorrect format" % origtext)
        if text[0] == ']':  # closing outer list
            break
        tmptext = text.lstrip(',').strip()
        if tmptext == text:
            raise ValueError("missing separating comma in %s" % origtext)
        text = tmptext.lstrip('[').strip()
        if tmptext == text:
            raise ValueError("missing opening bracket in %s" % origtext)

    return elements

def set_default_config():
    """Set up the default configuration"""

    config = ConfigParser.SafeConfigParser()

    config.add_section('database')
    config.set('database', 'enabled', 'True')
    config.set('database', 'host', 'ldb001')
    config.set('database', 'name', 'tkp')
    config.set('database', 'user', 'tkp')
    config.set('database', 'password', 'tkp')
    config.set('database', 'port', '50000')

    config.add_section('source_association')
    config.set('source_association', 'deruiter_radius', '0.0010325')

    config.add_section('source_extraction')
    config.set('source_extraction', 'back_sizex', '32')
    config.set('source_extraction', 'back_sizey', '32')
    config.set('source_extraction', 'median_filter', '0')
    config.set('source_extraction', 'mf_threshold', '0')
    config.set('source_extraction', 'interpolate_order', '1')
    config.set('source_extraction', 'margin', '0')
    config.set('source_extraction', 'max_degradation', '0.2')
    config.set('source_extraction', 'fdr_alpha', '1e-2')
    config.set('source_extraction', 'structuring_element',
               '[[0,1,0], [1,1,1], [0,1,0]]')
    config.set('source_extraction', 'deblend', 'False')
    config.set('source_extraction', 'deblend_nthresh', '32')
    config.set('source_extraction', 'deblend_mincont', '0.005')
    config.set('source_extraction', 'detection_threshold', '10.0')
    config.set('source_extraction', 'analysis_threshold', '3.0')
    config.set('source_extraction', 'residuals', 'True')
    config.set('source_extraction', 'alpha_maj1', '2.5')
    config.set('source_extraction', 'alpha_min1', '0.5')
    config.set('source_extraction', 'alpha_maj2', '0.5')
    config.set('source_extraction', 'alpha_min2', '2.5')
    config.set('source_extraction', 'alpha_maj3', '1.5')
    config.set('source_extraction', 'alpha_min3', '1.5')
    config.set('source_extraction', 'clean_bias', '0.0')
    config.set('source_extraction', 'clean_bias_error', '0.0')
    config.set('source_extraction', 'frac_flux_cal_error', '0.0')
    config.set('source_extraction', 'eps_ra', '0.')
    config.set('source_extraction', 'eps_dec', '0.')

    config.add_section('logging')
    config.set('logging', 'level', 'ERROR')
    config.set('logging', 'format',
               '%%(asctime)s - %%(name)s - %%(levelname)s - %%(message)s')
    config.set('logging', 'filename', '')

    from tkp.tests import __path__ as testpath
    config.add_section('test')
    config.set('test', 'datapath', os.path.join(testpath[0], "data"))
    return config

def write_config(config=None, filename=TKPCFGFILE[0]):
    """
    Dump configuration to file.

    Convenient for generating an initial template configuration file for the
    user to edit to suit their requirements.
    """
    if not config:
        config=set_default_config()
    filename = os.path.expanduser(filename)
    try:
        os.makedirs(os.path.dirname(filename))
    except OSError:   # directory already exists, presumably
        pass
    with open(filename, "w") as cfg_file:
        config.write(cfg_file)

def read_config(default_config):
    """Attempt to read a user configuration file"""

    config = ConfigParser.SafeConfigParser()
    # Don't try and read multiple files (which can be done with
    # ConfigParser.read()); instead, if one is missing, try the other:
    if not config.read(os.path.expanduser(TKPCFGFILE[0])):
        config.read(os.path.expanduser(TKPCFGFILE[1]))

    # Check for unknown sections or options
    for section in config.sections():
        if not default_config.has_section(section):
            raise ConfigParser.Error("unknown section %s" % section)
        for option in config.options(section):
            if not default_config.has_option(section, option):
                raise ConfigParser.Error(
                    "unknown option %s in section %s" % (option, section))
    # Now overwrite default options
    for section in default_config.sections():
        if not config.has_section(section):
            config.add_section(section)
        for option in default_config.options(section):
            if not config.has_option(section, option):
                config.set(section, option,
                           default_config.get(section, option))

    return config

def parse_config(config):
    """Parse the various config parameters into a dictionary,
    including type conversion"""

    # On to do list: create an inherited configparser that stores a type with
    # the options, and then does the parsing behind the scenes
    configuration = dict(database={}, source_association={},
                         source_extraction={})
    booleans = (('database', 'enabled'), ('source_extraction', 'deblend'),
                ('source_extraction', 'residuals'))
    integers = (('database', 'port'), ('source_extraction', 'back_sizex'),
                ('source_extraction', 'back_sizey'),
                ('source_extraction', 'median_filter'),
                ('source_extraction', 'interpolate_order'),
                ('source_extraction', 'deblend_nthresh'))
    floats = (('source_association', 'deruiter_radius'),
              ('source_extraction', 'mf_threshold'),
              ('source_extraction', 'margin'),
              ('source_extraction', 'max_degradation'),
              ('source_extraction', 'fdr_alpha'),
              ('source_extraction', 'deblend_mincont'),
              ('source_extraction', 'detection_threshold'),
              ('source_extraction', 'analysis_threshold'),
              ('source_extraction', 'alpha_maj1'),
              ('source_extraction', 'alpha_min1'),
              ('source_extraction', 'alpha_maj2'),
              ('source_extraction', 'alpha_min2'),
              ('source_extraction', 'alpha_maj3'),
              ('source_extraction', 'alpha_min3'),
              ('source_extraction', 'clean_bias'),
              ('source_extraction', 'clean_bias_error'),
              ('source_extraction', 'frac_flux_cal_error'),
              ('source_extraction', 'eps_ra'),
              ('source_extraction', 'eps_dec'))
    configuration.update(dict([(section, dict(config.items(section, raw=True)))
                               for section in config.sections()]))
    for section, option in booleans:
        try:
            configuration[section][option] = config.getboolean(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be boolean "
        "(True/False)" % (option, section))
    for section, option in integers:
        try:
            configuration[section][option] = config.getint(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be an integer" %
        (option, section))
    for section, option in floats:
        try:
            configuration[section][option] = config.getfloat(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be a real number" %
        (option, section))
    elements = double_list_from_string(config.get(
        'source_extraction', 'structuring_element'), contenttype=float)
    if (len(elements) != 3 or len(elements[0]) != 3 or len(elements[1]) != 3
        or len(elements[2]) != 3):
        raise ValueError("""\
incorrect type for structuring_element in section source_extraction""")
    configuration['source_extraction']['structuring_element'] = elements

    return configuration

def configure_logger(config):
    """Configure the TKP logger"""

    levels = {
        'CRITICAL': logging.CRITICAL,
        'ERROR': logging.ERROR,
        'WARNING': logging.WARNING,
        'INFO': logging.INFO,
        'DEBUG': logging.DEBUG
        }
    # Don't simply take the dict as is.
    # Eg, an empty filename means using a StreamHandler, otherwise a FileHandler
    if config['filename']:
        handler = logging.FileHandler(config['filename'])
    else:
        handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter(config['format']))
    logger = logging.getLogger('tkp')
    logger.addHandler(handler)
    logger.setLevel(levels[config['level'].upper()])

# This is a bit of a dirty trick; using some kind of singleton class
# with a class variable may be better, though I guess it's currently
# similar: a singleton module with module variable.
try:
    HAS_READ
except NameError, exc:
    config = read_config(set_default_config())
    config = parse_config(config)
    configure_logger(config['logging'])
    HAS_READ = True
else:
    pass
<commit_msg>Changed default De Ruiter radius to use arcsec instead of degrees<commit_after>"""

The config module stores the default values for various tunable parameters
in the TKP Python package.

The module also tries to read (in order):

- a :file:`.transientskp/tkp.cfg` file in the users home directory

- a :file:`.tkp.cfg` file in the users home directory,

where parameters can be overriden (such as the database login details).

There are three sections in the configuration:

- database

- source_association

- source_extraction

The parameters are stored in a dictionary, with each section as keyword.
Each value in itself is another dictionary, with (option, value) pairs.

The config module can be imported by the various subpackages of the
TKP package.  Since the :file:`.transientskp/tkp.cfg` or
:file:`.tkp.cfg` file is only read on first import, after which the
variable HAS_READ is set to False, there are no multiple reads of this
file.

"""

from __future__ import with_statement
import ConfigParser
import os
import os.path
import logging

_TO_DO = """\

To do:

- avoid the HAS_READ / NameError trick below,
  possibly by use of a singleton

- (optional) use class instead of dictionary to store options

"""

TKPCFGFILE = ('~/.transientskp/tkp.cfg', '~/.tkp.cfg')

# Avoid eval
# This is very simple, and likely may be fooled by incorrect
# input
def double_list_from_string(text, contenttype=str):
    """Helper function to parse a double list from a string"""

    origtext = text[:]
    if contenttype not in (str, int, float, bool):
        raise ValueError("unknown or not allowed contenttype")
    text = origtext.strip()
    if text[0] != '[':
        raise ValueError("%s does not start with a list" % origtext)
    text = text[1:].strip()
    if text[0] != '[':
        raise ValueError("%s is not a valid double list" % origtext)
    text = text[1:].strip()
    elements = []
    while True:
        i = text.find(']')
        part = text[:i]
        #elements.append(map(contenttype, part.split(',')))
        elements.append([contenttype(var) for var in part.split(',')])
        try:
            text = text[i+1:].strip()
        except IndexError:
            raise ValueError("%s is incorrect format" % origtext)
        if text[0] == ']':  # closing outer list
            break
        tmptext = text.lstrip(',').strip()
        if tmptext == text:
            raise ValueError("missing separating comma in %s" % origtext)
        text = tmptext.lstrip('[').strip()
        if tmptext == text:
            raise ValueError("missing opening bracket in %s" % origtext)

    return elements

def set_default_config():
    """Set up the default configuration"""

    config = ConfigParser.SafeConfigParser()

    config.add_section('database')
    config.set('database', 'enabled', 'True')
    config.set('database', 'host', 'ldb001')
    config.set('database', 'name', 'tkp')
    config.set('database', 'user', 'tkp')
    config.set('database', 'password', 'tkp')
    config.set('database', 'port', '50000')

    config.add_section('source_association')
    config.set('source_association', 'deruiter_radius', '3.7')

    config.add_section('source_extraction')
    config.set('source_extraction', 'back_sizex', '32')
    config.set('source_extraction', 'back_sizey', '32')
    config.set('source_extraction', 'median_filter', '0')
    config.set('source_extraction', 'mf_threshold', '0')
    config.set('source_extraction', 'interpolate_order', '1')
    config.set('source_extraction', 'margin', '0')
    config.set('source_extraction', 'max_degradation', '0.2')
    config.set('source_extraction', 'fdr_alpha', '1e-2')
    config.set('source_extraction', 'structuring_element',
               '[[0,1,0], [1,1,1], [0,1,0]]')
    config.set('source_extraction', 'deblend', 'False')
    config.set('source_extraction', 'deblend_nthresh', '32')
    config.set('source_extraction', 'deblend_mincont', '0.005')
    config.set('source_extraction', 'detection_threshold', '10.0')
    config.set('source_extraction', 'analysis_threshold', '3.0')
    config.set('source_extraction', 'residuals', 'True')
    config.set('source_extraction', 'alpha_maj1', '2.5')
    config.set('source_extraction', 'alpha_min1', '0.5')
    config.set('source_extraction', 'alpha_maj2', '0.5')
    config.set('source_extraction', 'alpha_min2', '2.5')
    config.set('source_extraction', 'alpha_maj3', '1.5')
    config.set('source_extraction', 'alpha_min3', '1.5')
    config.set('source_extraction', 'clean_bias', '0.0')
    config.set('source_extraction', 'clean_bias_error', '0.0')
    config.set('source_extraction', 'frac_flux_cal_error', '0.0')
    config.set('source_extraction', 'eps_ra', '0.')
    config.set('source_extraction', 'eps_dec', '0.')

    config.add_section('logging')
    config.set('logging', 'level', 'ERROR')
    config.set('logging', 'format',
               '%%(asctime)s - %%(name)s - %%(levelname)s - %%(message)s')
    config.set('logging', 'filename', '')

    from tkp.tests import __path__ as testpath
    config.add_section('test')
    config.set('test', 'datapath', os.path.join(testpath[0], "data"))
    return config

def write_config(config=None, filename=TKPCFGFILE[0]):
    """
    Dump configuration to file.

    Convenient for generating an initial template configuration file for the
    user to edit to suit their requirements.
    """
    if not config:
        config=set_default_config()
    filename = os.path.expanduser(filename)
    try:
        os.makedirs(os.path.dirname(filename))
    except OSError:   # directory already exists, presumably
        pass
    with open(filename, "w") as cfg_file:
        config.write(cfg_file)

def read_config(default_config):
    """Attempt to read a user configuration file"""

    config = ConfigParser.SafeConfigParser()
    # Don't try and read multiple files (which can be done with
    # ConfigParser.read()); instead, if one is missing, try the other:
    if not config.read(os.path.expanduser(TKPCFGFILE[0])):
        config.read(os.path.expanduser(TKPCFGFILE[1]))

    # Check for unknown sections or options
    for section in config.sections():
        if not default_config.has_section(section):
            raise ConfigParser.Error("unknown section %s" % section)
        for option in config.options(section):
            if not default_config.has_option(section, option):
                raise ConfigParser.Error(
                    "unknown option %s in section %s" % (option, section))
    # Now overwrite default options
    for section in default_config.sections():
        if not config.has_section(section):
            config.add_section(section)
        for option in default_config.options(section):
            if not config.has_option(section, option):
                config.set(section, option,
                           default_config.get(section, option))

    return config

def parse_config(config):
    """Parse the various config parameters into a dictionary,
    including type conversion"""

    # On to do list: create an inherited configparser that stores a type with
    # the options, and then does the parsing behind the scenes
    configuration = dict(database={}, source_association={},
                         source_extraction={})
    booleans = (('database', 'enabled'), ('source_extraction', 'deblend'),
                ('source_extraction', 'residuals'))
    integers = (('database', 'port'), ('source_extraction', 'back_sizex'),
                ('source_extraction', 'back_sizey'),
                ('source_extraction', 'median_filter'),
                ('source_extraction', 'interpolate_order'),
                ('source_extraction', 'deblend_nthresh'))
    floats = (('source_association', 'deruiter_radius'),
              ('source_extraction', 'mf_threshold'),
              ('source_extraction', 'margin'),
              ('source_extraction', 'max_degradation'),
              ('source_extraction', 'fdr_alpha'),
              ('source_extraction', 'deblend_mincont'),
              ('source_extraction', 'detection_threshold'),
              ('source_extraction', 'analysis_threshold'),
              ('source_extraction', 'alpha_maj1'),
              ('source_extraction', 'alpha_min1'),
              ('source_extraction', 'alpha_maj2'),
              ('source_extraction', 'alpha_min2'),
              ('source_extraction', 'alpha_maj3'),
              ('source_extraction', 'alpha_min3'),
              ('source_extraction', 'clean_bias'),
              ('source_extraction', 'clean_bias_error'),
              ('source_extraction', 'frac_flux_cal_error'),
              ('source_extraction', 'eps_ra'),
              ('source_extraction', 'eps_dec'))
    configuration.update(dict([(section, dict(config.items(section, raw=True)))
                               for section in config.sections()]))
    for section, option in booleans:
        try:
            configuration[section][option] = config.getboolean(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be boolean "
        "(True/False)" % (option, section))
    for section, option in integers:
        try:
            configuration[section][option] = config.getint(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be an integer" %
        (option, section))
    for section, option in floats:
        try:
            configuration[section][option] = config.getfloat(section, option)
        except ValueError:
            raise ValueError(
        "incorrect type for option %s in section %s; must be a real number" %
        (option, section))
    elements = double_list_from_string(config.get(
        'source_extraction', 'structuring_element'), contenttype=float)
    if (len(elements) != 3 or len(elements[0]) != 3 or len(elements[1]) != 3
        or len(elements[2]) != 3):
        raise ValueError("""\
incorrect type for structuring_element in section source_extraction""")
    configuration['source_extraction']['structuring_element'] = elements

    return configuration

def configure_logger(config):
    """Configure the TKP logger"""

    levels = {
        'CRITICAL': logging.CRITICAL,
        'ERROR': logging.ERROR,
        'WARNING': logging.WARNING,
        'INFO': logging.INFO,
        'DEBUG': logging.DEBUG
        }
    # Don't simply take the dict as is.
    # Eg, an empty filename means using a StreamHandler, otherwise a FileHandler
    if config['filename']:
        handler = logging.FileHandler(config['filename'])
    else:
        handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter(config['format']))
    logger = logging.getLogger('tkp')
    logger.addHandler(handler)
    logger.setLevel(levels[config['level'].upper()])

# This is a bit of a dirty trick; using some kind of singleton class
# with a class variable may be better, though I guess it's currently
# similar: a singleton module with module variable.
try:
    HAS_READ
except NameError, exc:
    config = read_config(set_default_config())
    config = parse_config(config)
    configure_logger(config['logging'])
    HAS_READ = True
else:
    pass
<|endoftext|>


<commit_before># Source Generated with Decompyle++
# File: lammps_wrapper.pyc (Python 2.7)

''' LAMMPS SimPhoNy Wrapper

This module provides a wrapper for  LAMMPS-md
'''
import contextlib
import os
import shutil
import tempfile

from io.lammps_fileio_data_manager import LammpsFileIoDataManager
from io.lammps_process import LammpsProcess

from common.atom_style import AtomStyle

from config.script_writer import ScriptWriter

from cuba_extension import CUBAExtension

from internal.lammps_internal_data_manager import LammpsInternalDataManager

from simphony.core import CUBA
from simphony.core.data_container import DataContainer
from simphony.cuds import CUDS
from simphony.cuds.abc_modeling_engine import ABCModelingEngine
from simphony.cuds.abc_particles import ABCParticles
from simphony.cuds.meta import api
import simphony.cuds.particles as scp

def _temp_directory():
    ''' context manager that provides temp directory

    The name of the created temp directory is returned when context is entered
    and this directory is deleted when context is exited
    '''
    temp_dir = tempfile.mkdtemp()
    yield temp_dir
    shutil.rmtree(temp_dir)

_temp_directory = contextlib.contextmanager(_temp_directory)

class LammpsWrapper(ABCModelingEngine):
    ''' Wrapper to LAMMPS-md'''
    def __init__(self, use_internal_interface=False, **kwargs):
        ''' Constructor.

        Parameters
        ----------
        engine_type : EngineType
            type of engine

        use_internal_interface : bool, optional
            If true, then the internal interface (library) is used when
            communicating with LAMMPS, if false, then file-io interface is
            used where input/output files are used to communicate with LAMMPS
        '''

        self.BC = DataContainer()
        self.CM = DataContainer()
        self.SP = DataContainer()
        self.CM_extension = {}
        self.SP_extension = {}
        self.BC_extension = {}
        self.SD = kwargs.get('cuds', CUDS())

        self._use_internal_interface = use_internal_interface
        atom_style = AtomStyle.ATOMIC
        self._executable_name = 'lammps'
        self._script_writer = ScriptWriter(atom_style)

        if self._use_internal_interface:
            import lammps
            # lammps.lammps(cmdargs=["-screen", "none", "-log", "none"])
            self._lammps = lammps.lammps()
            self._data_manager = LammpsInternalDataManager(self._lammps,
                                                           self.SD,
                                                           atom_style)
        else:
            self._data_manager = LammpsFileIoDataManager(self.SD, atom_style)

        # Call the base class in order to load CUDS
        super(LammpsWrapper, self).__init__(**kwargs)

    def _count_of(self, cuds, item_type):
        """Workaround for broken CUDS counter."""
        count = 0
        for c in cuds.iter():
            if isinstance(c, item_type):
                count += 1
        return count

    def _load_cuds(self):
        '''Load CUDS data into lammps engine.'''
        cuds = self.get_cuds()
        if not cuds:
            return

        # FIXME: `count_of` is broken for non-meta classes,
        # e.g. Particles, Particle, Mesh, etc.
        # if cuds.count_of(CUBA.PARTICLES) != 1:
        if self._count_of(cuds, scp.Particles) != 1:
            raise Exception('This version of simlammps'
                            ' needs only one particle dataset, not %s' %
                            self._count_of(cuds, scp.Particles))
        if cuds.count_of(CUBA.MATERIAL) != 1:
            raise Exception('This version of simlammps needs'
                            ' only one material not %s' %
                            cuds.count_of(CUBA.MATERIAL))

        material_to_atom = {}
        number_atom_types = 0
        for mat in cuds.iter(item_type=CUBA.MATERIAL):
            if CUBA.MASS not in mat.data:
                raise Exception('Material needs mass property!')
            number_atom_types += 1
            material_to_atom[mat.uid] = number_atom_types

        for particles in cuds.iter(item_type=CUBA.PARTICLES):
            update_list = []
            for single_particle in particles.iter():
                single_particle.data[CUBA.MATERIAL_TYPE] = mat.uid
                update_list.append(single_particle)
            particles.update(update_list)

        for b in cuds.iter(item_type=CUBA.BOX):
            pass

        particle_sum = 0
        for ds in cuds.iter(item_type=CUBA.PARTICLES):
            particle_sum += len(ds)
            ds.data = mat.data
            ds.data_extension = {
                CUBAExtension.BOX_VECTORS: b.vector}
            self.add_dataset(ds)

        if particle_sum == 0:
            raise Exception('simlammps needs some particles')

        if cuds.count_of(CUBA.MOLECULAR_DYNAMICS) != 1:
            raise Exception('simlammps supports only MD')

        if cuds.count_of(CUBA.BOX) != 1:
            raise Exception('simlammps needs one box')

        for termo in cuds.iter(item_type=CUBA.THERMOSTAT):
            if isinstance(termo, api.TemperatureRescaling):
                self.CM_extension[CUBA.THERMODYNAMIC_ENSEMBLE] = 'NVE'
                continue

        if not cuds.count_of(CUBA.INTEGRATION_TIME):
            raise Exception('Only one integration time setup is accepted')

        for i in cuds.iter(item_type=CUBA.INTEGRATION_TIME):
            self.CM[CUBA.TIME_STEP] = i.step
            self.CM[CUBA.NUMBER_OF_TIME_STEPS] = int(i.final / i.step)

        if cuds.count_of(CUBA.CONDITION) != 1:
            raise Exception('Sorry only one condition is accepted, not %s' %
                            cuds.count_of(CUBA.CONDITION))

        for c in cuds.iter(item_type=CUBA.CONDITION):
            if isinstance(c, api.Periodic):
                self.BC_extension[CUBAExtension.BOX_FACES] = [
                    'periodic',
                    'periodic',
                    'periodic']
                continue
            raise Exception('Sorry, I am confused!')

        if cuds.count_of(CUBA.INTERATOMIC_POTENTIAL) != 1:
            raise Exception('Lammps needs one interatomic potention')

        for ip in cuds.iter(item_type=CUBA.INTERATOMIC_POTENTIAL):
            pass

        self.SP_extension[CUBAExtension.PAIR_POTENTIALS] =\
            'lj:\n  global_cutoff: {global_cutoff}\n'\
            '  parameters:\n  - pair: [{mat1}, {mat2}]\n'\
            '    epsilon: {energy_well_depth}\n'\
            '    sigma: {vanderwaals_radious}\n'\
            '    cutoff: {cutoff}\n'\
            .format(global_cutoff=1.12246,
                    mat1=material_to_atom[ip.material[0].uid],
                    mat2=material_to_atom[ip.material[1].uid],
                    energy_well_depth=ip.energy_well_depth,
                    vanderwaals_radious=ip.van_der_waals_radius,
                    cutoff=ip.cutoff_distance)

    def add_dataset(self, container):
        '''Add a CUDS container

        Parameters
        ----------
        container : {ABCParticles}
            The CUDS container to add to the engine.

        Raises
        ------
        TypeError:
            If the container type is not supported (i.e. ABCLattice, ABCMesh).
        ValueError:
            If there is already a dataset with the given name.

        '''
        if not isinstance(container, ABCParticles):
            raise TypeError(
                'The type of the dataset container is not supported')
        if container.name in self._data_manager:
            raise ValueError("Particle container '{}' already exists"
                             .format(container.name))
        self._data_manager.new_particles(container)

    def get_dataset(self, name):
        ''' Get the dataset

        The returned particle container can be used to query
        and change the related data inside LAMMPS.

        Parameters
        ----------
        name: str
            name of CUDS container to be retrieved.

        Returns
        -------
        container :
            A proxy of the dataset named ``name`` that is stored
            internally in the Engine.

        Raises
        ------
        ValueError:
            If there is no dataset with the given name

        '''
        if name in self._data_manager:
            return self._data_manager[name]

        raise ValueError("Particle container '{}` does not exist"
                         .format(name))

    def get_dataset_names(self):
        ''' Returns the names of all the datasets
        '''
        return [name for name in self._data_manager]

    def remove_dataset(self, name):
        ''' Remove a dataset

        Parameters
        ----------
        name: str
            name of CUDS container to be deleted

        Raises
        ------
        ValueError:
            If there is no dataset with the given name

        '''
        if name in self._data_manager:
            del self._data_manager[name]
        else:
            raise ValueError("Particles '{}\\' does not exist".format(name))

    def iter_datasets(self, names=None):
        ''' Returns an iterator over a subset or all of the containers.

        Parameters
        ----------
        names : sequence of str, optional
            names of specific containers to be iterated over. If names is not
            given, then all containers will be iterated over.

        '''
        if not names:
            for name in self._data_manager:
                yield self._data_manager[name]
        else:
            for name in names:
                if name in self._data_manager:
                    yield self._data_manager[name]
                    continue
                raise ValueError("Particle container '{}\\` does not exist"
                                 .format(name))

    def run(self):
        ''' Run lammps-engine based on configuration and data

        '''
        if self._use_internal_interface:
            self._data_manager.flush()
            commands = ''
            commands += ScriptWriter.get_pair_style(
                _combine(self.SP, self.SP_extension))
            commands += ScriptWriter.get_fix(CM=_combine(self.CM,
                                                         self.CM_extension))
            commands += ScriptWriter.get_pair_coeff(_combine(
                self.SP, self.SP_extension))
            commands += ScriptWriter.get_boundary(_combine(
                self.BC, self.BC_extension), change_existing_boundary=True)
            commands += ScriptWriter.get_run(CM=_combine(
                self.CM, self.CM_extension))
            for command in commands.splitlines():
                self._lammps.command(command)
            # after running, we read any changes from lammps
            self._data_manager.read()
        else:
            with _temp_directory() as temp_dir:
                input_data_filename = os.path.join(temp_dir, 'data_in.lammps')
                output_data_filename = os.path.join(temp_dir,
                                                    'data_out.lammps')
                self._data_manager.flush(input_data_filename)

                commands = self._script_writer.get_configuration(
                    input_data_file=input_data_filename,
                    output_data_file=output_data_filename,
                    BC=_combine(self.BC, self.BC_extension),
                    CM=_combine(self.CM, self.CM_extension),
                    SP=_combine(self.SP, self.SP_extension),
                    materials=[
                        material for material in
                        self.SD.iter(item_type=CUBA.MATERIAL)])

                process = LammpsProcess(lammps_name=self._executable_name,
                                        log_directory=temp_dir)

                process.run(commands)
                self._data_manager.read(output_data_filename)

def _combine(data_container, data_container_extension):
    ''' Combine a the approved CUBA with non-approved CUBA key-values

    Parameters
    ----------
    data_container : DataContainer
        data container with CUBA attributes
    data_container_extension : dict
        data container with non-approved CUBA attributes

    Returns
    ----------
    dict
        dictionary containing the approved adn non-approved
        CUBA key-values

    '''
    result = dict(data_container_extension)
    result.update(data_container)
    return result
<commit_msg>Update docs.<commit_after>"""LAMMPS SimPhoNy Wrapper.

This module provides a wrapper for LAMMPS-md
"""
import contextlib
import os
import shutil
import tempfile

from io.lammps_fileio_data_manager import LammpsFileIoDataManager
from io.lammps_process import LammpsProcess

from common.atom_style import AtomStyle

from config.script_writer import ScriptWriter

from cuba_extension import CUBAExtension

from internal.lammps_internal_data_manager import LammpsInternalDataManager

from simphony.core import CUBA
from simphony.core.data_container import DataContainer
from simphony.cuds import CUDS
from simphony.cuds.abc_modeling_engine import ABCModelingEngine
from simphony.cuds.abc_particles import ABCParticles
from simphony.cuds.meta import api
import simphony.cuds.particles as scp

def _temp_directory():
    """Provide a temp directory.

    The name of the created temp directory is returned when context is entered
    and this directory is deleted when context is exited
    """
    temp_dir = tempfile.mkdtemp()
    yield temp_dir
    shutil.rmtree(temp_dir)

_temp_directory = contextlib.contextmanager(_temp_directory)

class LammpsWrapper(ABCModelingEngine):
    """Wrapper to LAMMPS-md."""

    def __init__(self, use_internal_interface=False, **kwargs):
        """Constructor.

        Parameters
        ----------
        engine_type : EngineType
            type of engine

        use_internal_interface : bool, optional
            If true, then the internal interface (library) is used when
            communicating with LAMMPS, if false, then file-io interface is
            used where input/output files are used to communicate with LAMMPS
        """
        self.BC = DataContainer()
        self.CM = DataContainer()
        self.SP = DataContainer()
        self.CM_extension = {}
        self.SP_extension = {}
        self.BC_extension = {}
        self.SD = kwargs.get('cuds', CUDS())

        self._use_internal_interface = use_internal_interface
        atom_style = AtomStyle.ATOMIC
        self._executable_name = 'lammps'
        self._script_writer = ScriptWriter(atom_style)

        if self._use_internal_interface:
            import lammps
            # lammps.lammps(cmdargs=["-screen", "none", "-log", "none"])
            self._lammps = lammps.lammps()
            self._data_manager = LammpsInternalDataManager(self._lammps,
                                                           self.SD,
                                                           atom_style)
        else:
            self._data_manager = LammpsFileIoDataManager(self.SD, atom_style)

        # Call the base class in order to load CUDS
        super(LammpsWrapper, self).__init__(**kwargs)

    def _count_of(self, cuds, item_type):
        """Workaround for broken CUDS counter."""
        count = 0
        for c in cuds.iter():
            if isinstance(c, item_type):
                count += 1
        return count

    def _load_cuds(self):
        '''Load CUDS data into lammps engine.'''
        cuds = self.get_cuds()
        if not cuds:
            return

        # FIXME: `count_of` is broken for non-meta classes,
        # e.g. Particles, Particle, Mesh, etc.
        # if cuds.count_of(CUBA.PARTICLES) != 1:
        if self._count_of(cuds, scp.Particles) != 1:
            raise Exception('This version of simlammps'
                            ' needs only one particle dataset, not %s' %
                            self._count_of(cuds, scp.Particles))
        if cuds.count_of(CUBA.MATERIAL) != 1:
            raise Exception('This version of simlammps needs'
                            ' only one material not %s' %
                            cuds.count_of(CUBA.MATERIAL))

        material_to_atom = {}
        number_atom_types = 0
        for mat in cuds.iter(item_type=CUBA.MATERIAL):
            if CUBA.MASS not in mat.data:
                raise Exception('Material needs mass property!')
            number_atom_types += 1
            material_to_atom[mat.uid] = number_atom_types

        for particles in cuds.iter(item_type=CUBA.PARTICLES):
            update_list = []
            for single_particle in particles.iter():
                single_particle.data[CUBA.MATERIAL_TYPE] = mat.uid
                update_list.append(single_particle)
            particles.update(update_list)

        for b in cuds.iter(item_type=CUBA.BOX):
            pass

        particle_sum = 0
        for ds in cuds.iter(item_type=CUBA.PARTICLES):
            particle_sum += len(ds)
            ds.data = mat.data
            ds.data_extension = {
                CUBAExtension.BOX_VECTORS: b.vector}
            self.add_dataset(ds)

        if particle_sum == 0:
            raise Exception('simlammps needs some particles')

        if cuds.count_of(CUBA.MOLECULAR_DYNAMICS) != 1:
            raise Exception('simlammps supports only MD')

        if cuds.count_of(CUBA.BOX) != 1:
            raise Exception('simlammps needs one box')

        for termo in cuds.iter(item_type=CUBA.THERMOSTAT):
            if isinstance(termo, api.TemperatureRescaling):
                self.CM_extension[CUBA.THERMODYNAMIC_ENSEMBLE] = 'NVE'
                continue

        if not cuds.count_of(CUBA.INTEGRATION_TIME):
            raise Exception('Only one integration time setup is accepted')

        for i in cuds.iter(item_type=CUBA.INTEGRATION_TIME):
            self.CM[CUBA.TIME_STEP] = i.step
            self.CM[CUBA.NUMBER_OF_TIME_STEPS] = int(i.final / i.step)

        if cuds.count_of(CUBA.CONDITION) != 1:
            raise Exception('Sorry only one condition is accepted, not %s' %
                            cuds.count_of(CUBA.CONDITION))

        for c in cuds.iter(item_type=CUBA.CONDITION):
            if isinstance(c, api.Periodic):
                self.BC_extension[CUBAExtension.BOX_FACES] = [
                    'periodic',
                    'periodic',
                    'periodic']
                continue
            raise Exception('Sorry, I am confused!')

        if cuds.count_of(CUBA.INTERATOMIC_POTENTIAL) != 1:
            raise Exception('Lammps needs one interatomic potention')

        for ip in cuds.iter(item_type=CUBA.INTERATOMIC_POTENTIAL):
            pass

        self.SP_extension[CUBAExtension.PAIR_POTENTIALS] =\
            'lj:\n  global_cutoff: {global_cutoff}\n'\
            '  parameters:\n  - pair: [{mat1}, {mat2}]\n'\
            '    epsilon: {energy_well_depth}\n'\
            '    sigma: {vanderwaals_radious}\n'\
            '    cutoff: {cutoff}\n'\
            .format(global_cutoff=1.12246,
                    mat1=material_to_atom[ip.material[0].uid],
                    mat2=material_to_atom[ip.material[1].uid],
                    energy_well_depth=ip.energy_well_depth,
                    vanderwaals_radious=ip.van_der_waals_radius,
                    cutoff=ip.cutoff_distance)

    def add_dataset(self, container):
        """Add a CUDS container.

        Parameters
        ----------
        container : {ABCParticles}
            The CUDS container to add to the engine.

        Raises
        ------
        TypeError:
            If the container type is not supported (i.e. ABCLattice, ABCMesh).
        ValueError:
            If there is already a dataset with the given name.

        """
        if not isinstance(container, ABCParticles):
            raise TypeError(
                'The type of the dataset container is not supported')
        if container.name in self._data_manager:
            raise ValueError("Particle container '{}' already exists"
                             .format(container.name))
        self._data_manager.new_particles(container)

    def get_dataset(self, name):
        """Get the dataset.

        The returned particle container can be used to query
        and change the related data inside LAMMPS.

        Parameters
        ----------
        name: str
            name of CUDS container to be retrieved.

        Returns
        -------
        container :
            A proxy of the dataset named ``name`` that is stored
            internally in the Engine.

        Raises
        ------
        ValueError:
            If there is no dataset with the given name

        """
        if name in self._data_manager:
            return self._data_manager[name]

        raise ValueError("Particle container '{}` does not exist"
                         .format(name))

    def get_dataset_names(self):
        """Return the names of all the datasets."""
        return [name for name in self._data_manager]

    def remove_dataset(self, name):
        """Remove a dataset.

        Parameters
        ----------
        name: str
            name of CUDS container to be deleted

        Raises
        ------
        ValueError:
            If there is no dataset with the given name

        """
        if name in self._data_manager:
            del self._data_manager[name]
        else:
            raise ValueError("Particles '{}\\' does not exist".format(name))

    def iter_datasets(self, names=None):
        """Return an iterator over a subset or all of the containers.

        Parameters
        ----------
        names : sequence of str, optional
            names of specific containers to be iterated over. If names is not
            given, then all containers will be iterated over.

        """
        if not names:
            for name in self._data_manager:
                yield self._data_manager[name]
        else:
            for name in names:
                if name in self._data_manager:
                    yield self._data_manager[name]
                    continue
                raise ValueError("Particle container '{}\\` does not exist"
                                 .format(name))

    def run(self):
        ''' Run lammps-engine based on configuration and data

        '''
        if self._use_internal_interface:
            self._data_manager.flush()
            commands = ''
            commands += ScriptWriter.get_pair_style(
                _combine(self.SP, self.SP_extension))
            commands += ScriptWriter.get_fix(CM=_combine(self.CM,
                                                         self.CM_extension))
            commands += ScriptWriter.get_pair_coeff(_combine(
                self.SP, self.SP_extension))
            commands += ScriptWriter.get_boundary(_combine(
                self.BC, self.BC_extension), change_existing_boundary=True)
            commands += ScriptWriter.get_run(CM=_combine(
                self.CM, self.CM_extension))
            for command in commands.splitlines():
                self._lammps.command(command)
            # after running, we read any changes from lammps
            self._data_manager.read()
        else:
            with _temp_directory() as temp_dir:
                input_data_filename = os.path.join(temp_dir, 'data_in.lammps')
                output_data_filename = os.path.join(temp_dir,
                                                    'data_out.lammps')
                self._data_manager.flush(input_data_filename)

                commands = self._script_writer.get_configuration(
                    input_data_file=input_data_filename,
                    output_data_file=output_data_filename,
                    BC=_combine(self.BC, self.BC_extension),
                    CM=_combine(self.CM, self.CM_extension),
                    SP=_combine(self.SP, self.SP_extension),
                    materials=[
                        material for material in
                        self.SD.iter(item_type=CUBA.MATERIAL)])

                process = LammpsProcess(lammps_name=self._executable_name,
                                        log_directory=temp_dir)

                process.run(commands)
                self._data_manager.read(output_data_filename)

def _combine(data_container, data_container_extension):
    ''' Combine a the approved CUBA with non-approved CUBA key-values

    Parameters
    ----------
    data_container : DataContainer
        data container with CUBA attributes
    data_container_extension : dict
        data container with non-approved CUBA attributes

    Returns
    ----------
    dict
        dictionary containing the approved adn non-approved
        CUBA key-values

    """
    result = dict(data_container_extension)
    result.update(data_container)
    return result
<|endoftext|>


<commit_before>import base64
from datetime import datetime
import os.path
import magic
import subprocess
import urllib
from restish import http, resource

from formish.filehandler import TempFileHandler

# Binaries
IDENTIFY = '/usr/bin/identify'
CONVERT = '/usr/bin/convert'

class FileAccessor(object):
    """
    A skeleton class that should be implemented so that the files resource can
    build caches, etc
    """

    def get_mimetype(self, id):
        """
        Get the mime type of the file with this id
        """

    def get_file(self, id):
        """
        Get the file object for this id
        """

class FileResource(resource.Resource):
    """
    Resource for serving files from the database or temporary upload area.
    """

    def __init__(self, fileaccessor=FileAccessor(),
                 filehandler=TempFileHandler(), segments=None):
        resource.Resource.__init__(self)
        self.fileaccessor = fileaccessor
        self.filehandler = filehandler
        if segments is None:
            return
        # If it's a temp file, just return it...
        # XXX This it wrong... it should still cache and resize
        filename = '/'.join(segments)
        splits = filename.split('.')
        filename, suffix = ''.join(splits[:-1]), splits[-1]
        self.tempfile = filehandler.get_path_for_file(urllib.unquote_plus(filename))
        if os.path.exists(self.tempfile):
            return
        # Otherwise it must be a resource so check if it needs cacheing
        self.tempfile =  'cache/%s'%(filename.replace('/','-'))
        if segments[0] != '':
            if os.path.exists(self.tempfile):
                mtime = self.fileaccessor.get_mtime(filename)
                cache_mtime = datetime.utcfromtimestamp(os.path.getmtime(self.tempfile))
                if mtime is None or mtime > cache_mtime:
                    rebuild_cache = True
                else:
                    rebuild_cache = False
            else:
                rebuild_cache = True
            if rebuild_cache:
                data = self.fileaccessor.get_file(filename)
                cache_fp = file(self.tempfile,'w')
                cache_fp.write(data)
                cache_fp.close()

    def resource_child(self, request, segments):
        return FileResource(self.fileaccessor, self.filehandler, segments), ()

    def make_response(self, filename, request):
        width, height = getSizeFromDict(request.GET)
        if not (width is None and height is None):
            mtime = datetime.utcfromtimestamp(os.path.getmtime(filename))
            filename = '%s-%sx%s'%(filename,width,height)
            if os.path.isfile(filename):
                cache_mtime = datetime.utcfromtimestamp(os.path.getmtime(filename))
                if mtime > cache_mtime:
                    rebuild_cache = True
                else:
                    rebuild_cache = False
            else:
                rebuild_cache = True

            if not os.path.exists(filename) or rebuild_cache:
                resizeImage(self.tempfile, filename, width, height)

        return http.ok([('content-type', get_mimetype(filename) )],open(filename, 'rb').read())

    @resource.GET()
    def get_file(self, request):
        return self.make_response(self.tempfile, request)

def resizeImage(srcPath, targetPath, width, height, quality=70):
    # this is an example identify
    # '/home/tim/Desktop/tim.jpg JPEG 48x48 48x48+0+0 DirectClass 8-bit 920b \n'
    stdout = subprocess.Popen([IDENTIFY,srcPath],stdout=subprocess.PIPE).communicate()[0]
    iwidth, iheight = [ int(s) for s in stdout.split(' ')[2].split('x')]
    if width is None:
        width = int(iwidth*(float(height)/float(iheight)))
    if height is None:
        height = int(iheight*(float(width)/float(iwidth)))
    subprocess.call([CONVERT, '-thumbnail', '%sx%s'%(width, height), '-quality', str(quality), srcPath, targetPath])

def getSizeFromDict(d):
    size = d.get('size',None)
    if size is not None:
        width = int(size.split('x')[0])
        height = int(size.split('x')[1])
    else:
        width = d.get('width',None)
        if width is not None:
            width = int(width)
        height = d.get('height',None)
        if height is not None:
            height = int(height)
    return (width, height)

def get_mimetype(filename):
    mimetype = magic.from_file(filename,mime=True)
    return mimetype or 'application/octet-stream'

<commit_msg>fixed tempfile with no suffix<commit_after>import base64
from datetime import datetime
import os.path
import magic
import subprocess
import urllib
from restish import http, resource

from formish.filehandler import TempFileHandler

# Binaries
IDENTIFY = '/usr/bin/identify'
CONVERT = '/usr/bin/convert'

class FileAccessor(object):
    """
    A skeleton class that should be implemented so that the files resource can
    build caches, etc
    """

    def get_mimetype(self, id):
        """
        Get the mime type of the file with this id
        """

    def get_file(self, id):
        """
        Get the file object for this id
        """

class FileResource(resource.Resource):
    """
    Resource for serving files from the database or temporary upload area.
    """

    def __init__(self, fileaccessor=FileAccessor(),
                 filehandler=TempFileHandler(), segments=None):
        resource.Resource.__init__(self)
        self.fileaccessor = fileaccessor
        self.filehandler = filehandler
        if segments is None:
            return
        # If it's a temp file, just return it...
        # XXX This it wrong... it should still cache and resize
        filepath = '/'.join(segments)
        splits = filepath.split('.')
        filename, suffix = ''.join(splits[:-1]), splits[-1]
        self.tempfile = filehandler.get_path_for_file(urllib.unquote_plus(filepath))
        if os.path.exists(self.tempfile):
            return
        # Otherwise it must be a resource so check if it needs cacheing
        self.tempfile =  'cache/%s'%(filename.replace('/','-'))
        if segments[0] != '':
            if os.path.exists(self.tempfile):
                mtime = self.fileaccessor.get_mtime(filename)
                cache_mtime = datetime.utcfromtimestamp(os.path.getmtime(self.tempfile))
                if mtime is None or mtime > cache_mtime:
                    rebuild_cache = True
                else:
                    rebuild_cache = False
            else:
                rebuild_cache = True
            if rebuild_cache:
                data = self.fileaccessor.get_file(filename)
                cache_fp = file(self.tempfile,'w')
                cache_fp.write(data)
                cache_fp.close()

    def resource_child(self, request, segments):
        return FileResource(self.fileaccessor, self.filehandler, segments), ()

    def make_response(self, filename, request):
        width, height = getSizeFromDict(request.GET)
        if not (width is None and height is None):
            mtime = datetime.utcfromtimestamp(os.path.getmtime(filename))
            filename = '%s-%sx%s'%(filename,width,height)
            if os.path.isfile(filename):
                cache_mtime = datetime.utcfromtimestamp(os.path.getmtime(filename))
                if mtime > cache_mtime:
                    rebuild_cache = True
                else:
                    rebuild_cache = False
            else:
                rebuild_cache = True

            if not os.path.exists(filename) or rebuild_cache:
                resizeImage(self.tempfile, filename, width, height)

        return http.ok([('content-type', get_mimetype(filename) )],open(filename, 'rb').read())

    @resource.GET()
    def get_file(self, request):
        return self.make_response(self.tempfile, request)

def resizeImage(srcPath, targetPath, width, height, quality=70):
    # this is an example identify
    # '/home/tim/Desktop/tim.jpg JPEG 48x48 48x48+0+0 DirectClass 8-bit 920b \n'
    stdout = subprocess.Popen([IDENTIFY,srcPath],stdout=subprocess.PIPE).communicate()[0]
    iwidth, iheight = [ int(s) for s in stdout.split(' ')[2].split('x')]
    if width is None:
        width = int(iwidth*(float(height)/float(iheight)))
    if height is None:
        height = int(iheight*(float(width)/float(iwidth)))
    subprocess.call([CONVERT, '-thumbnail', '%sx%s'%(width, height), '-quality', str(quality), srcPath, targetPath])

def getSizeFromDict(d):
    size = d.get('size',None)
    if size is not None:
        width = int(size.split('x')[0])
        height = int(size.split('x')[1])
    else:
        width = d.get('width',None)
        if width is not None:
            width = int(width)
        height = d.get('height',None)
        if height is not None:
            height = int(height)
    return (width, height)

def get_mimetype(filename):
    mimetype = magic.from_file(filename,mime=True)
    return mimetype or 'application/octet-stream'

<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import io
import time
import errno
import operator
import subprocess
import functools
from functools import wraps
from tempfile import mkstemp
import logging
logger = logging.getLogger(__name__)

logdir = 'tldp-document-build-logs'

def logtimings(logmethod):
    def anon(f):
        @wraps(f)
        def timing(*args, **kwargs):
            s = time.time()
            result = f(*args, **kwargs)
            e = time.time()
            logmethod('running %s(%r, %r) took %.3f s',
                      f.__name__, args, kwargs, e - s)
            return result
        return timing
    return anon

def firstfoundfile(locations):
    '''return the first existing file from a list of filenames (or None)'''
    for option in locations:
        if isreadablefile(option):
            return option
    return None

def arg_isloglevel(l):
    try:
        level = int(l)
        return level
    except ValueError:
        pass
    level = getattr(logging, l.upper(), None)
    if not level:
        level = logging.ERROR
    return level

def arg_isreadablefile(f):
    if isreadablefile(f):
        return f
    return None

def arg_isdirectory(d):
    if os.path.isdir(d):
        return d
    return None

def arg_isexecutable(f):
    if isexecutable(f):
        return f
    return None

def stem_and_ext(name):
    '''return (stem, ext) for any relative or absolute filename'''
    return os.path.splitext(os.path.basename(os.path.normpath(name)))

def conditionallogging(result, prefix, fname):
    if logger.isEnabledFor(logging.DEBUG):
        logfilecontents(logger.info, prefix, fname)  # -- always
    elif logger.isEnabledFor(logging.INFO):
        if result != 0:
            logfilecontents(logger.info, prefix, fname)  # -- error

def logfilecontents(logmethod, prefix, fname):
    '''log all lines of a file with a prefix '''
    with open(fname) as f:
        for line in f:
            logmethod("%s: %s", prefix, line.rstrip())

def execute(cmd, stdin=None, stdout=None, stderr=None,
            logdir=None, env=os.environ):
    '''(yet another) wrapper around subprocess.Popen()

    The processing tools for handling DocBook SGML, DocBook XML and Linuxdoc
    all use different conventions for writing outputs.  Some write into the
    working directory.  Others write to STDOUT.  Others accept the output file
    as a required option.

    To allow for automation and flexibility, this wrapper function does what
    most other synchronous subprocess.Popen() wrappers does, but it adds a
    feature to record the STDOUT and STDERR of the executable.  This is
    helpful when trying to diagnose build failures of individual documents.

    Required:

      - cmd: (list form only; the paranoid prefer shell=False)
        this must include the whole command-line
      - logdir: an existing directory in which temporary log files
        will be created

    Optional:

      - stdin: if not supplied, STDIN (FD 0) will be left as is
      - stdout: if not supplied, STDOUT (FD 1) will be connected
        to a named file in the logdir (and left for later inspection)
      - stderr: if not supplied, STDERR (FD 2) will be connected
        to a named file in the logdir (and left for later inspection)
      - env: if not supplied, just use current environment

    Returns: the numeric exit code of the process

    Side effects:

      * will probably create temporary files in logdir
      * function calls wait(); process execution will intentionally block
        until the child process terminates

    Possible exceptions:

      * if the first element of list cmd does not contain an executable,
        this function will raise an AssertionError
      * if logdir is not a directory, this function will raise ValueError or
        IOError
      * and, of course, any exceptions passed up from calling subprocess.Popen

    '''
    prefix = os.path.basename(cmd[0]) + '.' + str(os.getpid()) + '-'

    assert isexecutable(cmd[0])

    if logdir is None:
        raise ValueError("logdir must be a directory, cannot be None.")

    if not os.path.isdir(logdir):
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), logdir)

    # -- not remapping STDIN, because that doesn't make sense here
    mytfile = functools.partial(mkstemp, prefix=prefix, dir=logdir)
    if stdout is None:
        stdout, stdoutname = mytfile(suffix='.stdout')
    else:
        stdoutname = None

    if stderr is None:
        stderr, stderrname = mytfile(suffix='.stderr')
    else:
        stderrname = None

    logger.debug("About to execute: %r", cmd)
    proc = subprocess.Popen(cmd, shell=False, close_fds=True,
                            stdin=stdin, stdout=stdout, stderr=stderr,
                            env=env, preexec_fn=os.setsid)
    result = proc.wait()
    if result != 0:
        logger.error("Non-zero exit (%s) for process: %r", result, cmd)
        logger.error("Find STDOUT/STDERR in %s/%s*", logdir, prefix)
    if isinstance(stdout, int) and stdoutname:
        os.close(stdout)
        conditionallogging(result, 'STDOUT', stdoutname)
    if isinstance(stderr, int) and stderrname:
        os.close(stderr)
        conditionallogging(result, 'STDERR', stderrname)
    return result

def isexecutable(f):
    '''True if argument is executable'''
    return os.path.isfile(f) and os.access(f, os.X_OK)

def isreadablefile(f):
    '''True if argument is readable file'''
    return os.path.isfile(f) and os.access(f, os.R_OK)

def which(program):
    '''return None or the full path to an executable (respecting $PATH)
http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    '''
    fpath, fname = os.path.split(program)
    if fpath and isexecutable(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            sut = os.path.join(path, program)
            if isexecutable(sut):
                return sut
    return None

def makefh(thing):
    '''return a file object; given an existing filename name or file object'''
    if isinstance(thing, io.IOBase):
        f = thing
    elif isinstance(thing, str) and os.path.isfile(thing):
        f = open(thing)
    else:
        raise TypeError("Cannot make file from %s of %r" %
                        (type(thing), thing,))
    return f

def statfile(name):
    '''return posix.stat_result (or None) for a single file name'''
    try:
        st = os.lstat(name)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e
        st = None
    return st

def statfiles(name, relative=None):
    '''return a dict() with keys being filenames and posix.stat_result values

    Required:

      name: the name should be an existing file, but accessing filesystems
            can be a racy proposition, so if the name is ENOENT, returns an
            empty dict()
            if name is a directory, os.walk() over the entire subtree and
            record and return all stat() results

    Optional:

      relative: if the filenames in the keys should be relative some other
                directory, then supply that path here (see examples)

    Bugs:
      Dealing with filesystems is always potentially a racy affair.  They go
      out for lunch sometimes.  They don't call.  They don't write.  But, at
      least we can try to rely on them as best we can--mostly, by just
      excluding any files (in the output dict()) which did not return a valid
      posix.stat_result.

    Examples:

    >>> statfiles('./docs/x509').keys()
    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./').keys()
    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
    ['index.rst', 'tutorial.rst', 'reference.rst']
    '''
    statinfo = dict()
    if not os.path.exists(name):
        return statinfo
    if not os.path.isdir(name):
        if relative:
            relpath = os.path.relpath(name, start=relative)
        else:
            relpath = name
        statinfo[relpath] = statfile(name)
        if statinfo[relpath] is None:
            del statinfo[relpath]
    else:
        for root, dirs, files in os.walk(name):
            inodes = list()
            inodes.extend(dirs)
            inodes.extend(files)
            for x in inodes:
                foundpath = os.path.join(root, x)
                if relative:
                    relpath = os.path.relpath(foundpath, start=relative)
                else:
                    relpath = foundpath
                statinfo[relpath] = statfile(foundpath)
                if statinfo[relpath] is None:
                    del statinfo[relpath]
    return statinfo

def att_statinfo(statinfo, attr='st_mtime', func=max):
    if statinfo:
        return func([getattr(v, attr) for v in statinfo.values()])
    else:
        return 0

max_size = functools.partial(att_statinfo, attr='st_size', func=max)
min_size = functools.partial(att_statinfo, attr='st_size', func=min)

max_mtime = functools.partial(att_statinfo, attr='st_mtime', func=max)
min_mtime = functools.partial(att_statinfo, attr='st_mtime', func=min)

max_ctime = functools.partial(att_statinfo, attr='st_ctime', func=max)
min_ctime = functools.partial(att_statinfo, attr='st_ctime', func=min)

max_atime = functools.partial(att_statinfo, attr='st_atime', func=max)
min_atime = functools.partial(att_statinfo, attr='st_atime', func=min)

def sieve(operand, statinfo, attr='st_mtime', func=operator.gt):
    result = set()
    for fname, stbuf in statinfo.items():
        if func(getattr(stbuf, attr), operand):
            result.add(fname)
    return result

mtime_gt = functools.partial(sieve, attr='st_mtime', func=operator.gt)
mtime_lt = functools.partial(sieve, attr='st_mtime', func=operator.lt)

size_gt = functools.partial(sieve, attr='st_size', func=operator.gt)
size_lt = functools.partial(sieve, attr='st_size', func=operator.lt)

#
# -- end of file
<commit_msg>adding a function to swap directories<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import io
import time
import errno
import operator
import subprocess
import functools
from functools import wraps
from tempfile import mkstemp, mkdtemp
import logging
logger = logging.getLogger(__name__)

opa = os.path.abspath
opb = os.path.basename
opd = os.path.dirname
opj = os.path.join

logdir = 'tldp-document-build-logs'

def logtimings(logmethod):
    def anon(f):
        @wraps(f)
        def timing(*args, **kwargs):
            s = time.time()
            result = f(*args, **kwargs)
            e = time.time()
            logmethod('running %s(%r, %r) took %.3f s',
                      f.__name__, args, kwargs, e - s)
            return result
        return timing
    return anon

def firstfoundfile(locations):
    '''return the first existing file from a list of filenames (or None)'''
    for option in locations:
        if isreadablefile(option):
            return option
    return None

def arg_isloglevel(l):
    try:
        level = int(l)
        return level
    except ValueError:
        pass
    level = getattr(logging, l.upper(), None)
    if not level:
        level = logging.ERROR
    return level

def arg_isreadablefile(f):
    if isreadablefile(f):
        return f
    return None

def arg_isdirectory(d):
    if os.path.isdir(d):
        return d
    return None

def arg_isexecutable(f):
    if isexecutable(f):
        return f
    return None

def stem_and_ext(name):
    '''return (stem, ext) for any relative or absolute filename'''
    return os.path.splitext(os.path.basename(os.path.normpath(name)))

def swapdirs(a, b):
    '''use os.rename() to make "a" become "b"'''
    if not os.path.isdir(a):
        raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), a)
    tname = None
    if os.path.isdir(b):
        tdir = mkdtemp(prefix='swapdirs-', dir=opd(opa(a)))
        logger.debug("Created tempdir %s.", tdir)
        tname = opj(tdir, opb(b))
        logger.debug("About to rename %s to %s.", b, tname)
        os.rename(b, tname)
    logger.debug("About to rename %s to %s.", a, b)
    os.rename(a, b)
    if tname:
        logger.debug("About to rename %s to %s.", tname, a)
        os.rename(tname, a)
        logger.debug("About to remove %s.", tdir)
        os.rmdir(tdir)

def conditionallogging(result, prefix, fname):
    if logger.isEnabledFor(logging.DEBUG):
        logfilecontents(logger.info, prefix, fname)  # -- always
    elif logger.isEnabledFor(logging.INFO):
        if result != 0:
            logfilecontents(logger.info, prefix, fname)  # -- error

def logfilecontents(logmethod, prefix, fname):
    '''log all lines of a file with a prefix '''
    with open(fname) as f:
        for line in f:
            logmethod("%s: %s", prefix, line.rstrip())

def execute(cmd, stdin=None, stdout=None, stderr=None,
            logdir=None, env=os.environ):
    '''(yet another) wrapper around subprocess.Popen()

    The processing tools for handling DocBook SGML, DocBook XML and Linuxdoc
    all use different conventions for writing outputs.  Some write into the
    working directory.  Others write to STDOUT.  Others accept the output file
    as a required option.

    To allow for automation and flexibility, this wrapper function does what
    most other synchronous subprocess.Popen() wrappers does, but it adds a
    feature to record the STDOUT and STDERR of the executable.  This is
    helpful when trying to diagnose build failures of individual documents.

    Required:

      - cmd: (list form only; the paranoid prefer shell=False)
        this must include the whole command-line
      - logdir: an existing directory in which temporary log files
        will be created

    Optional:

      - stdin: if not supplied, STDIN (FD 0) will be left as is
      - stdout: if not supplied, STDOUT (FD 1) will be connected
        to a named file in the logdir (and left for later inspection)
      - stderr: if not supplied, STDERR (FD 2) will be connected
        to a named file in the logdir (and left for later inspection)
      - env: if not supplied, just use current environment

    Returns: the numeric exit code of the process

    Side effects:

      * will probably create temporary files in logdir
      * function calls wait(); process execution will intentionally block
        until the child process terminates

    Possible exceptions:

      * if the first element of list cmd does not contain an executable,
        this function will raise an AssertionError
      * if logdir is not a directory, this function will raise ValueError or
        IOError
      * and, of course, any exceptions passed up from calling subprocess.Popen

    '''
    prefix = os.path.basename(cmd[0]) + '.' + str(os.getpid()) + '-'

    assert isexecutable(cmd[0])

    if logdir is None:
        raise ValueError("logdir must be a directory, cannot be None.")

    if not os.path.isdir(logdir):
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), logdir)

    # -- not remapping STDIN, because that doesn't make sense here
    mytfile = functools.partial(mkstemp, prefix=prefix, dir=logdir)
    if stdout is None:
        stdout, stdoutname = mytfile(suffix='.stdout')
    else:
        stdoutname = None

    if stderr is None:
        stderr, stderrname = mytfile(suffix='.stderr')
    else:
        stderrname = None

    logger.debug("About to execute: %r", cmd)
    proc = subprocess.Popen(cmd, shell=False, close_fds=True,
                            stdin=stdin, stdout=stdout, stderr=stderr,
                            env=env, preexec_fn=os.setsid)
    result = proc.wait()
    if result != 0:
        logger.error("Non-zero exit (%s) for process: %r", result, cmd)
        logger.error("Find STDOUT/STDERR in %s/%s*", logdir, prefix)
    if isinstance(stdout, int) and stdoutname:
        os.close(stdout)
        conditionallogging(result, 'STDOUT', stdoutname)
    if isinstance(stderr, int) and stderrname:
        os.close(stderr)
        conditionallogging(result, 'STDERR', stderrname)
    return result

def isexecutable(f):
    '''True if argument is executable'''
    return os.path.isfile(f) and os.access(f, os.X_OK)

def isreadablefile(f):
    '''True if argument is readable file'''
    return os.path.isfile(f) and os.access(f, os.R_OK)

def which(program):
    '''return None or the full path to an executable (respecting $PATH)
http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    '''
    fpath, fname = os.path.split(program)
    if fpath and isexecutable(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            sut = os.path.join(path, program)
            if isexecutable(sut):
                return sut
    return None

def makefh(thing):
    '''return a file object; given an existing filename name or file object'''
    if isinstance(thing, io.IOBase):
        f = thing
    elif isinstance(thing, str) and os.path.isfile(thing):
        f = open(thing)
    else:
        raise TypeError("Cannot make file from %s of %r" %
                        (type(thing), thing,))
    return f

def statfile(name):
    '''return posix.stat_result (or None) for a single file name'''
    try:
        st = os.lstat(name)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e
        st = None
    return st

def statfiles(name, relative=None):
    '''return a dict() with keys being filenames and posix.stat_result values

    Required:

      name: the name should be an existing file, but accessing filesystems
            can be a racy proposition, so if the name is ENOENT, returns an
            empty dict()
            if name is a directory, os.walk() over the entire subtree and
            record and return all stat() results

    Optional:

      relative: if the filenames in the keys should be relative some other
                directory, then supply that path here (see examples)

    Bugs:
      Dealing with filesystems is always potentially a racy affair.  They go
      out for lunch sometimes.  They don't call.  They don't write.  But, at
      least we can try to rely on them as best we can--mostly, by just
      excluding any files (in the output dict()) which did not return a valid
      posix.stat_result.

    Examples:

    >>> statfiles('./docs/x509').keys()
    ['./docs/x509/tutorial.rst', './docs/x509/reference.rst', './docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./').keys()
    ['docs/x509/reference.rst', 'docs/x509/tutorial.rst', 'docs/x509/index.rst']
    >>> statfiles('./docs/x509', relative='./docs/x509/').keys()
    ['index.rst', 'tutorial.rst', 'reference.rst']
    '''
    statinfo = dict()
    if not os.path.exists(name):
        return statinfo
    if not os.path.isdir(name):
        if relative:
            relpath = os.path.relpath(name, start=relative)
        else:
            relpath = name
        statinfo[relpath] = statfile(name)
        if statinfo[relpath] is None:
            del statinfo[relpath]
    else:
        for root, dirs, files in os.walk(name):
            inodes = list()
            inodes.extend(dirs)
            inodes.extend(files)
            for x in inodes:
                foundpath = os.path.join(root, x)
                if relative:
                    relpath = os.path.relpath(foundpath, start=relative)
                else:
                    relpath = foundpath
                statinfo[relpath] = statfile(foundpath)
                if statinfo[relpath] is None:
                    del statinfo[relpath]
    return statinfo

def att_statinfo(statinfo, attr='st_mtime', func=max):
    if statinfo:
        return func([getattr(v, attr) for v in statinfo.values()])
    else:
        return 0

max_size = functools.partial(att_statinfo, attr='st_size', func=max)
min_size = functools.partial(att_statinfo, attr='st_size', func=min)

max_mtime = functools.partial(att_statinfo, attr='st_mtime', func=max)
min_mtime = functools.partial(att_statinfo, attr='st_mtime', func=min)

max_ctime = functools.partial(att_statinfo, attr='st_ctime', func=max)
min_ctime = functools.partial(att_statinfo, attr='st_ctime', func=min)

max_atime = functools.partial(att_statinfo, attr='st_atime', func=max)
min_atime = functools.partial(att_statinfo, attr='st_atime', func=min)

def sieve(operand, statinfo, attr='st_mtime', func=operator.gt):
    result = set()
    for fname, stbuf in statinfo.items():
        if func(getattr(stbuf, attr), operand):
            result.add(fname)
    return result

mtime_gt = functools.partial(sieve, attr='st_mtime', func=operator.gt)
mtime_lt = functools.partial(sieve, attr='st_mtime', func=operator.lt)

size_gt = functools.partial(sieve, attr='st_size', func=operator.gt)
size_lt = functools.partial(sieve, attr='st_size', func=operator.lt)

#
# -- end of file
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2001 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Script that generates the build.ninja for ninja itself.

Projects that use ninja themselves should either write a similar script
or use a meta-build system that supports Ninja output."""

from optparse import OptionParser
import os
import sys
sys.path.insert(0, 'misc')

import ninja_syntax

parser = OptionParser()
platforms = ['linux', 'freebsd', 'mingw']
profilers = ['gmon', 'pprof']
parser.add_option('--platform',
                  help='target platform (' + '/'.join(platforms) + ')',
                  choices=platforms)
parser.add_option('--host',
                  help='host platform (' + '/'.join(platforms) + ')',
                  choices=platforms)
parser.add_option('--debug', action='store_true',
                  help='enable debugging flags',)
parser.add_option('--profile', metavar='TYPE',
                  choices=profilers,
                  help='enable profiling (' + '/'.join(profilers) + ')',)
parser.add_option('--with-gtest', metavar='PATH',
                  help='use gtest unpacked in directory PATH')
(options, args) = parser.parse_args()

platform = options.platform
if platform is None:
    platform = sys.platform
    if platform.startswith('linux'):
        platform = 'linux'
    elif platform.startswith('freebsd'):
        platform = 'freebsd'
    elif platform.startswith('mingw') or platform.startswith('win'):
        platform = 'mingw'
host = options.host or platform

BUILD_FILENAME = 'build.ninja'
buildfile = open(BUILD_FILENAME, 'w')
n = ninja_syntax.Writer(buildfile)
n.comment('This file is used to build ninja itself.')
n.comment('It is generated by ' + os.path.basename(__file__) + '.')
n.newline()

n.comment('The arguments passed to configure.py, for rerunning it.')
n.variable('configure_args', ' '.join(sys.argv[1:]))
n.newline()

def src(filename):
    return os.path.join('src', filename)
def built(filename):
    return os.path.join('$builddir', filename)
def doc(filename):
    return os.path.join('doc', filename)
def cxx(name, **kwargs):
    return n.build(built(name + '.o'), 'cxx', src(name + '.cc'), **kwargs)
def binary(name):
    if platform == 'mingw':
        return name + '.exe'
    return name

n.variable('builddir', 'build')
n.variable('cxx', os.environ.get('CXX', 'g++'))
n.variable('ar', os.environ.get('AR', 'ar'))

cflags = ['-g', '-Wall', '-Wextra',
          '-Wno-deprecated',
          '-Wno-unused-parameter',
          '-fno-exceptions',
          '-fvisibility=hidden', '-pipe']
if not options.debug:
    cflags += ['-O2', '-DNDEBUG']
ldflags = ['-L$builddir']
libs = []

if platform == 'mingw':
    cflags.remove('-fvisibility=hidden');
    ldflags.append('-static')
else:
    if options.profile == 'gmon':
        cflags.append('-pg')
        ldflags.append('-pg')
    elif options.profile == 'pprof':
        libs.append('-lprofiler')

if 'CFLAGS' in os.environ:
    cflags.append(os.environ['CFLAGS'])
n.variable('cflags', ' '.join(cflags))
if 'LDFLAGS' in os.environ:
    ldflags.append(os.environ['LDFLAGS'])
n.variable('ldflags', ' '.join(ldflags))
n.newline()

n.rule('cxx',
       command='$cxx -MMD -MF $out.d $cflags -c $in -o $out',
       depfile='$out.d',
       description='CXX $out')
n.newline()

if host != 'mingw':
    n.rule('ar',
           command='rm -f $out && $ar crs $out $in',
           description='AR $out')
else:
    n.rule('ar',
           command='cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out',
           description='AR $out')
n.newline()

n.rule('link',
       command='$cxx $ldflags -o $out $in $libs',
       description='LINK $out')
n.newline()

objs = []

if platform != 'mingw':
    n.comment('browse_py.h is used to inline browse.py.')
    n.rule('inline',
           command='src/inline.sh $varname < $in > $out',
           description='INLINE $out')
    n.build(built('browse_py.h'), 'inline', src('browse.py'),
            implicit='src/inline.sh',
            variables=[('varname', 'kBrowsePy')])
    n.newline()

    objs += cxx('browse', order_only=built('browse_py.h'))
    n.newline()

n.comment('the depfile parser is generated using re2c.')
n.rule('re2c',
       command='re2c -b -i --no-generation-date -o $out $in',
       description='RE2C $out')
# Generate the .cc file in the source directory so we can check it in.
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
n.newline()

n.comment('Core source files all build into ninja library.')
for name in ['build', 'build_log', 'clean', 'edit_distance', 'eval_env',
             'graph', 'graphviz', 'parsers', 'util', 'depfile_parser',
             'disk_interface', 'state']:
    objs += cxx(name)
if platform == 'mingw':
    objs += cxx('subprocess-win32')
else:
    objs += cxx('subprocess')
ninja_lib = n.build(built('libninja.a'), 'ar', objs)
n.newline()

libs.append('-lninja')

n.comment('Main executable is library plus main() function.')
objs = cxx('ninja')
ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib,
                variables=[('libs', libs)])
n.newline()

n.comment('Tests all build into ninja_test executable.')

variables = []
test_cflags = None
test_ldflags = None
test_libs = libs
objs = []
if options.with_gtest:
    path = options.with_gtest

    gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include'))
    objs += n.build(built('gtest-all.o'), 'cxx',
                    os.path.join(path, 'src/gtest-all.cc'),
                    variables=[('cflags', gtest_all_incs)])
    objs += n.build(built('gtest_main.o'), 'cxx',
                    os.path.join(path, 'src/gtest_main.cc'),
                    variables=[('cflags', gtest_all_incs)])

    test_cflags = cflags + ['-I%s' % os.path.join(path, 'include')]
else:
    test_libs.extend(['-lgtest_main', '-lgtest'])

for name in ['build_log_test',
             'build_test',
             'clean_test',
             'depfile_parser_test',
             'disk_interface_test',
             'edit_distance_test',
             'eval_env_test',
             'graph_test',
             'parsers_test',
             'state_test',
             'subprocess_test',
             'test',
             'util_test']:
    objs += cxx(name, variables=[('cflags', test_cflags)])

if platform != 'mingw':
    test_libs.append('-lpthread')
n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib,
        variables=[('ldflags', test_ldflags),
                   ('libs', test_libs)])
n.newline()

n.comment('Perftest executable.')
objs = cxx('parser_perftest')
n.build(binary('parser_perftest'), 'link', objs, implicit=ninja_lib,
        variables=[('libs', '-L$builddir -lninja')])
n.newline()

n.comment('Generate a graph using the "graph" tool.')
n.rule('gendot',
       command='./ninja -t graph > $out')
n.rule('gengraph',
       command='dot -Tpng $in > $out')
dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja'])
n.build('graph.png', 'gengraph', dot)
n.newline()

n.comment('Generate the manual using asciidoc.')
n.rule('asciidoc',
       command='asciidoc -a toc -o $out $in',
       description='ASCIIDOC $in')
manual = n.build(doc('manual.html'), 'asciidoc', doc('manual.asciidoc'))
n.build('manual', 'phony',
        order_only=manual)
n.newline()

n.comment('Generate Doxygen.')
n.rule('doxygen',
       command='doxygen $in',
       description='DOXYGEN $in')
n.variable('doxygen_mainpage_generator',
           src('gen_doxygen_mainpage.sh'))
n.rule('doxygen_mainpage',
       command='$doxygen_mainpage_generator $in > $out',
       description='DOXYGEN_MAINPAGE $out')
mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage',
                   ['README', 'HACKING', 'COPYING'],
                   implicit=['$doxygen_mainpage_generator'])
n.build('doxygen', 'doxygen', doc('doxygen.config'),
        implicit=mainpage)
n.newline()

if host != 'mingw':
    n.comment('Regenerate build files if build script changes.')
    n.rule('configure',
           command='./configure.py $configure_args',
           generator=True)
    n.build('build.ninja', 'configure',
            implicit=['configure.py', 'misc/ninja_syntax.py'])
    n.newline()

n.comment('Build only the main binary by default.')
n.default(ninja)

print 'wrote %s.' % BUILD_FILENAME
<commit_msg>Put the list of core source files in alphabetical order.<commit_after>#!/usr/bin/env python
#
# Copyright 2001 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Script that generates the build.ninja for ninja itself.

Projects that use ninja themselves should either write a similar script
or use a meta-build system that supports Ninja output."""

from optparse import OptionParser
import os
import sys
sys.path.insert(0, 'misc')

import ninja_syntax

parser = OptionParser()
platforms = ['linux', 'freebsd', 'mingw']
profilers = ['gmon', 'pprof']
parser.add_option('--platform',
                  help='target platform (' + '/'.join(platforms) + ')',
                  choices=platforms)
parser.add_option('--host',
                  help='host platform (' + '/'.join(platforms) + ')',
                  choices=platforms)
parser.add_option('--debug', action='store_true',
                  help='enable debugging flags',)
parser.add_option('--profile', metavar='TYPE',
                  choices=profilers,
                  help='enable profiling (' + '/'.join(profilers) + ')',)
parser.add_option('--with-gtest', metavar='PATH',
                  help='use gtest unpacked in directory PATH')
(options, args) = parser.parse_args()

platform = options.platform
if platform is None:
    platform = sys.platform
    if platform.startswith('linux'):
        platform = 'linux'
    elif platform.startswith('freebsd'):
        platform = 'freebsd'
    elif platform.startswith('mingw') or platform.startswith('win'):
        platform = 'mingw'
host = options.host or platform

BUILD_FILENAME = 'build.ninja'
buildfile = open(BUILD_FILENAME, 'w')
n = ninja_syntax.Writer(buildfile)
n.comment('This file is used to build ninja itself.')
n.comment('It is generated by ' + os.path.basename(__file__) + '.')
n.newline()

n.comment('The arguments passed to configure.py, for rerunning it.')
n.variable('configure_args', ' '.join(sys.argv[1:]))
n.newline()

def src(filename):
    return os.path.join('src', filename)
def built(filename):
    return os.path.join('$builddir', filename)
def doc(filename):
    return os.path.join('doc', filename)
def cxx(name, **kwargs):
    return n.build(built(name + '.o'), 'cxx', src(name + '.cc'), **kwargs)
def binary(name):
    if platform == 'mingw':
        return name + '.exe'
    return name

n.variable('builddir', 'build')
n.variable('cxx', os.environ.get('CXX', 'g++'))
n.variable('ar', os.environ.get('AR', 'ar'))

cflags = ['-g', '-Wall', '-Wextra',
          '-Wno-deprecated',
          '-Wno-unused-parameter',
          '-fno-exceptions',
          '-fvisibility=hidden', '-pipe']
if not options.debug:
    cflags += ['-O2', '-DNDEBUG']
ldflags = ['-L$builddir']
libs = []

if platform == 'mingw':
    cflags.remove('-fvisibility=hidden');
    ldflags.append('-static')
else:
    if options.profile == 'gmon':
        cflags.append('-pg')
        ldflags.append('-pg')
    elif options.profile == 'pprof':
        libs.append('-lprofiler')

if 'CFLAGS' in os.environ:
    cflags.append(os.environ['CFLAGS'])
n.variable('cflags', ' '.join(cflags))
if 'LDFLAGS' in os.environ:
    ldflags.append(os.environ['LDFLAGS'])
n.variable('ldflags', ' '.join(ldflags))
n.newline()

n.rule('cxx',
       command='$cxx -MMD -MF $out.d $cflags -c $in -o $out',
       depfile='$out.d',
       description='CXX $out')
n.newline()

if host != 'mingw':
    n.rule('ar',
           command='rm -f $out && $ar crs $out $in',
           description='AR $out')
else:
    n.rule('ar',
           command='cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out',
           description='AR $out')
n.newline()

n.rule('link',
       command='$cxx $ldflags -o $out $in $libs',
       description='LINK $out')
n.newline()

objs = []

if platform != 'mingw':
    n.comment('browse_py.h is used to inline browse.py.')
    n.rule('inline',
           command='src/inline.sh $varname < $in > $out',
           description='INLINE $out')
    n.build(built('browse_py.h'), 'inline', src('browse.py'),
            implicit='src/inline.sh',
            variables=[('varname', 'kBrowsePy')])
    n.newline()

    objs += cxx('browse', order_only=built('browse_py.h'))
    n.newline()

n.comment('the depfile parser is generated using re2c.')
n.rule('re2c',
       command='re2c -b -i --no-generation-date -o $out $in',
       description='RE2C $out')
# Generate the .cc file in the source directory so we can check it in.
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
n.newline()

n.comment('Core source files all build into ninja library.')
for name in ['build',
             'build_log',
             'clean',
             'depfile_parser',
             'disk_interface',
             'edit_distance',
             'eval_env',
             'graph',
             'graphviz',
             'parsers',
             'state',
             'util']:
    objs += cxx(name)
if platform == 'mingw':
    objs += cxx('subprocess-win32')
else:
    objs += cxx('subprocess')
ninja_lib = n.build(built('libninja.a'), 'ar', objs)
n.newline()

libs.append('-lninja')

n.comment('Main executable is library plus main() function.')
objs = cxx('ninja')
ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib,
                variables=[('libs', libs)])
n.newline()

n.comment('Tests all build into ninja_test executable.')

variables = []
test_cflags = None
test_ldflags = None
test_libs = libs
objs = []
if options.with_gtest:
    path = options.with_gtest

    gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include'))
    objs += n.build(built('gtest-all.o'), 'cxx',
                    os.path.join(path, 'src/gtest-all.cc'),
                    variables=[('cflags', gtest_all_incs)])
    objs += n.build(built('gtest_main.o'), 'cxx',
                    os.path.join(path, 'src/gtest_main.cc'),
                    variables=[('cflags', gtest_all_incs)])

    test_cflags = cflags + ['-I%s' % os.path.join(path, 'include')]
else:
    test_libs.extend(['-lgtest_main', '-lgtest'])

for name in ['build_log_test',
             'build_test',
             'clean_test',
             'depfile_parser_test',
             'disk_interface_test',
             'edit_distance_test',
             'eval_env_test',
             'graph_test',
             'parsers_test',
             'state_test',
             'subprocess_test',
             'test',
             'util_test']:
    objs += cxx(name, variables=[('cflags', test_cflags)])

if platform != 'mingw':
    test_libs.append('-lpthread')
n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib,
        variables=[('ldflags', test_ldflags),
                   ('libs', test_libs)])
n.newline()

n.comment('Perftest executable.')
objs = cxx('parser_perftest')
n.build(binary('parser_perftest'), 'link', objs, implicit=ninja_lib,
        variables=[('libs', '-L$builddir -lninja')])
n.newline()

n.comment('Generate a graph using the "graph" tool.')
n.rule('gendot',
       command='./ninja -t graph > $out')
n.rule('gengraph',
       command='dot -Tpng $in > $out')
dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja'])
n.build('graph.png', 'gengraph', dot)
n.newline()

n.comment('Generate the manual using asciidoc.')
n.rule('asciidoc',
       command='asciidoc -a toc -o $out $in',
       description='ASCIIDOC $in')
manual = n.build(doc('manual.html'), 'asciidoc', doc('manual.asciidoc'))
n.build('manual', 'phony',
        order_only=manual)
n.newline()

n.comment('Generate Doxygen.')
n.rule('doxygen',
       command='doxygen $in',
       description='DOXYGEN $in')
n.variable('doxygen_mainpage_generator',
           src('gen_doxygen_mainpage.sh'))
n.rule('doxygen_mainpage',
       command='$doxygen_mainpage_generator $in > $out',
       description='DOXYGEN_MAINPAGE $out')
mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage',
                   ['README', 'HACKING', 'COPYING'],
                   implicit=['$doxygen_mainpage_generator'])
n.build('doxygen', 'doxygen', doc('doxygen.config'),
        implicit=mainpage)
n.newline()

if host != 'mingw':
    n.comment('Regenerate build files if build script changes.')
    n.rule('configure',
           command='./configure.py $configure_args',
           generator=True)
    n.build('build.ninja', 'configure',
            implicit=['configure.py', 'misc/ninja_syntax.py'])
    n.newline()

n.comment('Build only the main binary by default.')
n.default(ninja)

print 'wrote %s.' % BUILD_FILENAME
<|endoftext|>


<commit_before>from m5.SimObject import SimObject
from m5.params import *
class Repl(SimObject):
    type = 'Repl'
    abstract = True

class GenRepl(Repl):
    type = 'GenRepl'
    fresh_res = Param.Int("associativity")
    num_pools = Param.Int("capacity in bytes")
    pool_res = Param.Int("block size in bytes")
<commit_msg>Fix help strings on GenRepl params.<commit_after>from m5.SimObject import SimObject
from m5.params import *
class Repl(SimObject):
    type = 'Repl'
    abstract = True

class GenRepl(Repl):
    type = 'GenRepl'
    fresh_res = Param.Int("Fresh pool residency time")
    num_pools = Param.Int("Number of priority pools")
    pool_res = Param.Int("Pool residency time")
<|endoftext|>