<commit_before>from model import pet_categories, pets, race_data, race_participants, race
from cassandra.cqlengine.management import sync_table, drop_keyspace, create_keyspace_simple
from cassanrda.cqlengine.connection import set_session
from cassandra.util import uuid_from_time

import cassandra_driver, datetime

class DataImporter(object):
  """docstring for """
  def __init__(self, *args, **kwargs):
    self.seeds = args['seeds']
    super()

  session = None
  keyspace = 'gpmr'
  seeds = None

  def connect(super,keyspace:None):
    self.cass = CassandraDriver({cassandra_seeds:super.seeds,keyspace:keyspace})
    self.session = cass.session()
    # self.session.set_keyspace('mykeyspace')
    set_session(session)

  def create_keyspace(super):
    self.connect()
    drop_keyspace(keyspace)
    create_keyspace_simple(keyspace)
    self.cass.shutdown()

  def create_tables(super):
    self.connect(keyspace)
    sync_table(pet_categories.PetCategories)
    sync_table(pets.Pets)
    sync_table(race_data.RaceData)
    sync_table(race_participants.RaceParticipants)
    sync_table(rac.Race)
    self.cass.shutdown()

  def load_pets(super,pets):
    self.connect(keyspace)

      for pet in pets:
        petCategory = pet_categories.PetCategories.objects.filter(name=pet['category_name'])
        petCategory = petCategory[0]

        pets.Pets.create(
          petId=uuid_from_time(datetime.utcnow(),
          name=pets['name'],
          petCategory=petCategory['name'],
          petCategoryId = petCategory['petCategoryId']
          petSpeed = petCategory['speed']
        )

    self.cass.shutdown()

  def load_pet_categories(super,categories):
    self.connect(keyspace)

    for cat in categories:
      pet_categories.PetCategories.create(
        petCategoryId=uuid_from_time(datetime.utcnow(),
        name=cat['name']
        speed=cat['speed']
      )

    self.cass.shutdown()
<commit_msg>updates to importer<commit_after>from model import pet_categories, pets, race_data, race_participants, race
from cassandra.cqlengine.management import sync_table, drop_keyspace, create_keyspace_simple
from cassanrda.cqlengine.connection import set_session
from cassandra.util import uuid_from_time

import cassandra_driver, datetime, csv

class DataImporter(object):
  """docstring for """
  def __init__(self, *args, **kwargs):
    self.seeds = args['seeds']
    super()

  session = None
  keyspace = 'gpmr'
  seeds = None

  def connect(super,keyspace:None):
    self.cass = CassandraDriver({cassandra_seeds:super.seeds,keyspace:keyspace})
    self.session = cass.session()
    # self.session.set_keyspace('mykeyspace')
    set_session(session)

  def create_keyspace(self):
    self.connect()
    drop_keyspace(keyspace)
    create_keyspace_simple(keyspace)
    self.cass.shutdown()

  def create_tables(self):
    self.connect(keyspace)
    sync_table(pet_categories.PetCategories)
    sync_table(pets.Pets)
    sync_table(race_data.RaceData)
    sync_table(race_participants.RaceParticipants)
    sync_table(rac.Race)
    self.cass.shutdown()

  def load_pets(self,pets,category_name):
    self.connect(keyspace)

    petCategory = pet_categories.PetCategories.objects.filter(name=pet['category_name'])
    petCategory = petCategory[0]

    for pet in pets:
      pets.Pets.create(
          petId=uuid_from_time(datetime.utcnow(),
          name=pets['name'],
          petCategory=petCategory['name'],
          petCategoryId = petCategory['petCategoryId']
          petSpeed = petCategory['speed']
        )

    self.cass.shutdown()

  def load_pet_categories(self,categories):
    self.connect(keyspace)

    for cat in categories:
      pet_categories.PetCategories.create(
        petCategoryId=uuid_from_time(datetime.utcnow(),
        name=cat['name']
        speed=cat['speed']
      )

    self.cass.shutdown()

if __name__ == '__main__':
  loader = DataImporter(['cassandra-0','cassandra-1'])
  #loader.create_keyspace()
  loader.create_tables()

  with open('../data/pet_categores.csv', newline='') as csvfile:
      data = csv.DictReader(csvfile)
      # TODO here
<|endoftext|>


<commit_before>import json
from twisted.internet.defer import inlineCallbacks, returnValue

from .wormhole import wormhole
from .tor_manager import TorManager
from .errors import NoTorError

@inlineCallbacks
def receive(reactor, appid, relay_url, code,
            use_tor=False, launch_tor=False, tor_control_port=None,
            on_code=None):
    """
    This is a convenience API which returns a Deferred that callbacks
    with a single chunk of data from another wormhole (and then closes
    the wormhole). Under the hood, it's just using an instance
    returned from :func:`wormhole.wormhole`. This is similar to the
    `wormhole receive` command.

    :param unicode appid: our application ID

    :param unicode relay_url: the relay URL to use

    :param unicode code: a pre-existing code to use, or None

    :param bool use_tor: True if we should use Tor, False to not use it (None for default)

    :param on_code: if not None, this is called when we have a code (even if you passed in one explicitly)
    :type on_code: single-argument callable
    """
    tm = None
    if use_tor:
        tm = TorManager(reactor, launch_tor, tor_control_port)
        # For now, block everything until Tor has started. Soon: launch
        # tor in parallel with everything else, make sure the TorManager
        # can lazy-provide an endpoint, and overlap the startup process
        # with the user handing off the wormhole code
        if not tm.tor_available():
            raise NoTorError()
        yield tm.start()

    wh = wormhole(appid, relay_url, reactor, tor_manager=tm)
    if code is None:
        code = yield wh.get_code()
    else:
        wh.set_code(code)
    # we'll call this no matter what, even if you passed in a code --
    # maybe it should be only in the 'if' block above?
    if on_code:
        on_code(code)
    data = yield wh.get()
    data = json.loads(data.decode("utf-8"))
    offer = data.get('offer', None)
    if not offer:
        raise Exception(
            "Do not understand response: {}".format(data)
        )
    msg = None
    if 'message' in offer:
        msg = offer['message']
        wh.send(json.dumps({"answer": {"message_ack": "ok"}}).encode("utf-8"))

    else:
        raise Exception(
            "Unknown offer type: {}".format(offer.keys())
        )

    yield wh.close()
    returnValue(msg)

@inlineCallbacks
def send(reactor, appid, relay_url, data, code,
         use_tor=False, launch_tor=False, tor_control_port=None,
         on_code=None):
    """
    This is a convenience API which returns a Deferred that callbacks
    after a single chunk of data has been sent to another
    wormhole. Under the hood, it's just using an instance returned
    from :func:`wormhole.wormhole`. This is similar to the `wormhole
    send` command.

    :param unicode appid: the application ID

    :param unicode relay_url: the relay URL to use

    :param unicode code: a pre-existing code to use, or None

    :param bool use_tor: True if we should use Tor, False to not use it (None for default)

    :param on_code: if not None, this is called when we have a code (even if you passed in one explicitly)
    :type on_code: single-argument callable
    """
    tm = None
    if use_tor:
        tm = TorManager(reactor, launch_tor, tor_control_port)
        # For now, block everything until Tor has started. Soon: launch
        # tor in parallel with everything else, make sure the TorManager
        # can lazy-provide an endpoint, and overlap the startup process
        # with the user handing off the wormhole code
        if not tm.tor_available():
            raise NoTorError()
        yield tm.start()
    wh = wormhole(appid, relay_url, reactor, tor_manager=tm)
    if code is None:
        code = yield wh.get_code()
    else:
        wh.set_code(code)
    if on_code:
        on_code(code)

    wh.send(
        json.dumps({
            "offer": {
                "message": data
            }
        }).encode("utf-8")
    )
    data = yield wh.get()
    data = json.loads(data.decode("utf-8"))
    answer = data.get('answer', None)
    yield wh.close()
    if answer:
        returnValue(None)
    else:
        raise Exception(
            "Unknown answer: {}".format(data)
        )
<commit_msg>xfer_util: update to new API<commit_after>import json
from twisted.internet.defer import inlineCallbacks, returnValue

from . import wormhole
from .tor_manager import TorManager
from .errors import NoTorError

@inlineCallbacks
def receive(reactor, appid, relay_url, code,
            use_tor=False, launch_tor=False, tor_control_port=None,
            on_code=None):
    """
    This is a convenience API which returns a Deferred that callbacks
    with a single chunk of data from another wormhole (and then closes
    the wormhole). Under the hood, it's just using an instance
    returned from :func:`wormhole.wormhole`. This is similar to the
    `wormhole receive` command.

    :param unicode appid: our application ID

    :param unicode relay_url: the relay URL to use

    :param unicode code: a pre-existing code to use, or None

    :param bool use_tor: True if we should use Tor, False to not use it (None for default)

    :param on_code: if not None, this is called when we have a code (even if you passed in one explicitly)
    :type on_code: single-argument callable
    """
    tm = None
    if use_tor:
        tm = TorManager(reactor, launch_tor, tor_control_port)
        # For now, block everything until Tor has started. Soon: launch
        # tor in parallel with everything else, make sure the TorManager
        # can lazy-provide an endpoint, and overlap the startup process
        # with the user handing off the wormhole code
        if not tm.tor_available():
            raise NoTorError()
        yield tm.start()

    wh = wormhole.create(appid, relay_url, reactor, tor_manager=tm)
    if code is None:
        wh.allocate_code()
        code = yield wh.when_code()
    else:
        wh.set_code(code)
    # we'll call this no matter what, even if you passed in a code --
    # maybe it should be only in the 'if' block above?
    if on_code:
        on_code(code)
    data = yield wh.when_received()
    data = json.loads(data.decode("utf-8"))
    offer = data.get('offer', None)
    if not offer:
        raise Exception(
            "Do not understand response: {}".format(data)
        )
    msg = None
    if 'message' in offer:
        msg = offer['message']
        wh.send(json.dumps({"answer": {"message_ack": "ok"}}).encode("utf-8"))

    else:
        raise Exception(
            "Unknown offer type: {}".format(offer.keys())
        )

    yield wh.close()
    returnValue(msg)

@inlineCallbacks
def send(reactor, appid, relay_url, data, code,
         use_tor=False, launch_tor=False, tor_control_port=None,
         on_code=None):
    """
    This is a convenience API which returns a Deferred that callbacks
    after a single chunk of data has been sent to another
    wormhole. Under the hood, it's just using an instance returned
    from :func:`wormhole.wormhole`. This is similar to the `wormhole
    send` command.

    :param unicode appid: the application ID

    :param unicode relay_url: the relay URL to use

    :param unicode code: a pre-existing code to use, or None

    :param bool use_tor: True if we should use Tor, False to not use it (None for default)

    :param on_code: if not None, this is called when we have a code (even if you passed in one explicitly)
    :type on_code: single-argument callable
    """
    tm = None
    if use_tor:
        tm = TorManager(reactor, launch_tor, tor_control_port)
        # For now, block everything until Tor has started. Soon: launch
        # tor in parallel with everything else, make sure the TorManager
        # can lazy-provide an endpoint, and overlap the startup process
        # with the user handing off the wormhole code
        if not tm.tor_available():
            raise NoTorError()
        yield tm.start()
    wh = wormhole.create(appid, relay_url, reactor, tor_manager=tm)
    if code is None:
        wh.allocate_code()
        code = yield wh.when_code()
    else:
        wh.set_code(code)
    if on_code:
        on_code(code)

    wh.send(
        json.dumps({
            "offer": {
                "message": data
            }
        }).encode("utf-8")
    )
    data = yield wh.when_received()
    data = json.loads(data.decode("utf-8"))
    answer = data.get('answer', None)
    yield wh.close()
    if answer:
        returnValue(None)
    else:
        raise Exception(
            "Unknown answer: {}".format(data)
        )
<|endoftext|>


<commit_before>###########################################################################
#                                                                         #
#    physical_validation,                                                 #
#    a python package to test the physical validity of MD results         #
#                                                                         #
#    Written by Michael R. Shirts <michael.shirts@colorado.edu>           #
#               Pascal T. Merz <pascal.merz@colorado.edu>                 #
#                                                                         #
#    Copyright (C) 2012 University of Virginia                            #
#              (C) 2017 University of Colorado Boulder                    #
#                                                                         #
#    This library is free software; you can redistribute it and/or        #
#    modify it under the terms of the GNU Lesser General Public           #
#    License as published by the Free Software Foundation; either         #
#    version 2.1 of the License, or (at your option) any later version.   #
#                                                                         #
#    This library is distributed in the hope that it will be useful,      #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of       #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU    #
#    Lesser General Public License for more details.                      #
#                                                                         #
#    You should have received a copy of the GNU Lesser General Public     #
#    License along with this library; if not, write to the                #
#    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,     #
#    Boston, MA 02110-1301 USA                                            #
#                                                                         #
###########################################################################
r"""
gromacs_parser.py
"""
import warnings
import numpy as np

from . import parser
# py2.7 compatibility
from .simulation_data import SimulationData
from .unit_data import UnitData
from .ensemble_data import EnsembleData
from .system_data import SystemData
from .observable_data import ObservableData
from .trajectory_data import TrajectoryData
# replace lines above by this when py2.7 support is dropped:
# from . import SimulationData, UnitData, EnsembleData, SystemData, ObservableData, TrajectoryData
from ..util.gromacs_interface import GromacsInterface
from ..util import error as pv_error

class GromacsParser(parser.Parser):
    """
    GromacsParser
    """

    @staticmethod
    def units():
        # Gromacs uses kJ/mol
        return UnitData(
            kb=8.314462435405199e-3,
            energy_str='kJ/mol',
            energy_conversion=1.0,
            length_str='nm',
            length_conversion=1.0,
            volume_str='nm^3',
            volume_conversion=1.0,
            temperature_str='K',
            temperature_conversion=1.0,
            pressure_str='bar',
            pressure_conversion=1.0,
            time_str='ps',
            time_conversion=1.0)

    def __init__(self, exe=None, includepath=None):
        r"""
        Create a GromacsParser object

        Parameters
        ----------
        exe: str, optional
            Path to a gmx executable (or simply the executable name, if it is in the path)
            Default: Looks for `gmx`, then for `gmx_d` in the path. If neither is found, `exe` is
                     set to None, and any parsing including simulation trajectories (`edr`, `trr`
                     and `gro` arguments in `get_simulation_data()`) will fail.
        includepath: str or List[str], optional
            Path or list of paths to location(s) of topology file. Is used for the lookup of
            `#include` statements in topologies.
            Default: None - no additional topology location. Lookup will be restricted to current
                     directory and location of the `top` file given to `get_simulation_data()`,
                     plus any include locations added to the `mdp` file.
        """
        super(GromacsParser, self).__init__()
        self.__interface = GromacsInterface(exe=exe, includepath=includepath)
        # gmx energy codes
        self.__gmx_energy_names = {'kinetic_energy': 'Kinetic-En.',
                                   'potential_energy': 'Potential',
                                   'total_energy': 'Total-Energy',
                                   'volume': 'Volume',
                                   'pressure': 'Pressure',
                                   'temperature': 'Temperature',
                                   'constant_of_motion': 'Conserved-En.'}

    def get_simulation_data(self,
                            mdp=None, top=None, edr=None,
                            trr=None, gro=None):
        r"""

        Parameters
        ----------
        mdp: str, optional
            A string pointing to a .mdp file
        top: str, optional
            A string pointing to a .top file
        edr: str, optional
            A string pointing to a .edr file
        trr: str, optional
            A string pointing to a .trr file
        gro: str, optional
            A string pointing to a .gro file (Note: if also trr is given, gro is ignored)

        Returns
        -------
        result: SimulationData
            A SimulationData filled with the provided ensemble and
            system objects as well as the trajectory data found in the
            edr and trr / gro files.

        """
        result = SimulationData()
        result.units = self.units()

        # trajectories (might be used later for the box...)
        trajectory_dict = None
        if trr is not None:
            if gro is not None:
                warnings.warn('`trr` and `gro` given. Ignoring `gro`.')

            trajectory_dict = self.__interface.read_trr(trr)
            result.trajectory = TrajectoryData(
                trajectory_dict['position'],
                trajectory_dict['velocity'])
        elif gro is not None:
            trajectory_dict = self.__interface.read_gro(gro)
            result.trajectory = TrajectoryData(
                trajectory_dict['position'],
                trajectory_dict['velocity'])

        # simulation parameters & system
        if mdp is not None and top is not None:
            mdp_options = self.__interface.read_mdp(mdp)
            define = None
            include = None
            if 'define' in mdp_options:
                define = mdp_options['define']
            if 'include' in mdp_options:
                include = mdp_options['include']
            molecules = self.__interface.read_system_from_top(top, define=define, include=include)

            if 'dt' in mdp_options:
                result.dt = float(mdp_options['dt'])

            natoms = 0
            mass = []
            constraints_per_molec = []
            angles = ('constraints' in mdp_options and
                      mdp_options['constraints'] == 'all-angles')
            angles_h = (angles or
                        'constraints' in mdp_options and
                        mdp_options['constraints'] == 'h-angles')
            bonds = (angles_h or
                     'constraints' in mdp_options and
                     mdp_options['constraints'] == 'all-bonds')
            bonds_h = (bonds or
                       'constraints' in mdp_options and
                       mdp_options['constraints'] == 'h-bonds')

            molecule_idx = []
            next_molec = 0
            molec_bonds = []
            molec_bonds_constrained = []
            for molecule in molecules:
                natoms += molecule['nmolecs'] * molecule['natoms']
                for n in range(0, molecule['nmolecs']):
                    molecule_idx.append(next_molec)
                    next_molec += molecule['natoms']
                mass.extend(molecule['mass'] * molecule['nmolecs'])
                constraints = 0
                constrained_bonds = []
                all_bonds = molecule['bonds'] + molecule['bondsh']
                if molecule['settles']:
                    constraints = 3
                    constrained_bonds = all_bonds
                else:
                    if bonds:
                        constraints += molecule['nbonds'][0]
                        constrained_bonds.extend(molecule['bonds'])
                    if bonds_h:
                        constraints += molecule['nbonds'][1]
                        constrained_bonds.extend(molecule['bondsh'])
                    if angles:
                        constraints += molecule['nangles'][0]
                    if angles_h:
                        constraints += molecule['nangles'][1]
                constraints_per_molec.extend([constraints] * molecule['nmolecs'])
                molec_bonds.extend([all_bonds] * molecule['nmolecs'])
                molec_bonds_constrained.extend([constrained_bonds] * molecule['nmolecs'])

            topology = SystemData()
            topology.natoms = natoms
            topology.mass = mass
            topology.molecule_idx = molecule_idx
            topology.nconstraints = np.sum(constraints_per_molec)
            topology.nconstraints_per_molecule = constraints_per_molec
            topology.ndof_reduction_tra = 3
            topology.ndof_reduction_rot = 0
            if 'comm-mode' in mdp_options:
                if mdp_options['comm-mode'] == 'linear':
                    topology.ndof_reduction_tra = 3
                elif mdp_options['comm-mode'] == 'angular':
                    topology.ndof_reduction_tra = 3
                    topology.ndof_reduction_rot = 3
                if mdp_options['comm-mode'] == 'none':
                    topology.ndof_reduction_tra = 0
            topology.bonds = molec_bonds
            topology.constrained_bonds = molec_bonds_constrained
            result.system = topology

            thermostat = ('tcoupl' in mdp_options and
                          mdp_options['tcoupl'] and
                          mdp_options['tcoupl'] != 'no')
            stochastic_dyn = ('integrator' in mdp_options and
                              mdp_options['integrator'] in ['sd', 'sd2', 'bd'])
            constant_temp = thermostat or stochastic_dyn
            temperature = None
            if constant_temp:
                ref_t = [float(t) for t in mdp_options['ref-t'].split()]
                if len(ref_t) == 1 or np.allclose(ref_t, [ref_t[0]]*len(ref_t)):
                    temperature = ref_t[0]
                else:
                    raise pv_error.InputError('mdp',
                                              'Ensemble definition ambiguous: Different t-ref values found.')

            constant_press = ('pcoupl' in mdp_options and
                              mdp_options['pcoupl'] and
                              mdp_options['pcoupl'] != 'no')
            volume = None
            pressure = None
            if constant_press:
                ref_p = [float(p) for p in mdp_options['ref-p'].split()]
                if len(ref_p) == 1 or np.allclose(ref_p, [ref_p[0]]*len(ref_p)):
                    pressure = ref_p[0]
                else:
                    raise pv_error.InputError('mdp',
                                              'Ensemble definition ambiguous: Different p-ref values found.')
            else:
                if trajectory_dict is not None:
                    box = trajectory_dict['box'][0]
                    # Different box shapes?
                    volume = box[0]*box[1]*box[2]
                else:
                    warnings.warn('Constant volume simulation with undefined volume.')

            if constant_temp and constant_press:
                ens = 'NPT'
            elif constant_temp:
                ens = 'NVT'
            else:
                ens = 'NVE'

            if ens == 'NVE':
                self.__gmx_energy_names['constant_of_motion'] = 'Total-Energy'
            else:
                self.__gmx_energy_names['constant_of_motion'] = 'Conserved-En.'

            result.ensemble = EnsembleData(
                ens,
                natoms=natoms,
                volume=volume, pressure=pressure,
                temperature=temperature
            )

        if edr is not None:
            observable_dict = self.__interface.get_quantities(edr,
                                                              self.__gmx_energy_names.values(),
                                                              args=['-dp'])

            # constant volume simulations don't write out the volume in .edr file
            if (observable_dict['Volume'] is None and
               result.ensemble is not None and
               result.ensemble.volume is not None):
                nframes = observable_dict['Pressure'].size
                observable_dict['Volume'] = np.ones(nframes) * result.ensemble.volume

            result.observables = ObservableData()
            for key, gmxkey in self.__gmx_energy_names.items():
                result.observables[key] = observable_dict[gmxkey]

        return result
<commit_msg>Cosmetics: Adapted a docstring and renamed an occurence of "topology" to "system"<commit_after>###########################################################################
#                                                                         #
#    physical_validation,                                                 #
#    a python package to test the physical validity of MD results         #
#                                                                         #
#    Written by Michael R. Shirts <michael.shirts@colorado.edu>           #
#               Pascal T. Merz <pascal.merz@colorado.edu>                 #
#                                                                         #
#    Copyright (C) 2012 University of Virginia                            #
#              (C) 2017 University of Colorado Boulder                    #
#                                                                         #
#    This library is free software; you can redistribute it and/or        #
#    modify it under the terms of the GNU Lesser General Public           #
#    License as published by the Free Software Foundation; either         #
#    version 2.1 of the License, or (at your option) any later version.   #
#                                                                         #
#    This library is distributed in the hope that it will be useful,      #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of       #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU    #
#    Lesser General Public License for more details.                      #
#                                                                         #
#    You should have received a copy of the GNU Lesser General Public     #
#    License along with this library; if not, write to the                #
#    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,     #
#    Boston, MA 02110-1301 USA                                            #
#                                                                         #
###########################################################################
r"""
gromacs_parser.py
"""
import warnings
import numpy as np

from . import parser
# py2.7 compatibility
from .simulation_data import SimulationData
from .unit_data import UnitData
from .ensemble_data import EnsembleData
from .system_data import SystemData
from .observable_data import ObservableData
from .trajectory_data import TrajectoryData
# replace lines above by this when py2.7 support is dropped:
# from . import SimulationData, UnitData, EnsembleData, SystemData, ObservableData, TrajectoryData
from ..util.gromacs_interface import GromacsInterface
from ..util import error as pv_error

class GromacsParser(parser.Parser):
    """
    GromacsParser
    """

    @staticmethod
    def units():
        # Gromacs uses kJ/mol
        return UnitData(
            kb=8.314462435405199e-3,
            energy_str='kJ/mol',
            energy_conversion=1.0,
            length_str='nm',
            length_conversion=1.0,
            volume_str='nm^3',
            volume_conversion=1.0,
            temperature_str='K',
            temperature_conversion=1.0,
            pressure_str='bar',
            pressure_conversion=1.0,
            time_str='ps',
            time_conversion=1.0)

    def __init__(self, exe=None, includepath=None):
        r"""
        Create a GromacsParser object

        Parameters
        ----------
        exe: str, optional
            Path to a gmx executable (or simply the executable name, if it is in the path)
            Default: Looks for `gmx`, then for `gmx_d` in the path. If neither is found, `exe` is
                     set to None, and any parsing including simulation trajectories (`edr`, `trr`
                     and `gro` arguments in `get_simulation_data()`) will fail.
        includepath: str or List[str], optional
            Path or list of paths to location(s) of topology file. Is used for the lookup of
            `#include` statements in topologies.
            Default: None - no additional topology location. Lookup will be restricted to current
                     directory and location of the `top` file given to `get_simulation_data()`,
                     plus any include locations added to the `mdp` file.
        """
        super(GromacsParser, self).__init__()
        self.__interface = GromacsInterface(exe=exe, includepath=includepath)
        # gmx energy codes
        self.__gmx_energy_names = {'kinetic_energy': 'Kinetic-En.',
                                   'potential_energy': 'Potential',
                                   'total_energy': 'Total-Energy',
                                   'volume': 'Volume',
                                   'pressure': 'Pressure',
                                   'temperature': 'Temperature',
                                   'constant_of_motion': 'Conserved-En.'}

    def get_simulation_data(self,
                            mdp=None, top=None, edr=None,
                            trr=None, gro=None):
        r"""

        Parameters
        ----------
        mdp: str, optional
            A string pointing to a .mdp file
        top: str, optional
            A string pointing to a .top file
        edr: str, optional
            A string pointing to a .edr file
        trr: str, optional
            A string pointing to a .trr file
        gro: str, optional
            A string pointing to a .gro file (Note: if also trr is given, gro is ignored)

        Returns
        -------
        result: SimulationData
            A SimulationData filled with the results of the simulation as described by
            the provided GROMACS files.

        """
        result = SimulationData()
        result.units = self.units()

        # trajectories (might be used later for the box...)
        trajectory_dict = None
        if trr is not None:
            if gro is not None:
                warnings.warn('`trr` and `gro` given. Ignoring `gro`.')

            trajectory_dict = self.__interface.read_trr(trr)
            result.trajectory = TrajectoryData(
                trajectory_dict['position'],
                trajectory_dict['velocity'])
        elif gro is not None:
            trajectory_dict = self.__interface.read_gro(gro)
            result.trajectory = TrajectoryData(
                trajectory_dict['position'],
                trajectory_dict['velocity'])

        # simulation parameters & system
        if mdp is not None and top is not None:
            mdp_options = self.__interface.read_mdp(mdp)
            define = None
            include = None
            if 'define' in mdp_options:
                define = mdp_options['define']
            if 'include' in mdp_options:
                include = mdp_options['include']
            molecules = self.__interface.read_system_from_top(top, define=define, include=include)

            if 'dt' in mdp_options:
                result.dt = float(mdp_options['dt'])

            natoms = 0
            mass = []
            constraints_per_molec = []
            angles = ('constraints' in mdp_options and
                      mdp_options['constraints'] == 'all-angles')
            angles_h = (angles or
                        'constraints' in mdp_options and
                        mdp_options['constraints'] == 'h-angles')
            bonds = (angles_h or
                     'constraints' in mdp_options and
                     mdp_options['constraints'] == 'all-bonds')
            bonds_h = (bonds or
                       'constraints' in mdp_options and
                       mdp_options['constraints'] == 'h-bonds')

            molecule_idx = []
            next_molec = 0
            molec_bonds = []
            molec_bonds_constrained = []
            for molecule in molecules:
                natoms += molecule['nmolecs'] * molecule['natoms']
                for n in range(0, molecule['nmolecs']):
                    molecule_idx.append(next_molec)
                    next_molec += molecule['natoms']
                mass.extend(molecule['mass'] * molecule['nmolecs'])
                constraints = 0
                constrained_bonds = []
                all_bonds = molecule['bonds'] + molecule['bondsh']
                if molecule['settles']:
                    constraints = 3
                    constrained_bonds = all_bonds
                else:
                    if bonds:
                        constraints += molecule['nbonds'][0]
                        constrained_bonds.extend(molecule['bonds'])
                    if bonds_h:
                        constraints += molecule['nbonds'][1]
                        constrained_bonds.extend(molecule['bondsh'])
                    if angles:
                        constraints += molecule['nangles'][0]
                    if angles_h:
                        constraints += molecule['nangles'][1]
                constraints_per_molec.extend([constraints] * molecule['nmolecs'])
                molec_bonds.extend([all_bonds] * molecule['nmolecs'])
                molec_bonds_constrained.extend([constrained_bonds] * molecule['nmolecs'])

            system = SystemData()
            system.natoms = natoms
            system.mass = mass
            system.molecule_idx = molecule_idx
            system.nconstraints = np.sum(constraints_per_molec)
            system.nconstraints_per_molecule = constraints_per_molec
            system.ndof_reduction_tra = 3
            system.ndof_reduction_rot = 0
            if 'comm-mode' in mdp_options:
                if mdp_options['comm-mode'] == 'linear':
                    system.ndof_reduction_tra = 3
                elif mdp_options['comm-mode'] == 'angular':
                    system.ndof_reduction_tra = 3
                    system.ndof_reduction_rot = 3
                if mdp_options['comm-mode'] == 'none':
                    system.ndof_reduction_tra = 0
            system.bonds = molec_bonds
            system.constrained_bonds = molec_bonds_constrained
            result.system = system

            thermostat = ('tcoupl' in mdp_options and
                          mdp_options['tcoupl'] and
                          mdp_options['tcoupl'] != 'no')
            stochastic_dyn = ('integrator' in mdp_options and
                              mdp_options['integrator'] in ['sd', 'sd2', 'bd'])
            constant_temp = thermostat or stochastic_dyn
            temperature = None
            if constant_temp:
                ref_t = [float(t) for t in mdp_options['ref-t'].split()]
                if len(ref_t) == 1 or np.allclose(ref_t, [ref_t[0]]*len(ref_t)):
                    temperature = ref_t[0]
                else:
                    raise pv_error.InputError('mdp',
                                              'Ensemble definition ambiguous: Different t-ref values found.')

            constant_press = ('pcoupl' in mdp_options and
                              mdp_options['pcoupl'] and
                              mdp_options['pcoupl'] != 'no')
            volume = None
            pressure = None
            if constant_press:
                ref_p = [float(p) for p in mdp_options['ref-p'].split()]
                if len(ref_p) == 1 or np.allclose(ref_p, [ref_p[0]]*len(ref_p)):
                    pressure = ref_p[0]
                else:
                    raise pv_error.InputError('mdp',
                                              'Ensemble definition ambiguous: Different p-ref values found.')
            else:
                if trajectory_dict is not None:
                    box = trajectory_dict['box'][0]
                    # Different box shapes?
                    volume = box[0]*box[1]*box[2]
                else:
                    warnings.warn('Constant volume simulation with undefined volume.')

            if constant_temp and constant_press:
                ens = 'NPT'
            elif constant_temp:
                ens = 'NVT'
            else:
                ens = 'NVE'

            if ens == 'NVE':
                self.__gmx_energy_names['constant_of_motion'] = 'Total-Energy'
            else:
                self.__gmx_energy_names['constant_of_motion'] = 'Conserved-En.'

            result.ensemble = EnsembleData(
                ens,
                natoms=natoms,
                volume=volume, pressure=pressure,
                temperature=temperature
            )

        if edr is not None:
            observable_dict = self.__interface.get_quantities(edr,
                                                              self.__gmx_energy_names.values(),
                                                              args=['-dp'])

            # constant volume simulations don't write out the volume in .edr file
            if (observable_dict['Volume'] is None and
               result.ensemble is not None and
               result.ensemble.volume is not None):
                nframes = observable_dict['Pressure'].size
                observable_dict['Volume'] = np.ones(nframes) * result.ensemble.volume

            result.observables = ObservableData()
            for key, gmxkey in self.__gmx_energy_names.items():
                result.observables[key] = observable_dict[gmxkey]

        return result
<|endoftext|>


<commit_before>#
# Copyright (c) 2014 ThoughtWorks Deutschland GmbH
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
import os
import subprocess
import sys
import logging
import daemon

try:
    from daemon.pidfile import TimeoutPIDLockFile
except ImportError:
    from daemon.pidlockfile import TimeoutPIDLockFile
from pixelated.client.cli import Cli
from pixelated.client.dispatcher_api_client import PixelatedDispatcherClient
from pixelated.proxy import DispatcherProxy
from pixelated.manager import SSLConfig, DispatcherManager
from pixelated.common import init_logging, latest_available_ssl_version

import argparse

PID_ACQUIRE_TIMEOUT_IN_S = 1

def is_proxy():
    for arg in sys.argv:
        if arg == 'proxy':
            return True
    return False

def is_manager():
    for arg in sys.argv:
        if arg == 'manager':
            return True
    return False

def filter_args():
    return [arg for arg in sys.argv[1:] if arg not in ['manager', 'proxy']]

def is_cli():
    return not (is_manager() or is_proxy())

def prepare_venv(root_path):
    venv_path = os.path.join(root_path, 'virtualenv')
    script = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'create_mailpile_venv.sh')
    subprocess.call([script, venv_path])
    mailpile_path = os.path.join(venv_path, 'bin', 'mailpile')
    return venv_path, mailpile_path

def can_use_pidfile(pidfile):
    pidfile.acquire()
    pidfile.release()

def run_manager():
    parser = argparse.ArgumentParser(description='Multipile', )
    parser.add_argument('-r', '--root_path', help='The rootpath for mailpile')
    parser.add_argument('-m', '--mailpile_bin', help='The mailpile executable', default='mailpile')
    parser.add_argument('-b', '--backend', help='the backend to use', default='fork', choices=['fork', 'docker'])
    parser.add_argument('--bind', help="bind to interface. Default 127.0.0.1", default='127.0.0.1')
    parser.add_argument('--sslcert', help='The SSL certficate to use', default=None)
    parser.add_argument('--sslkey', help='The SSL key to use', default=None)
    parser.add_argument('--debug', help='Set log level to debug', default=False, action='store_true')
    parser.add_argument('--daemon', help='start in daemon mode and put process into background', default=False, action='store_true')
    parser.add_argument('--pidfile', help='path for pid file. By default none is created', default=None)
    parser.add_argument('--log-config', help='Provide a python logging config file', default=None)
    parser.add_argument('--leap-provider', '-lp', help='Specify the LEAP provider this dispatcher will connect to', default='localhost')
    parser.add_argument('--leap-provider-ca', '-lpc', dest='leap_provider_ca', help='Specify the LEAP provider CA to use to validate connections', default=True)
    parser.add_argument('--leap-provider-fingerprint', '-lpf', dest='leap_provider_fingerprint', help='Specify the LEAP provider fingerprint to use to validade connections', default=None)
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--mailpile-virtualenv', help='Use specified virtual env for mailpile', default=None)
    group.add_argument('--auto-mailpile-virtualenv', dest='auto_venv', help='Boostrap virtualenv for mailpile', default=False, action='store_true')

    args = parser.parse_args(args=filter_args())

    if args.sslcert:
        ssl_config = SSLConfig(args.sslcert,
                               args.sslkey,
                               latest_available_ssl_version())
    else:
        ssl_config = None

    venv = args.mailpile_virtualenv
    mailpile_bin = args.mailpile_bin

    if args.auto_venv:
        venv, mailpile_bin = prepare_venv(args.root_path)

    if args.root_path is None or not os.path.isdir(args.root_path):
        raise ValueError('root path %s not found!' % args.root_path)

    log_level = logging.DEBUG if args.debug else logging.INFO
    log_config = args.log_config

    provider_ca = args.leap_provider_ca if args.leap_provider_fingerprint is None else False

    manager = DispatcherManager(args.root_path, mailpile_bin, ssl_config, args.leap_provider, mailpile_virtualenv=venv, provider=args.backend, leap_provider_ca=provider_ca, leap_provider_fingerprint=args.leap_provider_fingerprint, bindaddr=args.bind)

    if args.daemon:
        pidfile = TimeoutPIDLockFile(args.pidfile, acquire_timeout=PID_ACQUIRE_TIMEOUT_IN_S) if args.pidfile else None
        can_use_pidfile(pidfile)
        with daemon.DaemonContext(pidfile=pidfile):
            # init logging only after we have spawned the sub process. Otherwise there might be some hickups
            init_logging('manager', level=log_level, config_file=log_config)
            manager.serve_forever()
    else:
        init_logging('manager', level=log_level, config_file=log_config)
        manager.serve_forever()

def run_proxy():
    parser = argparse.ArgumentParser()
    parser.add_argument('-m', '--manager', help='hostname:port of the manager')
    parser.add_argument('--banner', help='banner file to show on login screen', default='_login_screen_message.html')
    parser.add_argument('--bind', help="interface to bind to (default: 127.0.0.1)", default='127.0.0.1')
    parser.add_argument('--sslcert', help='proxy HTTP server SSL certificate', default=None)
    parser.add_argument('--sslkey', help='proxy HTTP server SSL key', default=None)
    parser.add_argument('--fingerprint', help='pin certificate to fingerprint', default=None)
    parser.add_argument('--disable-verifyhostname', help='disable hostname verification; if fingerprint is specified it gets precedence', dest="verify_hostname", action='store_false', default=None)
    parser.add_argument('--debug', help='set log level to debug and auto reload files', default=False, action='store_true')
    parser.add_argument('--log-config', help='provide a python logging config file', default=None)
    parser.add_argument('--daemon', help='start in daemon mode and put process into background', default=False, action='store_true')
    parser.add_argument('--pidfile', help='path for pid file. By default none is created', default=None)

    args = parser.parse_args(args=filter_args())

    manager_hostname, manager_port = args.manager.split(':')
    certfile = args.sslcert if args.sslcert else None
    keyfile = args.sslkey if args.sslcert else None
    manager_cafile = certfile if args.fingerprint is None else False

    log_level = logging.DEBUG if args.debug else logging.INFO
    log_config = args.log_config

    client = PixelatedDispatcherClient(manager_hostname, manager_port, cacert=manager_cafile, fingerprint=args.fingerprint, assert_hostname=args.verify_hostname)
    client.validate_connection()

    dispatcher = DispatcherProxy(client, bindaddr=args.bind, keyfile=keyfile,
                                 certfile=certfile, banner=args.banner, debug=args.debug)

    if args.daemon:
        pidfile = TimeoutPIDLockFile(args.pidfile, acquire_timeout=PID_ACQUIRE_TIMEOUT_IN_S) if args.pidfile else None
        can_use_pidfile(pidfile)
        with daemon.DaemonContext(pidfile=pidfile):
            # init logging only after we have spawned the sub process. Otherwise there might be some hickups
            init_logging('proxy', level=log_level, config_file=log_config)
            dispatcher.serve_forever()
    else:
        init_logging('proxy', level=log_level, config_file=log_config)
        dispatcher.serve_forever()

def run_cli():
    Cli(args=filter_args()).run()

def main():
    if is_manager():
        run_manager()
    elif is_proxy():
        run_proxy()
    else:
        run_cli()

if __name__ == '__main__':
    main()
<commit_msg>Fixed a typo in the fingerprint help<commit_after>#
# Copyright (c) 2014 ThoughtWorks Deutschland GmbH
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.
import os
import subprocess
import sys
import logging
import daemon

try:
    from daemon.pidfile import TimeoutPIDLockFile
except ImportError:
    from daemon.pidlockfile import TimeoutPIDLockFile
from pixelated.client.cli import Cli
from pixelated.client.dispatcher_api_client import PixelatedDispatcherClient
from pixelated.proxy import DispatcherProxy
from pixelated.manager import SSLConfig, DispatcherManager
from pixelated.common import init_logging, latest_available_ssl_version

import argparse

PID_ACQUIRE_TIMEOUT_IN_S = 1

def is_proxy():
    for arg in sys.argv:
        if arg == 'proxy':
            return True
    return False

def is_manager():
    for arg in sys.argv:
        if arg == 'manager':
            return True
    return False

def filter_args():
    return [arg for arg in sys.argv[1:] if arg not in ['manager', 'proxy']]

def is_cli():
    return not (is_manager() or is_proxy())

def prepare_venv(root_path):
    venv_path = os.path.join(root_path, 'virtualenv')
    script = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'create_mailpile_venv.sh')
    subprocess.call([script, venv_path])
    mailpile_path = os.path.join(venv_path, 'bin', 'mailpile')
    return venv_path, mailpile_path

def can_use_pidfile(pidfile):
    pidfile.acquire()
    pidfile.release()

def run_manager():
    parser = argparse.ArgumentParser(description='Multipile', )
    parser.add_argument('-r', '--root_path', help='The rootpath for mailpile')
    parser.add_argument('-m', '--mailpile_bin', help='The mailpile executable', default='mailpile')
    parser.add_argument('-b', '--backend', help='the backend to use', default='fork', choices=['fork', 'docker'])
    parser.add_argument('--bind', help="bind to interface. Default 127.0.0.1", default='127.0.0.1')
    parser.add_argument('--sslcert', help='The SSL certficate to use', default=None)
    parser.add_argument('--sslkey', help='The SSL key to use', default=None)
    parser.add_argument('--debug', help='Set log level to debug', default=False, action='store_true')
    parser.add_argument('--daemon', help='start in daemon mode and put process into background', default=False, action='store_true')
    parser.add_argument('--pidfile', help='path for pid file. By default none is created', default=None)
    parser.add_argument('--log-config', help='Provide a python logging config file', default=None)
    parser.add_argument('--leap-provider', '-lp', help='Specify the LEAP provider this dispatcher will connect to', default='localhost')
    parser.add_argument('--leap-provider-ca', '-lpc', dest='leap_provider_ca', help='Specify the LEAP provider CA to use to validate connections', default=True)
    parser.add_argument('--leap-provider-fingerprint', '-lpf', dest='leap_provider_fingerprint', help='Specify the LEAP provider fingerprint to use to validate connections', default=None)
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--mailpile-virtualenv', help='Use specified virtual env for mailpile', default=None)
    group.add_argument('--auto-mailpile-virtualenv', dest='auto_venv', help='Boostrap virtualenv for mailpile', default=False, action='store_true')

    args = parser.parse_args(args=filter_args())

    if args.sslcert:
        ssl_config = SSLConfig(args.sslcert,
                               args.sslkey,
                               latest_available_ssl_version())
    else:
        ssl_config = None

    venv = args.mailpile_virtualenv
    mailpile_bin = args.mailpile_bin

    if args.auto_venv:
        venv, mailpile_bin = prepare_venv(args.root_path)

    if args.root_path is None or not os.path.isdir(args.root_path):
        raise ValueError('root path %s not found!' % args.root_path)

    log_level = logging.DEBUG if args.debug else logging.INFO
    log_config = args.log_config

    provider_ca = args.leap_provider_ca if args.leap_provider_fingerprint is None else False

    manager = DispatcherManager(args.root_path, mailpile_bin, ssl_config, args.leap_provider, mailpile_virtualenv=venv, provider=args.backend, leap_provider_ca=provider_ca, leap_provider_fingerprint=args.leap_provider_fingerprint, bindaddr=args.bind)

    if args.daemon:
        pidfile = TimeoutPIDLockFile(args.pidfile, acquire_timeout=PID_ACQUIRE_TIMEOUT_IN_S) if args.pidfile else None
        can_use_pidfile(pidfile)
        with daemon.DaemonContext(pidfile=pidfile):
            # init logging only after we have spawned the sub process. Otherwise there might be some hickups
            init_logging('manager', level=log_level, config_file=log_config)
            manager.serve_forever()
    else:
        init_logging('manager', level=log_level, config_file=log_config)
        manager.serve_forever()

def run_proxy():
    parser = argparse.ArgumentParser()
    parser.add_argument('-m', '--manager', help='hostname:port of the manager')
    parser.add_argument('--banner', help='banner file to show on login screen', default='_login_screen_message.html')
    parser.add_argument('--bind', help="interface to bind to (default: 127.0.0.1)", default='127.0.0.1')
    parser.add_argument('--sslcert', help='proxy HTTP server SSL certificate', default=None)
    parser.add_argument('--sslkey', help='proxy HTTP server SSL key', default=None)
    parser.add_argument('--fingerprint', help='pin certificate to fingerprint', default=None)
    parser.add_argument('--disable-verifyhostname', help='disable hostname verification; if fingerprint is specified it gets precedence', dest="verify_hostname", action='store_false', default=None)
    parser.add_argument('--debug', help='set log level to debug and auto reload files', default=False, action='store_true')
    parser.add_argument('--log-config', help='provide a python logging config file', default=None)
    parser.add_argument('--daemon', help='start in daemon mode and put process into background', default=False, action='store_true')
    parser.add_argument('--pidfile', help='path for pid file. By default none is created', default=None)

    args = parser.parse_args(args=filter_args())

    manager_hostname, manager_port = args.manager.split(':')
    certfile = args.sslcert if args.sslcert else None
    keyfile = args.sslkey if args.sslcert else None
    manager_cafile = certfile if args.fingerprint is None else False

    log_level = logging.DEBUG if args.debug else logging.INFO
    log_config = args.log_config

    client = PixelatedDispatcherClient(manager_hostname, manager_port, cacert=manager_cafile, fingerprint=args.fingerprint, assert_hostname=args.verify_hostname)
    client.validate_connection()

    dispatcher = DispatcherProxy(client, bindaddr=args.bind, keyfile=keyfile,
                                 certfile=certfile, banner=args.banner, debug=args.debug)

    if args.daemon:
        pidfile = TimeoutPIDLockFile(args.pidfile, acquire_timeout=PID_ACQUIRE_TIMEOUT_IN_S) if args.pidfile else None
        can_use_pidfile(pidfile)
        with daemon.DaemonContext(pidfile=pidfile):
            # init logging only after we have spawned the sub process. Otherwise there might be some hickups
            init_logging('proxy', level=log_level, config_file=log_config)
            dispatcher.serve_forever()
    else:
        init_logging('proxy', level=log_level, config_file=log_config)
        dispatcher.serve_forever()

def run_cli():
    Cli(args=filter_args()).run()

def main():
    if is_manager():
        run_manager()
    elif is_proxy():
        run_proxy()
    else:
        run_cli()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import os
import sys
import errno
import logging

from tldp.ldpcollection import LDPDocumentCollection

from tldp.utils import md5files, stem_and_ext
from tldp.typeguesser import guess, knownextensions

logger = logging.getLogger(__name__)

IGNORABLE_SOURCE = ('index.sgml')

def scansourcedirs(dirnames):
    '''return a dict() of all SourceDocuments discovered in dirnames
    dirnames:  a list of directories containing SourceDocuments.

    scansourcedirs ensures it is operating on the absolute filesystem path for
    each of the source directories.

    If any of the supplied dirnames does not exist as a directory, the function
    will log the missing source directory names and then will raise an IOError
    and quit.

    For each document that it finds in a source directory, it creates a
    SourceDocument entry using the stem name as a key.

    The rules for identifying possible SourceDocuments go as follows.

      - Within any source directory, a source document can consist of a single
        file with an extension or a directory.

      - If the candidate entry is a directory, then, the stem is the full
        directory name, e.g. Masquerading-Simple-HOWTO

      - If the candidate entry is a file, the stem is the filename minus
        extension, e.g. Encrypted-Root-Filesystem-HOWTO

    Because the function accepts (and will scan) many source directories, it
    is possible that there will be stem name collisions.  If it discovers a
    stem collision, SourceCollection will issue a warning and skip the
    duplicated stem(s).  [It also tries to process the source directories and
    candidates in a stable order between runs.]
    '''
    found = dict()
    dirs = [os.path.abspath(x) for x in dirnames]
    results = [os.path.exists(x) for x in dirs]

    if not all(results):
        for result, sdir in zip(results, dirs):
            logger.critical("Source collection dir must already exist: %s",
                            sdir)
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), sdir)

    for sdir in sorted(dirs):
        logger.debug("Scanning for source documents in %s.", sdir)
        for fname in sorted(os.listdir(sdir)):
            candidates = list()
            possible = arg_issourcedoc(os.path.join(sdir, fname))
            if possible:
                candidates.append(SourceDocument(possible))
            else:
                logger.warning("Skipping non-document %s", fname)
                continue
            for candy in candidates:
                if candy.stem in found:
                    dup = found[candy.stem].filename
                    logger.warning("Ignoring duplicate is %s", candy.filename)
                    logger.warning("Existing dup-entry is %s", dup)
                else:
                    found[candy.stem] = candy
    logger.debug("Discovered %s source documents", len(found))
    return found

def arg_issourcedoc(filename):
    filename = os.path.abspath(filename)
    if os.path.isfile(filename):
        if os.path.basename(filename) in IGNORABLE_SOURCE:
            return None
        return filename
    elif os.path.isdir(filename):
        return sourcedoc_fromdir(filename)
    return None

def sourcedoc_fromdir(name):
    candidates = list()
    if not os.path.isdir(name):
        return None
    stem = os.path.basename(name)
    for ext in knownextensions:
        possible = os.path.join(name, stem + ext)
        if os.path.isfile(possible):
            candidates.append(possible)
    if len(candidates) > 1:
        logger.warning("%s multiple document choices in dir %s, bailing....",
                       stem, name)
        raise Exception("multiple document choices in " + name)
    elif len(candidates) == 0:
        return None
    else:
        doc = candidates.pop()
        logger.debug("%s identified main document %s.", stem, doc)
        return doc

class SourceCollection(LDPDocumentCollection):
    '''a dict-like container for SourceDocument objects

    The key in the SourceCollection is the stem name of the document, which
    allows convenient access and guarantees non-collision.

    The use of the stem as a key works conveniently with the
    OutputCollection which uses the same strategy on OutputDirectory.
    '''
    def __init__(self, dirnames=None):
        '''construct a SourceCollection

        delegates most responsibility to function scansourcedirs
        '''
        if dirnames is None:
            return
        self.update(scansourcedirs(dirnames))

class SourceDocument(object):
    '''a class providing a container for each set of source documents
    '''
    def __repr__(self):
        return '<%s:%s (%s)>' % \
               (self.__class__.__name__, self.filename, self.doctype)

    def __init__(self, filename):
        '''construct a SourceDocument

        filename is a required parameter

        The filename is the main (and sometimes sole) document representing
        the source of the LDP HOWTO or Guide.  It is the document that is
        passed by name to be handled by any document processing toolchains
        (see also tldp.doctypes).

        Each instantiation will raise an IOERror if the supplied filename does
        not exist or if the filename isn't a file (symlink is fine, directory
        or fifo is not).

        The remainder of the instantiation will set attributes that are useful
        later in the processing phase, for example, stem, status, enclosing
        directory name and file extension.

        There are two important attributes.  First, the document type guesser
        will try to infer the doctype (from file extension and signature).
        Note that it is not a fatal error if document type cannot be guessed,
        but the document will not be able to be processed.  Second, it is
        useful during the decision-making process to know if any of the source
        files are newer than the output files. Thus, the stat() information
        for every file in the source document directory (or just the single
        source document file) will be collected.
        '''
        self.filename = os.path.abspath(filename)

        if not os.path.exists(self.filename):
            fn = self.filename
            logger.critical("Missing source document: %s", fn)
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), fn)

        if os.path.isdir(self.filename):
            self.filename = sourcedoc_fromdir(self.filename)
        elif os.path.isfile(self.filename):
            pass
        else:
            # -- we did not receive a useable document file or directory name
            self.filename = None

        if self.filename is None:
            fn = filename
            logger.critical("Source document is not a plain file: %s", fn)
            raise ValueError(fn + " not identifiable as a document")

        self.doctype = guess(self.filename)
        self.status = 'source'
        self.output = None
        self.working = None
        self.differing = set()
        self.dirname, self.basename = os.path.split(self.filename)
        self.stem, self.ext = stem_and_ext(self.basename)
        parentbase = os.path.basename(self.dirname)
        logger.debug("%s found source %s", self.stem, self.filename)
        if parentbase == self.stem:
            parentdir = os.path.dirname(self.dirname)
            self.md5sums = md5files(self.dirname, relative=parentdir)
        else:
            self.md5sums = md5files(self.filename, relative=self.dirname)

    def detail(self, widths, verbose, file=sys.stdout):
        '''produce a small tabular output about the document'''
        template = ' '.join(('{s.status:{w.status}}',
                             '{s.doctype.formatname:{w.doctype}}',
                             '{s.stem:{w.stem}}'))
        outstr = template.format(s=self, w=widths)
        print(outstr, file=file)
        if verbose:
            print('         doctype {}'.format(self.doctype), file=file)
            if self.output:
                print('      output dir {}'.format(self.output.dirname),
                      file=file)
            print('     source file {}'.format(self.filename), file=file)
            for why, f in sorted(self.differing):
                fname = os.path.join(self.dirname, f)
                print('  {:7} source {}'.format(why, fname), file=file)
            if self.output:
                for f in sorted(self.output.missing):
                    print('  missing output {}'.format(f), file=file)

#
# -- end of file
<commit_msg>correct the justification of the text<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import os
import sys
import errno
import logging

from tldp.ldpcollection import LDPDocumentCollection

from tldp.utils import md5files, stem_and_ext
from tldp.typeguesser import guess, knownextensions

logger = logging.getLogger(__name__)

IGNORABLE_SOURCE = ('index.sgml')

def scansourcedirs(dirnames):
    '''return a dict() of all SourceDocuments discovered in dirnames
    dirnames:  a list of directories containing SourceDocuments.

    scansourcedirs ensures it is operating on the absolute filesystem path for
    each of the source directories.

    If any of the supplied dirnames does not exist as a directory, the function
    will log the missing source directory names and then will raise an IOError
    and quit.

    For each document that it finds in a source directory, it creates a
    SourceDocument entry using the stem name as a key.

    The rules for identifying possible SourceDocuments go as follows.

      - Within any source directory, a source document can consist of a single
        file with an extension or a directory.

      - If the candidate entry is a directory, then, the stem is the full
        directory name, e.g. Masquerading-Simple-HOWTO

      - If the candidate entry is a file, the stem is the filename minus
        extension, e.g. Encrypted-Root-Filesystem-HOWTO

    Because the function accepts (and will scan) many source directories, it
    is possible that there will be stem name collisions.  If it discovers a
    stem collision, SourceCollection will issue a warning and skip the
    duplicated stem(s).  [It also tries to process the source directories and
    candidates in a stable order between runs.]
    '''
    found = dict()
    dirs = [os.path.abspath(x) for x in dirnames]
    results = [os.path.exists(x) for x in dirs]

    if not all(results):
        for result, sdir in zip(results, dirs):
            logger.critical("Source collection dir must already exist: %s",
                            sdir)
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), sdir)

    for sdir in sorted(dirs):
        logger.debug("Scanning for source documents in %s.", sdir)
        for fname in sorted(os.listdir(sdir)):
            candidates = list()
            possible = arg_issourcedoc(os.path.join(sdir, fname))
            if possible:
                candidates.append(SourceDocument(possible))
            else:
                logger.warning("Skipping non-document %s", fname)
                continue
            for candy in candidates:
                if candy.stem in found:
                    dup = found[candy.stem].filename
                    logger.warning("Ignoring duplicate is %s", candy.filename)
                    logger.warning("Existing dup-entry is %s", dup)
                else:
                    found[candy.stem] = candy
    logger.debug("Discovered %s source documents", len(found))
    return found

def arg_issourcedoc(filename):
    filename = os.path.abspath(filename)
    if os.path.isfile(filename):
        if os.path.basename(filename) in IGNORABLE_SOURCE:
            return None
        return filename
    elif os.path.isdir(filename):
        return sourcedoc_fromdir(filename)
    return None

def sourcedoc_fromdir(name):
    candidates = list()
    if not os.path.isdir(name):
        return None
    stem = os.path.basename(name)
    for ext in knownextensions:
        possible = os.path.join(name, stem + ext)
        if os.path.isfile(possible):
            candidates.append(possible)
    if len(candidates) > 1:
        logger.warning("%s multiple document choices in dir %s, bailing....",
                       stem, name)
        raise Exception("multiple document choices in " + name)
    elif len(candidates) == 0:
        return None
    else:
        doc = candidates.pop()
        logger.debug("%s identified main document %s.", stem, doc)
        return doc

class SourceCollection(LDPDocumentCollection):
    '''a dict-like container for SourceDocument objects

    The key in the SourceCollection is the stem name of the document, which
    allows convenient access and guarantees non-collision.

    The use of the stem as a key works conveniently with the
    OutputCollection which uses the same strategy on OutputDirectory.
    '''
    def __init__(self, dirnames=None):
        '''construct a SourceCollection

        delegates most responsibility to function scansourcedirs
        '''
        if dirnames is None:
            return
        self.update(scansourcedirs(dirnames))

class SourceDocument(object):
    '''a class providing a container for each set of source documents
    '''
    def __repr__(self):
        return '<%s:%s (%s)>' % \
               (self.__class__.__name__, self.filename, self.doctype)

    def __init__(self, filename):
        '''construct a SourceDocument

        filename is a required parameter

        The filename is the main (and sometimes sole) document representing
        the source of the LDP HOWTO or Guide.  It is the document that is
        passed by name to be handled by any document processing toolchains
        (see also tldp.doctypes).

        Each instantiation will raise an IOERror if the supplied filename does
        not exist or if the filename isn't a file (symlink is fine, directory
        or fifo is not).

        The remainder of the instantiation will set attributes that are useful
        later in the processing phase, for example, stem, status, enclosing
        directory name and file extension.

        There are two important attributes.  First, the document type guesser
        will try to infer the doctype (from file extension and signature).
        Note that it is not a fatal error if document type cannot be guessed,
        but the document will not be able to be processed.  Second, it is
        useful during the decision-making process to know if any of the source
        files are newer than the output files. Thus, the stat() information
        for every file in the source document directory (or just the single
        source document file) will be collected.
        '''
        self.filename = os.path.abspath(filename)

        if not os.path.exists(self.filename):
            fn = self.filename
            logger.critical("Missing source document: %s", fn)
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), fn)

        if os.path.isdir(self.filename):
            self.filename = sourcedoc_fromdir(self.filename)
        elif os.path.isfile(self.filename):
            pass
        else:
            # -- we did not receive a useable document file or directory name
            self.filename = None

        if self.filename is None:
            fn = filename
            logger.critical("Source document is not a plain file: %s", fn)
            raise ValueError(fn + " not identifiable as a document")

        self.doctype = guess(self.filename)
        self.status = 'source'
        self.output = None
        self.working = None
        self.differing = set()
        self.dirname, self.basename = os.path.split(self.filename)
        self.stem, self.ext = stem_and_ext(self.basename)
        parentbase = os.path.basename(self.dirname)
        logger.debug("%s found source %s", self.stem, self.filename)
        if parentbase == self.stem:
            parentdir = os.path.dirname(self.dirname)
            self.md5sums = md5files(self.dirname, relative=parentdir)
        else:
            self.md5sums = md5files(self.filename, relative=self.dirname)

    def detail(self, widths, verbose, file=sys.stdout):
        '''produce a small tabular output about the document'''
        template = ' '.join(('{s.status:{w.status}}',
                             '{s.doctype.formatname:{w.doctype}}',
                             '{s.stem:{w.stem}}'))
        outstr = template.format(s=self, w=widths)
        print(outstr, file=file)
        if verbose:
            print('         doctype {}'.format(self.doctype), file=file)
            if self.output:
                print('      output dir {}'.format(self.output.dirname),
                      file=file)
            print('     source file {}'.format(self.filename), file=file)
            for why, f in sorted(self.differing):
                fname = os.path.join(self.dirname, f)
                print('  {:>7} source {}'.format(why, fname), file=file)
            if self.output:
                for f in sorted(self.output.missing):
                    print('  missing output {}'.format(f), file=file)

#
# -- end of file
<|endoftext|>


<commit_before>import logging

from paste.deploy.converters import asbool

from pylons import request, response
from pylons import tmpl_context as c
from pylons.controllers import WSGIController
from pylons.controllers.util import abort, redirect_to
from pylons.decorators import jsonify
from pylons.templating import render_mako as render

from sqlalchemy.orm.exc import NoResultFound

import mako

import simplejson as json

from restler.decorators import privileged
"""Set `privileged` BEFORE importing this module, if needed."""

log = logging.getLogger(__name__)

TemplateNotFoundExceptions = (mako.exceptions.TopLevelLookupException,)

class Controller(WSGIController):

    entity = None
    """Entity class assocated with this controller."""

    filter_params = {}
    """Request param names with defaults, for filtering collections."""

    filters = []
    """SQLAlchemy filters--anything that could be an arg to `q.filter`."""

    default_format = 'json'

    def __call__(self, environ, start_response):
        try:
            return super(Controller, self).__call__(environ, start_response)
        finally:
            log.debug('Clearing database session...')
            self.db_session.remove()

    def __before__(self, *args, **kwargs):
        route_info = request.environ['pylons.routes_dict']
        self.controller = route_info['controller']
        self.action = route_info['action']
        self.member_name = self.entity.member_name
        self.member_title = self.entity.member_title
        self.collection_name = self.entity.collection_name
        self.collection_title = self.entity.collection_title
        self.format = kwargs.get('format',
            request.params.get('format', self.default_format))
        self._init_properties()
        log.debug('Action: %s' % self.action)

    def _get_db_session(self):
        """Database session factory.

        The default implementation here assumes that subclasses have a public
        :meth:`get_db_session` method. The session returned from that method
        is cached per :class:`Controller` instance.

        """
        try:
            self._db_session
        except AttributeError:
            self._db_session = self.get_db_session()
        return self._db_session
    def _set_db_session(self, db_session):
        self._db_session = db_session
    db_session = property(_get_db_session, _set_db_session)

    def get_db_session(self):
        raise NotImplementedError

    def index(self):
        self.set_collection()
        return self._render()

    def show(self, id):
        self.set_member(id)
        return self._render()

    def new(self):
        self.set_member()
        return self._render()

    def edit(self, id):
        self.set_member(id)
        return self._render()

    @privileged
    def create(self):
        self.set_member()
        self._update_member_with_params()
        self.db_session.add(self.member)
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_member()

    @privileged
    def update(self, id):
        self.set_member(id)
        self._update_member_with_params()
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_member()

    @privileged
    def delete(self, id):
        self.set_member(id)
        self.db_session.delete(self.member)
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_collection()

    def set_member(self, id=None):
        if id is None:
            member = self.entity()
        else:
            member = self.get_entity_or_404(id)
        self.member = member

    # TODO: Allow pagination for collection methods. If pagination params
    # aren't set then we'd just fall through to returning the entire
    # collection OR if the collection is huge, we might use defaults.

    def set_collection(self):
        params = request.params
        kwargs = {}
        if self.filters:
            kwargs['filters'] = self.filters
        for name in self.filter_params:
            val = params.get(name, '').strip()
            if val == '':
                val = self.filter_params[name]
            kwargs[name] = val
        self.collection = self.entity.all(**kwargs) or abort(404)

    def get_entity_or_404(self, id):
        # TODO: Allow get by alt pk
        entity = self.db_session.query(self.entity).get(id) or abort(404)
        return entity

    def _update_member_with_params(self):
        params = request.params
        for name in params:
            val = self._convert_param_for_update(name, params[name])
            setattr(self.member, name, val)

    def _convert_param_for_update(self, name, val):
        return val

    def _redirect_to_member(self):
        redirect_to(self.member_name, id=self.member.id)

    def _redirect_to_collection(self):
        redirect_to(self.collection_name)

    def _render(self, *args, **kwargs):
        format = kwargs.get('format', self.format)
        log.debug('Output format: %s' % format)
        kwargs['format'] = format
        render = getattr(self, '_render_%s' % format, self._render_template)
        log.debug('Render method: %s *%s **%s' % (render.__name__, args, kwargs))
        response.status = kwargs.pop('code', 200)
        return render(*args, **kwargs)

    def _render_template(
        self, controller=None, action=None, format=None, namespace=None):
        """By default, render template /{controller}/{action}.{format}.

        This is the default rendering method, used when a specific
        ``render_<format>`` method doesn't exist. Templates can be any
        text-based template--HTML, text, XML, etc.

        Typically, args are set in the environment (on ``c``), except
        ``namespace``. Args passed explicitly will override args set in the
        env. If ``namespace`` is given, the template rendered will be
        /namespace/{controller}/{action}.{format}.

        """
        template = '/%%s/%s.%s' % (action or self.action, format or self.format)
        try:
            template_name = template % (controller or self.controller)
            return render(template_name)
        except TemplateNotFoundExceptions:
            if namespace is not None:
                template = '%s/%s' % (namespace, template)
            template_name = template % 'default'
            return render(template_name)
        finally:
            log.debug('(_render) template: %s' % template_name)

    def _render_json(self, block=None, **kwargs):
        """Render a JSON response from simplified ``member``s."""
        obj = self._get_json_object(block=block)
        return self._render_object_as_json(obj)

    @jsonify
    def _render_object_as_json(self, obj):
        """Render an object in JSON format with correct content type.

        ``obj`` must be JSONifiable by the (simple)json module.

        The final output of this method is returned by the Pylons ``jsonify``
        decorator, which creates a proper JSON response with the correct
        content type.

        """
        return obj

    def _get_json_object(self, wrap=True, block=None):
        """Get JSON object for current request.

        ``wrap``
             If set, the output will be wrapped in a dict.

        ``block``
            Can be passed to modify or wrap the object before JSONifying it.
            In this case the wrapping discussed above under ``obj`` won't
            happen.

        """
        fields = self.fields
        if self.collection is not None:
            log.debug('Rendering collection')
            if fields:
                simplifier = self.entity.simplify_object
                obj = []
                for member in self.collection:
                    result = {}
                    for name, as_name in fields:
                        result[as_name] = simplifier(getattr(member, name))
                    obj.append(result)
            else:
                obj = self.entity.to_simple_collection(self.collection)
            result_count = len(obj)
        elif self.member is not None:
            log.debug('Rendering member')
            if fields:
                simplifier = self.entity.simplify_object
                obj = {}
                for name, as_name in fields:
                    obj[as_name] = simplifier(getattr(self.member, name))
            else:
                obj = [self.member.to_simple_object()]
            result_count = 1
        else:
            log.debug('Neither collection nor member was set.')
            obj = None
            result_count = 0
        # Wrap ``obj`` (usually)
        if self.wrap:
            total_count = self.db_session.query(self.entity).count()
            obj = dict(
                response=dict(
                    results=obj,
                    result_count=result_count,
                    total_count=total_count
                )
            )
        # Further modify ``obj`` if ``block`` given
        if block is not None:
            obj = block(obj)
        return obj

    @property
    def fields(self):
        """Fields to include in response, with optional field name mapping.

        The ``fields`` request parameter should be a JSON list. Each item in
        the list may be either string or a dict. A string is used to
        indicate that the field's column name should be used in the response.
        A dict is used to map the field's column name to a different name
        in the response. In this case, the dict must contain a ``name`` key
        with the field's column name and a ``mapping`` key with the name
        desired in the reponse.

        Example::

            ?fields=["id", {"name": "description", "mapping": "text"}]

            In this example, the ``id`` column is not mapped to a new name
            while the ``description`` column will be referred to as ``text``
            in the response.

        """
        fields = request.params.get('fields', None)
        if fields:
            fields = json.loads(fields)
            mapped_fields = []
            if isinstance(fields, list):
                for item in fields:
                    if isinstance(item, basestring):
                        mapped_fields.append((item, item))
                    elif isinstance(item, dict):
                        mapped_fields.append((item['name'], item['mapping']))
            elif isinstance(fields, dict):
                # Support legacy dict format
                mapped_fields = fields.items()
            return mapped_fields
        return None

    def _get_wrap(self):
        try:
            self._wrap
        except AttributeError:
            self._set_wrap()
        return self._wrap
    def _set_wrap(self, value=None):
        """Set whether to wrap a template in its parent template.

        If ``value`` is given, use that value. If ``wrap`` is set in
        ``request.params``, convert its value to ``bool`` and use that
        value. Otherwise, set ``self.wrap`` to True.

        ``value``
            ``bool`` indicating whether or not a template should be
            wrapped in its inherited templates. ``wrap`` gets passed along
            to the template, which can decide to do whatever it wants with
            it.

        """
        if value is None:
            wrap = request.params.get('wrap', 'true')
            self._wrap = asbool(wrap)
        else:
            self._wrap = value
        c.wrap = self._wrap
    wrap = property(_get_wrap, _set_wrap)

    def __setattr__(self, name, value):
        """Set attribute on both ``self`` and ``c``."""
        attr = getattr(self.__class__, name, None)
        if isinstance(attr, property) and attr.fset is not None:
            # I.e., just call the property's _set method
            super(Controller, self).__setattr__(name, value)
        else:
            self._set_property([name], value)

    def _set_property(self, names, value):
        """Set attributes on both ``self`` and ``c``."""
        for name in names:
            self.__dict__[name] = value
            # Don't put "private" names in the template context
            if not name.startswith('_'):
                setattr(c, name, value)

    def _p_get_collection(self):
        return self.__dict__.get('collection', None)
    def _p_set_collection(self, collection):
        self._set_property(
            ['collection', self.collection_name], collection)
    collection = property(_p_get_collection, _p_set_collection)

    def _p_get_member(self):
        return self.__dict__.get('member', None)
    def _p_set_member(self, member):
        self._set_property(['member', self.member_name], member)
    member = property(_p_get_member, _p_set_member)

    def _init_properties(self):
        cls = self.__class__
        setattr(cls, self.collection_name, cls.collection)
        setattr(cls, self.member_name, cls.member)
<commit_msg>Use more robust method for redirecting to a member or the collection. Previous method would break for namespaced controllers. Added `member` keyword arg to :meth:`_redirect_to_member` to allow override of redirect to `self.member`.<commit_after>import logging

from paste.deploy.converters import asbool

from pylons import request, response
from pylons import tmpl_context as c
from pylons.controllers import WSGIController
from pylons.controllers.util import abort, redirect_to
from pylons.decorators import jsonify
from pylons.templating import render_mako as render

from sqlalchemy.orm.exc import NoResultFound

import mako

import simplejson as json

from restler.decorators import privileged
"""Set `privileged` BEFORE importing this module, if needed."""

log = logging.getLogger(__name__)

TemplateNotFoundExceptions = (mako.exceptions.TopLevelLookupException,)

class Controller(WSGIController):

    entity = None
    """Entity class assocated with this controller."""

    filter_params = {}
    """Request param names with defaults, for filtering collections."""

    filters = []
    """SQLAlchemy filters--anything that can be an arg to `query().filter`."""

    default_format = 'json'

    def __call__(self, environ, start_response):
        try:
            return super(Controller, self).__call__(environ, start_response)
        finally:
            log.debug('Clearing database session...')
            self.db_session.remove()

    def __before__(self, *args, **kwargs):
        route_info = request.environ['pylons.routes_dict']
        self.controller = route_info['controller']
        self.action = route_info['action']
        self.member_name = self.entity.member_name
        self.member_title = self.entity.member_title
        self.collection_name = self.entity.collection_name
        self.collection_title = self.entity.collection_title
        self.format = kwargs.get('format',
            request.params.get('format', self.default_format))
        self._init_properties()
        log.debug('Action: %s' % self.action)

    def _get_db_session(self):
        """Database session factory.

        The default implementation here assumes that subclasses have a public
        :meth:`get_db_session` method. The session returned from that method
        is cached per :class:`Controller` instance.

        """
        try:
            self._db_session
        except AttributeError:
            self._db_session = self.get_db_session()
        return self._db_session
    def _set_db_session(self, db_session):
        self._db_session = db_session
    db_session = property(_get_db_session, _set_db_session)

    def get_db_session(self):
        raise NotImplementedError

    def index(self):
        self.set_collection()
        return self._render()

    def show(self, id):
        self.set_member(id)
        return self._render()

    def new(self):
        self.set_member()
        return self._render()

    def edit(self, id):
        self.set_member(id)
        return self._render()

    @privileged
    def create(self):
        self.set_member()
        self._update_member_with_params()
        self.db_session.add(self.member)
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_member()

    @privileged
    def update(self, id):
        self.set_member(id)
        self._update_member_with_params()
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_member()

    @privileged
    def delete(self, id):
        self.set_member(id)
        self.db_session.delete(self.member)
        self.db_session.flush()
        self.db_session.commit()
        self._redirect_to_collection()

    def set_member(self, id=None):
        if id is None:
            member = self.entity()
        else:
            member = self.get_entity_or_404(id)
        self.member = member

    # TODO: Allow pagination for collection methods. If pagination params
    # aren't set then we'd just fall through to returning the entire
    # collection OR if the collection is huge, we might use defaults.

    def set_collection(self):
        params = request.params
        kwargs = {}
        if self.filters:
            kwargs['filters'] = self.filters
        for name in self.filter_params:
            val = params.get(name, '').strip()
            if val == '':
                val = self.filter_params[name]
            kwargs[name] = val
        self.collection = self.entity.all(**kwargs) or abort(404)

    def get_entity_or_404(self, id):
        # TODO: Allow get by alt pk
        entity = self.db_session.query(self.entity).get(id) or abort(404)
        return entity

    def _update_member_with_params(self):
        params = request.params
        for name in params:
            val = self._convert_param_for_update(name, params[name])
            setattr(self.member, name, val)

    def _convert_param_for_update(self, name, val):
        return val

    def _redirect_to_member(self, member=None):
        member = self.member if member is None else member
        redirect_to(
            controller=self.controller, action='show', id=member.id,
            format=self.format)

    def _redirect_to_collection(self):
        redirect_to(
            controller=self.controller, action='index', format=self.format)

    def _render(self, *args, **kwargs):
        format = kwargs.get('format', self.format)
        log.debug('Output format: %s' % format)
        kwargs['format'] = format
        render = getattr(self, '_render_%s' % format, self._render_template)
        log.debug('Render method: %s *%s **%s' % (render.__name__, args, kwargs))
        response.status = kwargs.pop('code', 200)
        return render(*args, **kwargs)

    def _render_template(
        self, controller=None, action=None, format=None, namespace=None):
        """By default, render template /{controller}/{action}.{format}.

        This is the default rendering method, used when a specific
        ``render_<format>`` method doesn't exist. Templates can be any
        text-based template--HTML, text, XML, etc.

        Typically, args are set in the environment (on ``c``), except
        ``namespace``. Args passed explicitly will override args set in the
        env. If ``namespace`` is given, the template rendered will be
        /namespace/{controller}/{action}.{format}.

        """
        template = '/%%s/%s.%s' % (action or self.action, format or self.format)
        try:
            template_name = template % (controller or self.controller)
            return render(template_name)
        except TemplateNotFoundExceptions:
            if namespace is not None:
                template = '%s/%s' % (namespace, template)
            template_name = template % 'default'
            return render(template_name)
        finally:
            log.debug('(_render) template: %s' % template_name)

    def _render_json(self, block=None, **kwargs):
        """Render a JSON response from simplified ``member``s."""
        obj = self._get_json_object(block=block)
        return self._render_object_as_json(obj)

    @jsonify
    def _render_object_as_json(self, obj):
        """Render an object in JSON format with correct content type.

        ``obj`` must be JSONifiable by the (simple)json module.

        The final output of this method is returned by the Pylons ``jsonify``
        decorator, which creates a proper JSON response with the correct
        content type.

        """
        return obj

    def _get_json_object(self, wrap=True, block=None):
        """Get JSON object for current request.

        ``wrap``
             If set, the output will be wrapped in a dict.

        ``block``
            Can be passed to modify or wrap the object before JSONifying it.
            In this case the wrapping discussed above under ``obj`` won't
            happen.

        """
        fields = self.fields
        if self.collection is not None:
            log.debug('Rendering collection')
            if fields:
                simplifier = self.entity.simplify_object
                obj = []
                for member in self.collection:
                    result = {}
                    for name, as_name in fields:
                        result[as_name] = simplifier(getattr(member, name))
                    obj.append(result)
            else:
                obj = self.entity.to_simple_collection(self.collection)
            result_count = len(obj)
        elif self.member is not None:
            log.debug('Rendering member')
            if fields:
                simplifier = self.entity.simplify_object
                obj = {}
                for name, as_name in fields:
                    obj[as_name] = simplifier(getattr(self.member, name))
            else:
                obj = [self.member.to_simple_object()]
            result_count = 1
        else:
            log.debug('Neither collection nor member was set.')
            obj = None
            result_count = 0
        # Wrap ``obj`` (usually)
        if self.wrap:
            total_count = self.db_session.query(self.entity).count()
            obj = dict(
                response=dict(
                    results=obj,
                    result_count=result_count,
                    total_count=total_count
                )
            )
        # Further modify ``obj`` if ``block`` given
        if block is not None:
            obj = block(obj)
        return obj

    @property
    def fields(self):
        """Fields to include in response, with optional field name mapping.

        The ``fields`` request parameter should be a JSON list. Each item in
        the list may be either string or a dict. A string is used to
        indicate that the field's column name should be used in the response.
        A dict is used to map the field's column name to a different name
        in the response. In this case, the dict must contain a ``name`` key
        with the field's column name and a ``mapping`` key with the name
        desired in the reponse.

        Example::

            ?fields=["id", {"name": "description", "mapping": "text"}]

            In this example, the ``id`` column is not mapped to a new name
            while the ``description`` column will be referred to as ``text``
            in the response.

        """
        fields = request.params.get('fields', None)
        if fields:
            fields = json.loads(fields)
            mapped_fields = []
            if isinstance(fields, list):
                for item in fields:
                    if isinstance(item, basestring):
                        mapped_fields.append((item, item))
                    elif isinstance(item, dict):
                        mapped_fields.append((item['name'], item['mapping']))
            elif isinstance(fields, dict):
                # Support legacy dict format
                mapped_fields = fields.items()
            return mapped_fields
        return None

    def _get_wrap(self):
        try:
            self._wrap
        except AttributeError:
            self._set_wrap()
        return self._wrap
    def _set_wrap(self, value=None):
        """Set whether to wrap a template in its parent template.

        If ``value`` is given, use that value. If ``wrap`` is set in
        ``request.params``, convert its value to ``bool`` and use that
        value. Otherwise, set ``self.wrap`` to True.

        ``value``
            ``bool`` indicating whether or not a template should be
            wrapped in its inherited templates. ``wrap`` gets passed along
            to the template, which can decide to do whatever it wants with
            it.

        """
        if value is None:
            wrap = request.params.get('wrap', 'true')
            self._wrap = asbool(wrap)
        else:
            self._wrap = value
        c.wrap = self._wrap
    wrap = property(_get_wrap, _set_wrap)

    def __setattr__(self, name, value):
        """Set attribute on both ``self`` and ``c``."""
        attr = getattr(self.__class__, name, None)
        if isinstance(attr, property) and attr.fset is not None:
            # I.e., just call the property's _set method
            super(Controller, self).__setattr__(name, value)
        else:
            self._set_property([name], value)

    def _set_property(self, names, value):
        """Set attributes on both ``self`` and ``c``."""
        for name in names:
            self.__dict__[name] = value
            # Don't put "private" names in the template context
            if not name.startswith('_'):
                setattr(c, name, value)

    def _p_get_collection(self):
        return self.__dict__.get('collection', None)
    def _p_set_collection(self, collection):
        self._set_property(
            ['collection', self.collection_name], collection)
    collection = property(_p_get_collection, _p_set_collection)

    def _p_get_member(self):
        return self.__dict__.get('member', None)
    def _p_set_member(self, member):
        self._set_property(['member', self.member_name], member)
    member = property(_p_get_member, _p_set_member)

    def _init_properties(self):
        cls = self.__class__
        setattr(cls, self.collection_name, cls.collection)
        setattr(cls, self.member_name, cls.member)
<|endoftext|>