<commit_before>import numpy as np
from openpathsampling.snapshot import Snapshot, Configuration, Momentum
from object_storage import ObjectStore
from openpathsampling.trajectory import Trajectory
import simtk.unit as u

class SnapshotStore(ObjectStore):
    """
    An ObjectStore for Snapshots in netCDF files.
    """

    def __init__(self):
        super(SnapshotStore, self).__init__(Snapshot, json=False)

    def load(self, idx=None):
        '''
        Load a snapshot from the storage.

        Parameters
        ----------
        idx : int
            the integer index of the snapshot to be loaded

        Returns
        -------
        snapshot : Snapshot
            the loaded snapshot instance
        '''

        configuration = self.vars['configuration'][idx]
        momentum = self.vars['momentum'][idx]
        momentum_reversed = self.vars['momentum_reversed'][idx]
        reversed_idx = self.vars['reversed'][idx]

        snapshot = Snapshot(
            configuration=configuration,
            momentum=momentum,
            is_reversed=momentum_reversed,
            reversed_copy=None
        )
        snapshot_reversed = Snapshot(
            configuration=configuration,
            momentum=momentum,
            is_reversed=not momentum_reversed,
            reversed_copy=None
        )

        snapshot._reversed = snapshot_reversed
        snapshot_reversed._reversed = snapshot

        # fix caching!
        snapshot_reversed.idx[self.storage] = reversed_idx
        self.cache[reversed_idx] = snapshot_reversed

        return snapshot

    def all(self):
        """
        Return a trajectory consisting of all (unordered) frames in the storage.

        Notes
        -----
        If you are interested in collectivevariables this is faster since it does not
        load the snapshots. Otherwise storage.snapshots is fine to get an
        iterator. Both should should be about the same speed.
        """
        #TODO: Might think about replacing the iterator with this since it is
        # faster for collectivevariables
        return Trajectory([ (self, idx) for idx in range(len(self)) ])

    def save(self, snapshot, idx=None):
        """
        Add the current state of the snapshot in the database.

        Parameters
        ----------
        snapshot : Snapshot()
            the snapshot to be saved
        idx : int or None
            if idx is not None the index will be used for saving in the storage.
            This might overwrite already existing trajectories!

        Notes
        -----
        This also saves all contained frames in the snapshot if not done yet.
        A single Snapshot object can only be saved once!
        """

        self.vars['configuration'][idx] = snapshot.configuration
        self.vars['momentum'][idx] = snapshot.momentum
        self.vars['reversed'][idx] = snapshot._reversed

        self.vars['momentum_reversed'][idx] = snapshot.is_reversed

    def configuration_idx(self, idx):
        '''
        Load snapshot index for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        list of int
            configuration indices
        '''
        return self.vars['configuration'][idx]

    def momentum_idx(self, idx):
        '''
        Load momentum index for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        int
            momentum indices
        '''
        return self.vars['momentum'][idx]

    def reversed_idx(self, idx):
        '''
        Load snapshot index for the reversed snapshot with ID 'idx'
        from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        int
            reversed snapshot indices
        '''
        return self.vars['reversed'][idx]

    def momentum_reversed(self, idx):
        '''
        Load reversed boolean for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        boolean
            boolean if the momentum of the snapshot is reversed
        '''
        return self.vars['momentum_reversed'][idx]

    def _init(self):
        '''
        Initializes the associated storage to index configuration_indices in it
        '''
        super(SnapshotStore, self)._init()

        self.init_variable('configuration', 'obj.configurations',
                description="the snapshot index (0..n_configuration-1) of snapshot '{idx}'.",
                chunksizes=(1, )
        )

        self.init_variable('momentum', 'obj.momenta',
                description="the snapshot index (0..n_momentum-1) 'frame' of snapshot '{idx}'.",
                chunksizes=(1, )
                )

        self.init_variable('momentum_reversed', 'bool', chunksizes=(1, ))

        self.init_variable('reversed', 'obj.snapshots',
                description="the idx of the reversed snapshot index (0..n_snapshot-1) 'snapshot' of snapshot '{idx}'.",
                chunksizes=(1, )
                )

#=============================================================================================
# COLLECTIVE VARIABLE UTILITY FUNCTIONS
#=============================================================================================

    @property
    def op_configuration_idx(self):
        """
        Returns aa function that returns for an object of this storage the idx

        Returns
        -------
        function
            the function that returns the idx of the configuration
        """
        def idx(obj):
            return obj.configuration.idx[self.storage]

        return idx

    @property
    def op_momentum_idx(self):
        """
        Returns aa function that returns for an object of this storage the idx

        Returns
        -------
        function
            the function that returns the idx of the configuration

        """
        def idx(obj):
            return obj.momenta.idx[self.storage]

        return idx

class MomentumStore(ObjectStore):
    """
    An ObjectStore for Momenta. Allows to store Momentum() instances in a netcdf file.
    """

    def __init__(self):
        super(MomentumStore, self).__init__(Momentum, json=False, load_partial=True)

        # attach delayed loaders
        self.set_variable_partial_loading('velocities')
        self.set_variable_partial_loading('kinetic_energy')

    def save(self, momentum, idx = None):
        """
        Save velocities and kinetic energies.

        Parameters
        ----------
        momentum : Momentum()
            the actual Momentum() instance to be saved.
        idx : int or None
            if not None `idx`is used as the index to index the Momentum()
            instance. Might overwrite existing Momentum in the database.
        """
        if momentum.velocities is not None:
            self.vars['velocities'][idx,:,:] = momentum.velocities
        else:
            print 'ERROR : Momentum should not be empty'

        if momentum.kinetic_energy is not None:
            self.vars['kinetic_energy'][idx] = momentum.kinetic_energy
        else:
            # TODO: No kinetic energy is not yet supported
            print 'Think about how to handle this. It should only be None if loaded lazy and in this case it will never be saved.'

        # Force sync to disk to avoid data loss.
        # storage.sync()

    def load(self, idx):
        '''
        Load a momentum from the storage

        Parameters
        ----------
        idx : int
            index of the momentum in the database 'idx' > 0

        Returns
        -------
        Momentum()
            the loaded momentum instance
        '''

        velocities = self.vars['velocities'][idx]
        kinetic_energy = self.vars['kinetic_energy'][idx]

        momentum = Momentum(velocities=velocities, kinetic_energy=kinetic_energy)

        return momentum

    def load_empty(self, idx):
        momentum = Momentum()
        del momentum.velocities
        del momentum.kinetic_energy
        return momentum

    def velocities_as_numpy(self, frame_indices=None, atom_indices=None):
        """
        Return a block of stored velocities in the database as a numpy array.

        Parameters
        ----------
        frame_indices : list of int or None
            the indices of Momentum objects to be retrieved from the database.
            If `None` is specified then all indices are returned!
        atom_indices : list of int of None
            if not None only the specified atom_indices are returned. Might
            speed up reading a lot.
        """

        if frame_indices is None:
            frame_indices = slice(None)

        if atom_indices is None:
            atom_indices = slice(None)

        return self.variables['velocities'][frame_indices,atom_indices,:].astype(np.float32).copy()

    def velocities_as_array(self, frame_indices=None, atom_indices=None):
        '''
        Returns a numpy array consisting of all velocities at the given indices

        Parameters
        ----------
        frame_indices : list of int
            momenta indices to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        return self.velocities_as_numpy(frame_indices, atom_indices)

    def _init(self):
        '''
        Initializes the associated storage to index momentums in it
        '''

        super(MomentumStore, self)._init()

        n_atoms = self.storage.n_atoms
        n_spatial = self.storage.n_spatial

        self.init_variable('velocities', 'float',
                dimensions=(self.prefix, 'atom', 'spatial'),
                units=self.dimension_units['velocity'],
                description="the velocity of atom 'atom' in dimension " +
                            "'coordinate' of momentum 'momentum'.",
                chunksizes=(1, n_atoms, n_spatial))

        self.init_variable('kinetic_energy', 'float', self.prefix,
                self.dimension_units['energy'],
                chunksizes=(1, ))

class ConfigurationStore(ObjectStore):
    def __init__(self):
        super(ConfigurationStore, self).__init__(Configuration, json=False, load_partial=True)

        # attach delayed loaders
        self.set_variable_partial_loading('coordinates')
        self.set_variable_partial_loading('box_vectors')
        self.set_variable_partial_loading('potential_energy')

    def save(self, configuration, idx = None):
        # Store configuration.
        self.vars['coordinates'][idx] = configuration.coordinates

        if configuration.potential_energy is not None:
            self.vars['potential'][idx] = configuration.potential_energy

        if configuration.box_vectors is not None:
            self.vars['box_vectors'][idx] = configuration.box_vectors

    def get(self, indices):
        return [ self.load(idx) for idx in indices ]

    def load(self, idx):
        coordinates = self.vars["coordinates"][idx]
        box_vectors = self.vars["box_vectors"][idx]
        potential_energy = self.vars["potential"][idx]

        configuration = Configuration(coordinates=coordinates, box_vectors = box_vectors, potential_energy=potential_energy)
        configuration.topology = self.storage.topology

        return configuration

    def load_empty(self, idx):
        """
        Loading function for partial loading. Constructs an empty Configuration
        object.

        Parameters
        ----------
        idx : int
            the integer index of the configuration to be loaded

        Returns
        -------
        Configuration
            an empty configuration object
        """
        configuration = Configuration()
        configuration.topology = self.storage.topology

        # if these still exist they will not be loaded using __getattr__
        del configuration.coordinates
        del configuration.box_vectors
        del configuration.potential_energy

        return configuration

    def coordinates_as_numpy(self, frame_indices=None, atom_indices=None):
        """
        Return the atom coordinates in the storage for given frame indices
        and atoms

        Parameters
        ----------
        frame_indices : list of int or None
            the frame indices to be included. If None all frames are returned
        atom_indices : list of int or None
            the atom indices to be included. If None all atoms are returned

        Returns
        -------
        numpy.array, shape=(n_frames, n_atoms)
            the array of atom coordinates in a float32 numpy array

        """
        if frame_indices is None:
            frame_indices = slice(None)

        if atom_indices is None:
            atom_indices = slice(None)

        return self.storage.variables[self.prefix + '_coordinates'][frame_indices,atom_indices,:].astype(np.float32).copy()

    def coordinates_as_array(self, frame_indices=None, atom_indices=None):
        '''
        Returns a numpy array consisting of all coordinates at the given indices

        Parameters
        ----------
        frame_indices : list of int
            configuration indices to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        return self.coordinates_as_numpy(frame_indices, atom_indices)

    def snapshot_coordinates_as_array(self, idx, atom_indices=None):
        '''
        Returns a numpy array consisting of all coordinates of a snapshot

        Parameters
        ----------
        idx : int
            index of the snapshot to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        frame_indices = self.configuration_indices(idx)
        return self.coordinates_as_array(frame_indices, atom_indices)

    def _init(self):
        super(ConfigurationStore, self)._init()
        n_atoms = self.storage.n_atoms
        n_spatial = self.storage.n_spatial

        self.init_variable('coordinates', 'float',
                (self.prefix, 'atom','spatial'), self.dimension_units['length'],
                description="coordinate of atom '{ix[1]}' in dimension " +
                            "'{ix[2]}' of configuration '{ix[0]}'.",
                chunksizes=(1,n_atoms,n_spatial))

        self.init_variable('box_vectors', 'float',
                (self.prefix, 'spatial', 'spatial'),
                self.dimension_units['length'],
                chunksizes=(1,n_spatial,n_spatial))

        self.init_variable('potential', 'float',
                self.prefix,
                self.dimension_units['energy'],
                chunksizes=(1, ))<commit_msg>restructuring of storage<commit_after>import numpy as np
from openpathsampling.snapshot import Snapshot, Configuration, Momentum
from object_storage import ObjectStore
from openpathsampling.trajectory import Trajectory
import simtk.unit as u

class SnapshotStore(ObjectStore):
    """
    An ObjectStore for Snapshots in netCDF files.
    """

    def __init__(self):
        super(SnapshotStore, self).__init__(Snapshot, json=False)

    def load(self, idx=None):
        '''
        Load a snapshot from the storage.

        Parameters
        ----------
        idx : int
            the integer index of the snapshot to be loaded

        Returns
        -------
        snapshot : Snapshot
            the loaded snapshot instance
        '''

        configuration = self.vars['configuration'][idx]
        momentum = self.vars['momentum'][idx]
        momentum_reversed = self.vars['momentum_reversed'][idx]
        reversed_idx = self.vars['reversed_idx'][idx]

        snapshot = Snapshot(
            configuration=configuration,
            momentum=momentum,
            is_reversed=momentum_reversed,
            reversed_copy=None
        )
        snapshot_reversed = Snapshot(
            configuration=configuration,
            momentum=momentum,
            is_reversed=not momentum_reversed,
            reversed_copy=None
        )

        snapshot._reversed = snapshot_reversed
        snapshot_reversed._reversed = snapshot

        # fix caching!
        snapshot_reversed.idx[self.storage] = reversed_idx
        self.cache[reversed_idx] = snapshot_reversed

        return snapshot

    def all(self):
        """
        Return a trajectory consisting of all (unordered) frames in the storage.

        Notes
        -----
        If you are interested in collectivevariables this is faster since it does not
        load the snapshots. Otherwise storage.snapshots is fine to get an
        iterator. Both should should be about the same speed.
        """
        #TODO: Might think about replacing the iterator with this since it is
        # faster for collectivevariables
        return Trajectory([ (self, idx) for idx in range(len(self)) ])

    def save(self, snapshot, idx=None):
        """
        Add the current state of the snapshot in the database.

        Parameters
        ----------
        snapshot : Snapshot()
            the snapshot to be saved
        idx : int or None
            if idx is not None the index will be used for saving in the storage.
            This might overwrite already existing trajectories!

        Notes
        -----
        This also saves all contained frames in the snapshot if not done yet.
        A single Snapshot object can only be saved once!
        """

        self.vars['configuration'][idx] = snapshot.configuration
        self.vars['momentum'][idx] = snapshot.momentum
        self.vars['reversed'][idx] = snapshot._reversed

        self.vars['momentum_reversed'][idx] = snapshot.is_reversed

    def configuration_idx(self, idx):
        '''
        Load snapshot index for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        list of int
            configuration indices
        '''
        return self.vars['configuration'][idx]

    def momentum_idx(self, idx):
        '''
        Load momentum index for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        int
            momentum indices
        '''
        return self.vars['momentum'][idx]

    def reversed_idx(self, idx):
        '''
        Load snapshot index for the reversed snapshot with ID 'idx'
        from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        int
            reversed snapshot indices
        '''
        return self.vars['reversed'][idx]

    def momentum_reversed(self, idx):
        '''
        Load reversed boolean for snapshot with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            index of the snapshot

        Returns
        -------
        boolean
            boolean if the momentum of the snapshot is reversed
        '''
        return self.vars['momentum_reversed'][idx]

    def _init(self):
        '''
        Initializes the associated storage to index configuration_indices in it
        '''
        super(SnapshotStore, self)._init()

        self.init_variable('configuration', 'obj.configurations',
                description="the snapshot index (0..n_configuration-1) of snapshot '{idx}'.",
                chunksizes=(1, )
        )

        self.init_variable('momentum', 'obj.momenta',
                description="the snapshot index (0..n_momentum-1) 'frame' of snapshot '{idx}'.",
                chunksizes=(1, )
                )

        self.init_variable('momentum_reversed', 'bool', chunksizes=(1, ))

        self.init_variable('reversed', 'index',
                description="the idx of the reversed snapshot index (0..n_snapshot-1) 'snapshot' of snapshot '{idx}'.",
                chunksizes=(1, )
                )

#=============================================================================================
# COLLECTIVE VARIABLE UTILITY FUNCTIONS
#=============================================================================================

    @property
    def op_configuration_idx(self):
        """
        Returns aa function that returns for an object of this storage the idx

        Returns
        -------
        function
            the function that returns the idx of the configuration
        """
        def idx(obj):
            return obj.configuration.idx[self.storage]

        return idx

    @property
    def op_momentum_idx(self):
        """
        Returns aa function that returns for an object of this storage the idx

        Returns
        -------
        function
            the function that returns the idx of the configuration

        """
        def idx(obj):
            return obj.momenta.idx[self.storage]

        return idx

class MomentumStore(ObjectStore):
    """
    An ObjectStore for Momenta. Allows to store Momentum() instances in a netcdf file.
    """

    def __init__(self):
        super(MomentumStore, self).__init__(Momentum, json=False, load_partial=True)

        # attach delayed loaders
        self.set_variable_partial_loading('velocities')
        self.set_variable_partial_loading('kinetic_energy')

    def save(self, momentum, idx = None):
        """
        Save velocities and kinetic energies.

        Parameters
        ----------
        momentum : Momentum()
            the actual Momentum() instance to be saved.
        idx : int or None
            if not None `idx`is used as the index to index the Momentum()
            instance. Might overwrite existing Momentum in the database.
        """
        if momentum.velocities is not None:
            self.vars['velocities'][idx,:,:] = momentum.velocities
        else:
            print 'ERROR : Momentum should not be empty'

        if momentum.kinetic_energy is not None:
            self.vars['kinetic_energy'][idx] = momentum.kinetic_energy
        else:
            # TODO: No kinetic energy is not yet supported
            print 'Think about how to handle this. It should only be None if loaded lazy and in this case it will never be saved.'

        # Force sync to disk to avoid data loss.
        # storage.sync()

    def load(self, idx):
        '''
        Load a momentum from the storage

        Parameters
        ----------
        idx : int
            index of the momentum in the database 'idx' > 0

        Returns
        -------
        Momentum()
            the loaded momentum instance
        '''

        velocities = self.vars['velocities'][idx]
        kinetic_energy = self.vars['kinetic_energy'][idx]

        momentum = Momentum(velocities=velocities, kinetic_energy=kinetic_energy)

        return momentum

    def load_empty(self, idx):
        momentum = Momentum()
        del momentum.velocities
        del momentum.kinetic_energy
        return momentum

    def velocities_as_numpy(self, frame_indices=None, atom_indices=None):
        """
        Return a block of stored velocities in the database as a numpy array.

        Parameters
        ----------
        frame_indices : list of int or None
            the indices of Momentum objects to be retrieved from the database.
            If `None` is specified then all indices are returned!
        atom_indices : list of int of None
            if not None only the specified atom_indices are returned. Might
            speed up reading a lot.
        """

        if frame_indices is None:
            frame_indices = slice(None)

        if atom_indices is None:
            atom_indices = slice(None)

        return self.variables['velocities'][frame_indices,atom_indices,:].astype(np.float32).copy()

    def velocities_as_array(self, frame_indices=None, atom_indices=None):
        '''
        Returns a numpy array consisting of all velocities at the given indices

        Parameters
        ----------
        frame_indices : list of int
            momenta indices to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        return self.velocities_as_numpy(frame_indices, atom_indices)

    def _init(self):
        '''
        Initializes the associated storage to index momentums in it
        '''

        super(MomentumStore, self)._init()

        n_atoms = self.storage.n_atoms
        n_spatial = self.storage.n_spatial

        self.init_variable('velocities', 'float',
                dimensions=(self.prefix, 'atom', 'spatial'),
                units=self.dimension_units['velocity'],
                description="the velocity of atom 'atom' in dimension " +
                            "'coordinate' of momentum 'momentum'.",
                chunksizes=(1, n_atoms, n_spatial))

        self.init_variable('kinetic_energy', 'float', self.prefix,
                self.dimension_units['energy'],
                chunksizes=(1, ))

class ConfigurationStore(ObjectStore):
    def __init__(self):
        super(ConfigurationStore, self).__init__(Configuration, json=False, load_partial=True)

        # attach delayed loaders
        self.set_variable_partial_loading('coordinates')
        self.set_variable_partial_loading('box_vectors')
        self.set_variable_partial_loading('potential_energy')

    def save(self, configuration, idx = None):
        # Store configuration.
        self.vars['coordinates'][idx] = configuration.coordinates

        if configuration.potential_energy is not None:
            self.vars['potential'][idx] = configuration.potential_energy

        if configuration.box_vectors is not None:
            self.vars['box_vectors'][idx] = configuration.box_vectors

    def get(self, indices):
        return [ self.load(idx) for idx in indices ]

    def load(self, idx):
        coordinates = self.vars["coordinates"][idx]
        box_vectors = self.vars["box_vectors"][idx]
        potential_energy = self.vars["potential"][idx]

        configuration = Configuration(coordinates=coordinates, box_vectors = box_vectors, potential_energy=potential_energy)
        configuration.topology = self.storage.topology

        return configuration

    def load_empty(self, idx):
        """
        Loading function for partial loading. Constructs an empty Configuration
        object.

        Parameters
        ----------
        idx : int
            the integer index of the configuration to be loaded

        Returns
        -------
        Configuration
            an empty configuration object
        """
        configuration = Configuration()
        configuration.topology = self.storage.topology

        # if these still exist they will not be loaded using __getattr__
        del configuration.coordinates
        del configuration.box_vectors
        del configuration.potential_energy

        return configuration

    def coordinates_as_numpy(self, frame_indices=None, atom_indices=None):
        """
        Return the atom coordinates in the storage for given frame indices
        and atoms

        Parameters
        ----------
        frame_indices : list of int or None
            the frame indices to be included. If None all frames are returned
        atom_indices : list of int or None
            the atom indices to be included. If None all atoms are returned

        Returns
        -------
        numpy.array, shape=(n_frames, n_atoms)
            the array of atom coordinates in a float32 numpy array

        """
        if frame_indices is None:
            frame_indices = slice(None)

        if atom_indices is None:
            atom_indices = slice(None)

        return self.storage.variables[self.prefix + '_coordinates'][frame_indices,atom_indices,:].astype(np.float32).copy()

    def coordinates_as_array(self, frame_indices=None, atom_indices=None):
        '''
        Returns a numpy array consisting of all coordinates at the given indices

        Parameters
        ----------
        frame_indices : list of int
            configuration indices to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        return self.coordinates_as_numpy(frame_indices, atom_indices)

    def snapshot_coordinates_as_array(self, idx, atom_indices=None):
        '''
        Returns a numpy array consisting of all coordinates of a snapshot

        Parameters
        ----------
        idx : int
            index of the snapshot to be loaded
        atom_indices : list of int
            selects only the atoms to be returned. If None (Default) all atoms
            will be selected

        Returns
        -------
        numpy.ndarray, shape = (l,n)
            returns an array with `l` the number of frames and `n` the number
            of atoms
        '''

        frame_indices = self.configuration_indices(idx)
        return self.coordinates_as_array(frame_indices, atom_indices)

    def _init(self):
        super(ConfigurationStore, self)._init()
        n_atoms = self.storage.n_atoms
        n_spatial = self.storage.n_spatial

        self.init_variable('coordinates', 'float',
                (self.prefix, 'atom','spatial'), self.dimension_units['length'],
                description="coordinate of atom '{ix[1]}' in dimension " +
                            "'{ix[2]}' of configuration '{ix[0]}'.",
                chunksizes=(1,n_atoms,n_spatial))

        self.init_variable('box_vectors', 'float',
                (self.prefix, 'spatial', 'spatial'),
                self.dimension_units['length'],
                chunksizes=(1,n_spatial,n_spatial))

        self.init_variable('potential', 'float',
                self.prefix,
                self.dimension_units['energy'],
                chunksizes=(1, ))<|endoftext|>


<commit_before>from datetime import datetime
from design import sync_design, endDate_view
from flask import Flask, render_template, request, abort, url_for, redirect, Response
from flask.ext.assets import Environment, Bundle
from flask_redis import Redis
from gevent import monkey
from paste.proxy import make_proxy
from pytz import timezone as tz
from urlparse import urljoin
from wsgiproxy import HostProxy
import couchdb
import time
from sse import Sse as PySse
from webob.exc import HTTPNotFound
from pkg_resources import parse_version
from requests.exceptions import ConnectionError

monkey.patch_all()

class AuctionsHostProxy(HostProxy):

    def process_request(self, uri, method, headers, environ):
        headers["X-Forwarded-Path"] = request.url
        headers["X-Request-ID"] = environ.get("x_request_id", "")
        try:
            return super(AuctionsHostProxy, self).process_request(
                uri, method, headers, environ
            )
        except ConnectionError, e:
            auctions_server.logger.warning("Error on request to {} with msg {}".format(uri, e))
            # status, location, headerslist, app_iter
            return (404, uri, {}, [])

auctions_server = Flask(
    __name__,
    static_url_path='',
    template_folder='templates'
)

################################################################################
assets = Environment(auctions_server)
css = Bundle("vendor/bootstrap/dist/css/bootstrap.min.css",
             "vendor/angular-growl-2/build/angular-growl.min.css",
             "static/css/starter-template.css",
             filters='cssmin,datauri', output='min/styles_%(version)s.css')
assets.register('all_css', css)

js = Bundle("vendor/event-source-polyfill/eventsource.min.js",
            "vendor/moment/min/moment.min.js",
            "vendor/angular/angular.min.js",
            "vendor/angular-cookies/angular-cookies.min.js",
            "vendor/pouchdb/dist/pouchdb.js",
            "vendor/angular-bootstrap/ui-bootstrap-tpls.min.js",
            "vendor/angular-timer/dist/angular-timer.min.js",
            "vendor/angular-translate/angular-translate.min.js",
            "vendor/angular-translate-storage-cookie/angular-translate-storage-cookie.min.js",
            "vendor/angular-translate-storage-local/angular-translate-storage-local.min.js",
            "vendor/angular-growl-2/build/angular-growl.js",
            "static/js/app.js",
            "static/js/utils.js",
            "static/js/translations.js",
            "static/js/controllers.js",
            "vendor/moment/locale/uk.js",
            "vendor/moment/locale/ru.js",
            filters='jsmin', output='min/all_js_%(version)s.js')
assets.register('all_js', js)

################################################################################

@auctions_server.before_request
def before_request():
    auctions_server.logger.debug('Start {0.method}: {0.url}'.format(request))

@auctions_server.after_request
def after_request(response):
    auctions_server.logger.debug(
        'End {1.status_code} : {0.method} : {0.url} '.format(request, response)
    )
    return response

@auctions_server.route('/tenders/<auction_doc_id>')
def auction_url(auction_doc_id):
    unsupported_browser = False
    if request.user_agent.browser == 'msie':
        if parse_version(request.user_agent.version) < parse_version('9'):
            unsupported_browser = True
    elif request.user_agent.browser == 'opera':
        if 'Opera Mini' in request.user_agent.string:
            unsupported_browser = True
    return render_template(
        'index.html',
        db_url=auctions_server.config.get('EXT_COUCH_DB'),
        auction_doc_id=auction_doc_id,
        unsupported_browser=unsupported_browser
    )

@auctions_server.route('/')
def archive_tenders_list_index():
    return render_template(
        'list.html',
        documents=reversed(
            [auction.doc
             for auction in endDate_view(auctions_server.db,
                                         startkey=time.time() * 1000,
                                         include_docs=True)
             ])
    )

@auctions_server.route('/archive')
def auction_list_index():
    return render_template(
        'list.html',
        documents=[auction.doc
                   for auction in endDate_view(auctions_server.db,
                                               endkey=time.time() * 1000,
                                               include_docs=True)
                   ]
    )

@auctions_server.route('/tenders/<auction_doc_id>/<path:path>',
                       methods=['GET', 'POST'])
def auctions_proxy(auction_doc_id, path):
    auctions_server.logger.debug('Auction_doc_id: {}'.format(auction_doc_id))
    proxy_path = auctions_server.redis.get(auction_doc_id)
    auctions_server.logger.debug('Proxy path: {}'.format(proxy_path))
    if proxy_path:
        request.environ['PATH_INFO'] = '/' + path
        auctions_server.logger.debug('Start proxy to path: {}'.format(path))
        return AuctionsHostProxy(proxy_path, client='requests', chunk_size=1)
    elif path == 'login' and auction_doc_id in auctions_server.db:
        return redirect((
            url_for('auction_url', auction_doc_id=auction_doc_id,
                    wait=1, **request.args)
        ))
    elif path == 'event_source':
        events_close = PySse()
        events_close.add_message("Close", "Disable")
        return Response(
            events_close,
            mimetype='text/event-stream',
            content_type='text/event-stream'
        )
    return abort(404)

@auctions_server.route('/get_current_server_time')
def auctions_server_current_server_time():
    response = Response(datetime.now(auctions_server.config['TIMEZONE']).isoformat())
    response.headers['Cache-Control'] = 'public, max-age=0'
    return response

def couch_server_proxy(path):
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def make_auctions_app(global_conf,
                      redis_url='redis://localhost:7777/0',
                      external_couch_url='http://localhost:5000/auction',
                      internal_couch_url='http://localhost:9000/',
                      proxy_internal_couch_url='http://localhost:9000/',
                      auctions_db='auctions',
                      hash_secret_key='',
                      timezone='Europe/Kiev',
                      preferred_url_scheme='http'
                      ):
    """
    [app:main]
    use = egg:openprocurement.auction#auctions_server
    redis_url = redis://:passwod@localhost:1111/0
    external_couch_url = http://localhost:1111/auction
    internal_couch_url = http://localhost:9011/
    auctions_db = auction
    timezone = Europe/Kiev
    """
    auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
    auctions_server.config['REDIS_URL'] = redis_url
    auctions_server.config['EXT_COUCH_DB'] = urljoin(
        external_couch_url,
        auctions_db
    )
    auctions_server.add_url_rule(
        '/' + auctions_db + '/<path:path>',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '/',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'], defaults={'path': ''})
    auctions_server.config['INT_COUCH_URL'] = internal_couch_url
    auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
    auctions_server.config['COUCH_DB'] = auctions_db
    auctions_server.config['TIMEZONE'] = tz(timezone)
    auctions_server.redis = Redis(auctions_server)
    auctions_server.db = couchdb.client.Database(
        urljoin(auctions_server.config.get('INT_COUCH_URL'),
                auctions_server.config['COUCH_DB'])
    )
    auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
    sync_design(auctions_server.db)
    # auctions_server.config['ASSETS_DEBUG'] = True
    return auctions_server
<commit_msg>Added debug option<commit_after>from datetime import datetime
from design import sync_design, endDate_view
from flask import Flask, render_template, request, abort, url_for, redirect, Response
from flask.ext.assets import Environment, Bundle
from flask_redis import Redis
from gevent import monkey
from paste.proxy import make_proxy
from pytz import timezone as tz
from urlparse import urljoin
from wsgiproxy import HostProxy
import couchdb
import time
from sse import Sse as PySse
from webob.exc import HTTPNotFound
from pkg_resources import parse_version
from requests.exceptions import ConnectionError

monkey.patch_all()

class AuctionsHostProxy(HostProxy):

    def process_request(self, uri, method, headers, environ):
        headers["X-Forwarded-Path"] = request.url
        headers["X-Request-ID"] = environ.get("x_request_id", "")
        try:
            return super(AuctionsHostProxy, self).process_request(
                uri, method, headers, environ
            )
        except ConnectionError, e:
            auctions_server.logger.warning("Error on request to {} with msg {}".format(uri, e))
            # status, location, headerslist, app_iter
            return (404, uri, {}, [])

auctions_server = Flask(
    __name__,
    static_url_path='',
    template_folder='templates'
)

################################################################################
assets = Environment(auctions_server)
css = Bundle("vendor/bootstrap/dist/css/bootstrap.min.css",
             "vendor/angular-growl-2/build/angular-growl.min.css",
             "static/css/starter-template.css",
             filters='cssmin,datauri', output='min/styles_%(version)s.css')
assets.register('all_css', css)

js = Bundle("vendor/event-source-polyfill/eventsource.min.js",
            "vendor/moment/min/moment.min.js",
            "vendor/angular/angular.min.js",
            "vendor/angular-cookies/angular-cookies.min.js",
            "vendor/pouchdb/dist/pouchdb.js",
            "vendor/angular-bootstrap/ui-bootstrap-tpls.min.js",
            "vendor/angular-timer/dist/angular-timer.min.js",
            "vendor/angular-translate/angular-translate.min.js",
            "vendor/angular-translate-storage-cookie/angular-translate-storage-cookie.min.js",
            "vendor/angular-translate-storage-local/angular-translate-storage-local.min.js",
            "vendor/angular-growl-2/build/angular-growl.js",
            "static/js/app.js",
            "static/js/utils.js",
            "static/js/translations.js",
            "static/js/controllers.js",
            "vendor/moment/locale/uk.js",
            "vendor/moment/locale/ru.js",
            filters='jsmin', output='min/all_js_%(version)s.js')
assets.register('all_js', js)

################################################################################

@auctions_server.before_request
def before_request():
    auctions_server.logger.debug('Start {0.method}: {0.url}'.format(request))

@auctions_server.after_request
def after_request(response):
    auctions_server.logger.debug(
        'End {1.status_code} : {0.method} : {0.url} '.format(request, response)
    )
    return response

@auctions_server.route('/tenders/<auction_doc_id>')
def auction_url(auction_doc_id):
    unsupported_browser = False
    if request.user_agent.browser == 'msie':
        if parse_version(request.user_agent.version) < parse_version('9'):
            unsupported_browser = True
    elif request.user_agent.browser == 'opera':
        if 'Opera Mini' in request.user_agent.string:
            unsupported_browser = True
    return render_template(
        'index.html',
        db_url=auctions_server.config.get('EXT_COUCH_DB'),
        auction_doc_id=auction_doc_id,
        unsupported_browser=unsupported_browser
    )

@auctions_server.route('/')
def archive_tenders_list_index():
    return render_template(
        'list.html',
        documents=reversed(
            [auction.doc
             for auction in endDate_view(auctions_server.db,
                                         startkey=time.time() * 1000,
                                         include_docs=True)
             ])
    )

@auctions_server.route('/archive')
def auction_list_index():
    return render_template(
        'list.html',
        documents=[auction.doc
                   for auction in endDate_view(auctions_server.db,
                                               endkey=time.time() * 1000,
                                               include_docs=True)
                   ]
    )

@auctions_server.route('/tenders/<auction_doc_id>/<path:path>',
                       methods=['GET', 'POST'])
def auctions_proxy(auction_doc_id, path):
    auctions_server.logger.debug('Auction_doc_id: {}'.format(auction_doc_id))
    proxy_path = auctions_server.redis.get(auction_doc_id)
    auctions_server.logger.debug('Proxy path: {}'.format(proxy_path))
    if proxy_path:
        request.environ['PATH_INFO'] = '/' + path
        auctions_server.logger.debug('Start proxy to path: {}'.format(path))
        return AuctionsHostProxy(proxy_path, client='requests', chunk_size=1)
    elif path == 'login' and auction_doc_id in auctions_server.db:
        return redirect((
            url_for('auction_url', auction_doc_id=auction_doc_id,
                    wait=1, **request.args)
        ))
    elif path == 'event_source':
        events_close = PySse()
        events_close.add_message("Close", "Disable")
        return Response(
            events_close,
            mimetype='text/event-stream',
            content_type='text/event-stream'
        )
    return abort(404)

@auctions_server.route('/get_current_server_time')
def auctions_server_current_server_time():
    response = Response(datetime.now(auctions_server.config['TIMEZONE']).isoformat())
    response.headers['Cache-Control'] = 'public, max-age=0'
    return response

def couch_server_proxy(path):
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def make_auctions_app(global_conf,
                      redis_url='redis://localhost:7777/0',
                      external_couch_url='http://localhost:5000/auction',
                      internal_couch_url='http://localhost:9000/',
                      proxy_internal_couch_url='http://localhost:9000/',
                      auctions_db='auctions',
                      hash_secret_key='',
                      timezone='Europe/Kiev',
                      preferred_url_scheme='http',
                      debug=False
                      ):
    """
    [app:main]
    use = egg:openprocurement.auction#auctions_server
    redis_url = redis://:passwod@localhost:1111/0
    external_couch_url = http://localhost:1111/auction
    internal_couch_url = http://localhost:9011/
    auctions_db = auction
    timezone = Europe/Kiev
    """
    auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
    auctions_server.config['REDIS_URL'] = redis_url
    auctions_server.config['EXT_COUCH_DB'] = urljoin(
        external_couch_url,
        auctions_db
    )
    auctions_server.add_url_rule(
        '/' + auctions_db + '/<path:path>',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '/',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'], defaults={'path': ''})
    auctions_server.config['INT_COUCH_URL'] = internal_couch_url
    auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
    auctions_server.config['COUCH_DB'] = auctions_db
    auctions_server.config['TIMEZONE'] = tz(timezone)
    auctions_server.redis = Redis(auctions_server)
    auctions_server.db = couchdb.client.Database(
        urljoin(auctions_server.config.get('INT_COUCH_URL'),
                auctions_server.config['COUCH_DB'])
    )
    auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
    sync_design(auctions_server.db)
    auctions_server.config['ASSETS_DEBUG'] = True if debug else False
    return auctions_server
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import unittest
import numpy
from openquake.baselib.general import gettemp
from openquake.hazardlib.probability_map import combine
from openquake.calculators import getters
from openquake.calculators.views import view
from openquake.calculators.export import export
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.qa_tests_data.disagg import (
    case_1, case_2, case_3, case_4, case_5, case_master)

class DisaggregationTestCase(CalculatorTestCase):

    def assert_curves_ok(self, expected, test_dir, fmt='xml', delta=None):
        if sys.platform == 'win32':  # disable concurrency on windows
            out = self.run_calc(test_dir, 'job.ini', exports=fmt,
                                concurrent_tasks='0')
        else:
            out = self.run_calc(test_dir, 'job.ini', exports=fmt)
        got = out['disagg', fmt]
        self.assertEqual(len(expected), len(got))
        for fname, actual in zip(expected, got):
            self.assertEqualFiles('expected_output/%s' % fname, actual)
        return out

    def test_case_1(self):
        out = self.assert_curves_ok(
            ['poe-0.02-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv'],
            case_1.__file__,
            fmt='csv')

        # check disagg_by_src, poe=0.02, 0.1, imt=PGA, SA(0.025)
        self.assertEqual(len(out['disagg_by_src', 'csv']), 4)
        for fname in out['disagg_by_src', 'csv']:
            self.assertEqualFiles('expected_output/%s' % strip_calc_id(fname),
                                  fname)

        # disaggregation by source group
        rlzs_assoc = self.calc.datastore['csm_info'].get_rlzs_assoc()
        by_grp = rlzs_assoc.by_grp()
        ws = [rlz.weight for rlz in rlzs_assoc.realizations]
        pgetter = getters.PmapGetter(self.calc.datastore, by_grp, ws)
        pgetter.init()
        pmaps = []
        for grp in sorted(pgetter.dstore['poes']):
            pmaps.append(pgetter.get_mean(grp))
        # make sure that the combination of the contributions is okay
        pmap = pgetter.get_mean()  # total mean map
        cmap = combine(pmaps)  # combination of the mean maps per source group
        for sid in pmap:
            numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)

    def test_case_2(self):
        # this is a case with disagg_outputs = Mag and 4 realizations
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.assert_curves_ok([
            'rlz-0-PGA--3.0--3.0.xml', 'rlz-0-PGA-0.0-0.0.xml',
            'rlz-1-PGA--3.0--3.0.xml', 'rlz-1-PGA-0.0-0.0.xml',
            'rlz-2-PGA-0.0-0.0.xml', 'rlz-3-PGA-0.0-0.0.xml'],
            case_2.__file__)

        # check that the CSV exporter does not break
        fnames = export(('disagg', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 6)  # number of CSV files

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 2)  # 2 sid x 1 key x 1 poe x 1 imt
        for fname in fnames:
            self.assertEqualFiles(
                'expected_output/%s' % strip_calc_id(fname), fname)

    def test_case_3(self):
        with self.assertRaises(ValueError) as ctx:
            self.run_calc(case_3.__file__, 'job.ini')
        self.assertEqual(str(ctx.exception), '''\
You are trying to disaggregate for poe=0.1.
However the source model #0, 'source_model_test_complex.xml',
produces at most probabilities of 0.0362321 for rlz=#0, IMT=PGA.
The disaggregation PoE is too big or your model is wrong,
producing too small PoEs.''')

    def test_case_4(self):
        # this is case with number of lon/lat bins different for site 0/site 1
        # this exercise sampling
        self.run_calc(case_4.__file__, 'job.ini')

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 64)  # 2 sid x 8 keys x 2 poe x 2 imt

    def test_case_5(self):
        # this exercise gridded nonparametric sources
        self.run_calc(case_5.__file__, 'job.ini')

    def test_case_master(self):
        # this tests exercise the case of a complex logic tree; it also
        # prints the warning on poe_agg very different from the expected poe
        self.run_calc(case_master.__file__, 'job.ini')
        fname = gettemp(view('mean_disagg', self.calc.datastore))
        self.assertEqualFiles('expected/mean_disagg.rst', fname)
        os.remove(fname)

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 192)  # 2 sid x 8 keys x 2 poe x 2 imt
        # = 64 x 3 for mean, quantile-0.15, quantile-0.85

    def test_disagg_by_src(self):
        # this is a case with iml_disagg and disagg_by_src
        self.run_calc(case_master.__file__, 'job1.ini')
        arr = self.calc.datastore[
            'disagg_by_src/iml-0.02-PGA--122.6-38.3'][()]
        numpy.testing.assert_almost_equal(arr, [0.6757448, 0.1780308])
<commit_msg>Commented the tests on statistics<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import unittest
import numpy
from openquake.baselib.general import gettemp
from openquake.hazardlib.probability_map import combine
from openquake.calculators import getters
from openquake.calculators.views import view
from openquake.calculators.export import export
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.qa_tests_data.disagg import (
    case_1, case_2, case_3, case_4, case_5, case_master)

class DisaggregationTestCase(CalculatorTestCase):

    def assert_curves_ok(self, expected, test_dir, fmt='xml', delta=None):
        if sys.platform == 'win32':  # disable concurrency on windows
            out = self.run_calc(test_dir, 'job.ini', exports=fmt,
                                concurrent_tasks='0')
        else:
            out = self.run_calc(test_dir, 'job.ini', exports=fmt)
        got = out['disagg', fmt]
        self.assertEqual(len(expected), len(got))
        for fname, actual in zip(expected, got):
            self.assertEqualFiles('expected_output/%s' % fname, actual)
        return out

    def test_case_1(self):
        out = self.assert_curves_ok(
            ['poe-0.02-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv'],
            case_1.__file__,
            fmt='csv')

        # check disagg_by_src, poe=0.02, 0.1, imt=PGA, SA(0.025)
        self.assertEqual(len(out['disagg_by_src', 'csv']), 4)
        for fname in out['disagg_by_src', 'csv']:
            self.assertEqualFiles('expected_output/%s' % strip_calc_id(fname),
                                  fname)

        # disaggregation by source group
        rlzs_assoc = self.calc.datastore['csm_info'].get_rlzs_assoc()
        by_grp = rlzs_assoc.by_grp()
        ws = [rlz.weight for rlz in rlzs_assoc.realizations]
        pgetter = getters.PmapGetter(self.calc.datastore, by_grp, ws)
        pgetter.init()
        pmaps = []
        for grp in sorted(pgetter.dstore['poes']):
            pmaps.append(pgetter.get_mean(grp))
        # make sure that the combination of the contributions is okay
        pmap = pgetter.get_mean()  # total mean map
        cmap = combine(pmaps)  # combination of the mean maps per source group
        for sid in pmap:
            numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)

    def test_case_2(self):
        # this is a case with disagg_outputs = Mag and 4 realizations
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.assert_curves_ok([
            'rlz-0-PGA--3.0--3.0.xml', 'rlz-0-PGA-0.0-0.0.xml',
            'rlz-1-PGA--3.0--3.0.xml', 'rlz-1-PGA-0.0-0.0.xml',
            'rlz-2-PGA-0.0-0.0.xml', 'rlz-3-PGA-0.0-0.0.xml'],
            case_2.__file__)

        # check that the CSV exporter does not break
        fnames = export(('disagg', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 6)  # number of CSV files

        #fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        #self.assertEqual(len(fnames), 2)  # 2 sid x 1 key x 1 poe x 1 imt
        #for fname in fnames:
        #    self.assertEqualFiles(
        #        'expected_output/%s' % strip_calc_id(fname), fname)

    def test_case_3(self):
        with self.assertRaises(ValueError) as ctx:
            self.run_calc(case_3.__file__, 'job.ini')
        self.assertEqual(str(ctx.exception), '''\
You are trying to disaggregate for poe=0.1.
However the source model #0, 'source_model_test_complex.xml',
produces at most probabilities of 0.0362321 for rlz=#0, IMT=PGA.
The disaggregation PoE is too big or your model is wrong,
producing too small PoEs.''')

    def test_case_4(self):
        # this is case with number of lon/lat bins different for site 0/site 1
        # this exercise sampling
        self.run_calc(case_4.__file__, 'job.ini')

        #fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        #self.assertEqual(len(fnames), 64)  # 2 sid x 8 keys x 2 poe x 2 imt

    def test_case_5(self):
        # this exercise gridded nonparametric sources
        self.run_calc(case_5.__file__, 'job.ini')

    def test_case_master(self):
        # this tests exercise the case of a complex logic tree; it also
        # prints the warning on poe_agg very different from the expected poe
        self.run_calc(case_master.__file__, 'job.ini')
        fname = gettemp(view('mean_disagg', self.calc.datastore))
        self.assertEqualFiles('expected/mean_disagg.rst', fname)
        os.remove(fname)

        #fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        #self.assertEqual(len(fnames), 192)  # 2 sid x 8 keys x 2 poe x 2 imt
        # = 64 x 3 for mean, quantile-0.15, quantile-0.85

    def test_disagg_by_src(self):
        # this is a case with iml_disagg and disagg_by_src
        self.run_calc(case_master.__file__, 'job1.ini')
        arr = self.calc.datastore[
            'disagg_by_src/iml-0.02-PGA--122.6-38.3'][()]
        numpy.testing.assert_almost_equal(arr, [0.6757448, 0.1780308])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import unittest
import numpy
from nose.plugins.attrib import attr
from openquake.baselib.general import gettemp
from openquake.hazardlib.probability_map import combine
from openquake.calculators import getters
from openquake.calculators.views import view
from openquake.calculators.export import export
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.qa_tests_data.disagg import (
    case_1, case_2, case_3, case_4, case_5, case_master)

class DisaggregationTestCase(CalculatorTestCase):

    def assert_curves_ok(self, expected, test_dir, fmt='xml', delta=None):
        if sys.platform == 'win32':  # disable concurrency on windows
            out = self.run_calc(test_dir, 'job.ini', exports=fmt,
                                concurrent_tasks='0')
        else:
            out = self.run_calc(test_dir, 'job.ini', exports=fmt)
        got = out['disagg', fmt]
        self.assertEqual(len(expected), len(got))
        for fname, actual in zip(expected, got):
            self.assertEqualFiles('expected_output/%s' % fname, actual)
        return out

    @attr('qa', 'hazard', 'disagg')
    def test_case_1(self):
        out = self.assert_curves_ok(
            ['poe-0.02-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv'],
            case_1.__file__,
            fmt='csv')

        # check disagg_by_src, poe=0.02, 0.1, imt=PGA, SA(0.025)
        self.assertEqual(len(out['disagg_by_src', 'csv']), 4)
        for fname in out['disagg_by_src', 'csv']:
            self.assertEqualFiles('expected_output/%s' % strip_calc_id(fname),
                                  fname)

        # disaggregation by source group
        rlzs_assoc = self.calc.datastore['csm_info'].get_rlzs_assoc()
        pgetter = getters.PmapGetter(self.calc.datastore, rlzs_assoc)
        pgetter.init()
        pmaps = []
        for grp in sorted(pgetter.dstore['poes']):
            pmaps.append(pgetter.get_mean(grp))
        # make sure that the combination of the contributions is okay
        pmap = pgetter.get_mean()  # total mean map
        cmap = combine(pmaps)  # combination of the mean maps per source group
        for sid in pmap:
            numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)

    @attr('qa', 'hazard', 'disagg')
    def test_case_2(self):
        # this is a case with disagg_outputs = Mag and 4 realizations
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.assert_curves_ok([
            'rlz-0-PGA--3.0--3.0.xml', 'rlz-0-PGA-0.0-0.0.xml',
            'rlz-1-PGA--3.0--3.0.xml', 'rlz-1-PGA-0.0-0.0.xml',
            'rlz-2-PGA-0.0-0.0.xml', 'rlz-3-PGA-0.0-0.0.xml'],
            case_2.__file__)

        # check that the CSV exporter does not break
        fnames = export(('disagg', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 6)  # number of CSV files

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 2)  # 2 sid x 1 key x 1 poe x 1 imt
        for fname in fnames:
            self.assertEqualFiles(
                'expected_output/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'hazard', 'disagg')
    def test_case_3(self):
        with self.assertRaises(ValueError) as ctx:
            self.run_calc(case_3.__file__, 'job.ini')
        self.assertEqual(str(ctx.exception), '''\
You are trying to disaggregate for poe=0.1.
However the source model #0, 'source_model_test_complex.xml',
produces at most probabilities of 0.0362321 for rlz=#0, IMT=PGA.
The disaggregation PoE is too big or your model is wrong,
producing too small PoEs.''')

    @attr('qa', 'hazard', 'disagg')
    def test_case_4(self):
        # this is case with number of lon/lat bins different for site 0/site 1
        # this exercise sampling
        self.run_calc(case_4.__file__, 'job.ini')

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 64)  # 2 sid x 8 keys x 2 poe x 2 imt

    @attr('qa', 'hazard', 'disagg')
    def test_case_5(self):
        # this exercise gridded nonparametric sources
        self.run_calc(case_5.__file__, 'job.ini')

    @attr('qa', 'hazard', 'disagg')
    def test_case_master(self):
        # this tests exercise the case of a complex logic tree; it also
        # prints the warning on poe_agg very different from the expected poe
        self.run_calc(case_master.__file__, 'job.ini')
        fname = gettemp(view('mean_disagg', self.calc.datastore))
        self.assertEqualFiles('expected/mean_disagg.rst', fname)
        os.remove(fname)

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 192)  # 2 sid x 8 keys x 2 poe x 2 imt
        # = 64 x 3 for mean, quantile-0.15, quantile-0.85

    @attr('qa', 'hazard', 'disagg')
    def test_disagg_by_src(self):
        # this is a case with iml_disagg and disagg_by_src
        self.run_calc(case_master.__file__, 'job1.ini')
        arr = self.calc.datastore[
            'disagg_by_src/iml-0.02-PGA--122.6-38.3'].value
        numpy.testing.assert_almost_equal(arr, [0.6757448, 0.1780308])
<commit_msg>Added test [skip hazardlib]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import unittest
import numpy
from nose.plugins.attrib import attr
from openquake.baselib.general import gettemp
from openquake.hazardlib.probability_map import combine
from openquake.calculators import getters
from openquake.calculators.views import view
from openquake.calculators.export import export
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.qa_tests_data.disagg import (
    case_1, case_2, case_3, case_4, case_5, case_master)

class DisaggregationTestCase(CalculatorTestCase):

    def assert_curves_ok(self, expected, test_dir, fmt='xml', delta=None):
        if sys.platform == 'win32':  # disable concurrency on windows
            out = self.run_calc(test_dir, 'job.ini', exports=fmt,
                                concurrent_tasks='0')
        else:
            out = self.run_calc(test_dir, 'job.ini', exports=fmt)
        got = out['disagg', fmt]
        self.assertEqual(len(expected), len(got))
        for fname, actual in zip(expected, got):
            self.assertEqualFiles('expected_output/%s' % fname, actual)
        return out

    @attr('qa', 'hazard', 'disagg')
    def test_case_1(self):
        out = self.assert_curves_ok(
            ['poe-0.02-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.02-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-PGA-10.1-40.1_Lon_Lat.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Mag_Dist.csv',
             'poe-0.1-rlz-0-SA(0.025)-10.1-40.1_Lon_Lat.csv'],
            case_1.__file__,
            fmt='csv')

        # check disagg_by_src, poe=0.02, 0.1, imt=PGA, SA(0.025)
        self.assertEqual(len(out['disagg_by_src', 'csv']), 4)
        for fname in out['disagg_by_src', 'csv']:
            self.assertEqualFiles('expected_output/%s' % strip_calc_id(fname),
                                  fname)

        # disaggregation by source group
        rlzs_assoc = self.calc.datastore['csm_info'].get_rlzs_assoc()
        pgetter = getters.PmapGetter(self.calc.datastore, rlzs_assoc)
        pgetter.init()
        pmaps = []
        for grp in sorted(pgetter.dstore['poes']):
            pmaps.append(pgetter.get_mean(grp))
        # make sure that the combination of the contributions is okay
        pmap = pgetter.get_mean()  # total mean map
        cmap = combine(pmaps)  # combination of the mean maps per source group
        for sid in pmap:
            numpy.testing.assert_almost_equal(pmap[sid].array, cmap[sid].array)

    @attr('qa', 'hazard', 'disagg')
    def test_case_2(self):
        # this is a case with disagg_outputs = Mag and 4 realizations
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.assert_curves_ok([
            'rlz-0-PGA--3.0--3.0.xml', 'rlz-0-PGA-0.0-0.0.xml',
            'rlz-1-PGA--3.0--3.0.xml', 'rlz-1-PGA-0.0-0.0.xml',
            'rlz-2-PGA-0.0-0.0.xml', 'rlz-3-PGA-0.0-0.0.xml'],
            case_2.__file__)

        # check that the CSV exporter does not break
        fnames = export(('disagg', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 6)  # number of CSV files

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 2)  # 2 sid x 1 key x 1 poe x 1 imt
        for fname in fnames:
            self.assertEqualFiles(
                'expected_output/%s' % strip_calc_id(fname), fname)

        # test predisagg
        self.run_calc(case_2.__file__, 'job.ini', calculation_mode='predisagg')
        self.assertEqual(list(self.calc.datastore['rup_data']),
                         ['Active Shallow Crust', 'Subduction IntraSlab'])

    @attr('qa', 'hazard', 'disagg')
    def test_case_3(self):
        with self.assertRaises(ValueError) as ctx:
            self.run_calc(case_3.__file__, 'job.ini')
        self.assertEqual(str(ctx.exception), '''\
You are trying to disaggregate for poe=0.1.
However the source model #0, 'source_model_test_complex.xml',
produces at most probabilities of 0.0362321 for rlz=#0, IMT=PGA.
The disaggregation PoE is too big or your model is wrong,
producing too small PoEs.''')

    @attr('qa', 'hazard', 'disagg')
    def test_case_4(self):
        # this is case with number of lon/lat bins different for site 0/site 1
        # this exercise sampling
        self.run_calc(case_4.__file__, 'job.ini')

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 64)  # 2 sid x 8 keys x 2 poe x 2 imt

    @attr('qa', 'hazard', 'disagg')
    def test_case_5(self):
        # this exercise gridded nonparametric sources
        self.run_calc(case_5.__file__, 'job.ini')

    @attr('qa', 'hazard', 'disagg')
    def test_case_master(self):
        # this tests exercise the case of a complex logic tree; it also
        # prints the warning on poe_agg very different from the expected poe
        self.run_calc(case_master.__file__, 'job.ini')
        fname = gettemp(view('mean_disagg', self.calc.datastore))
        self.assertEqualFiles('expected/mean_disagg.rst', fname)
        os.remove(fname)

        # check stats
        fnames = export(('disagg-stats', 'csv'), self.calc.datastore)
        self.assertEqual(len(fnames), 192)  # 2 sid x 8 keys x 2 poe x 2 imt
        # = 64 x 3 for mean, quantile-0.15, quantile-0.85

    @attr('qa', 'hazard', 'disagg')
    def test_disagg_by_src(self):
        # this is a case with iml_disagg and disagg_by_src
        self.run_calc(case_master.__file__, 'job1.ini')
        arr = self.calc.datastore[
            'disagg_by_src/iml-0.02-PGA--122.6-38.3'].value
        numpy.testing.assert_almost_equal(arr, [0.6757448, 0.1780308])
<|endoftext|>


<commit_before>from PLC.Method import Method
from PLC.Parameter import Parameter, Mixed
from PLC.Filter import Filter
from PLC.Auth import Auth
from PLC.Slices import Slice, Slices
from PLC.Nodes import Node, Nodes
from PLC.Methods.GetSlices import GetSlices

class SliceNodesList(GetSlices):
    """
    Deprecated. Can be implemented with GetSlices.

    """

    status = "deprecated"

    roles = ['admin', 'pi', 'user']

    accepts = [
        Auth(),
        Slice.fields['name']
        ]

    returns = [Node.fields['hostname']]

    def call(self, auth, slice_name):
	slices = GetSlices.call(self, auth, [slice_name])
	slice = slices[0]
	nodes = Nodes(self.api, slice['node_ids'])
	if not nodes:
	    return []

	node_hostnames = [node['hostname'] for node in nodes]

        return node_hostnames
<commit_msg>- call GetNodes<commit_after>from PLC.Method import Method
from PLC.Parameter import Parameter, Mixed
from PLC.Filter import Filter
from PLC.Auth import Auth
from PLC.Slices import Slice, Slices
from PLC.Nodes import Node, Nodes
from PLC.Methods.GetSlices import GetSlices
from PLC.Methods.GetNodes import GetNodes

class SliceNodesList(GetSlices, GetNodes):
    """
    Deprecated. Can be implemented with GetSlices and GetNodes.

    """

    status = "deprecated"

    roles = ['admin', 'pi', 'user']

    accepts = [
        Auth(),
        Slice.fields['name']
        ]

    returns = [Node.fields['hostname']]

    def call(self, auth, slice_name):
	slices = GetSlices.call(self, auth, [slice_name])
	if not slices:
	    return []

	slice = slices[0]
	nodes = GetNodes.call(self, auth, slice['node_ids'])
	if not nodes:
	    return []

	node_hostnames = [node['hostname'] for node in nodes]

        return node_hostnames
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Module to compute and plot an aggregate loss curve.
"""

import os

from openquake.logs import LOG
from openquake.output import curve
from openquake.risk import job as risk_job
from openquake.risk import probabilistic_event_based as prob

def _filename(job_id):
    """Return the name of the generated file."""
    return "%s-aggregate-loss-curve.svg" % job_id

def _for_plotting(loss_curve, time_span):
    """Translate a loss curve into a dictionary compatible to
    the interface defined in CurvePlot.write."""
    data = {}

    data["AggregateLossCurve"] = {}
    data["AggregateLossCurve"]["abscissa"] = tuple(loss_curve.abscissae)
    data["AggregateLossCurve"]["ordinate"] = tuple(loss_curve.ordinates)
    data["AggregateLossCurve"]["abscissa_property"] = "Economic Losses"
    data["AggregateLossCurve"]["ordinate_property"] = \
            "PoE in %s years" % (str(time_span))

    data["AggregateLossCurve"]["curve_title"] = "Aggregate Loss Curve"

    return data

def compute_aggregate_curve(job):
    """Compute and plot an aggreate loss curve.

    This function expects to find in kvs a set of pre computed
    GMFs and assets.

    This function is triggered only if the AGGREGATE_LOSS_CURVE
    parameter is specified in the configuration file.

    :param job: the job the engine is currently processing.
    :type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
    """
    print "compute_aggregate_curve"
    if not job.has("AGGREGATE_LOSS_CURVE"):
        LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
                "skipping aggregate loss curve computation...")

        return

    epsilon_provider = risk_job.EpsilonProvider(job.params)
    print "type(epsilon_provider)", type(epsilon_provider)
    aggregate_loss_curve = prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)

    path = os.path.join(job.params["BASE_PATH"],
            job.params["OUTPUT_DIR"], _filename(job.id))

    plotter = curve.CurvePlot(path)
    plotter.write(_for_plotting(
            aggregate_loss_curve.compute(),
            job.params["INVESTIGATION_TIME"]), autoscale_y=False)

    plotter.close()
    LOG.debug("Aggregate loss curve stored at %s" % path)
<commit_msg>removed debug statement<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Module to compute and plot an aggregate loss curve.
"""

import os

from openquake.logs import LOG
from openquake.output import curve
from openquake.risk import job as risk_job
from openquake.risk import probabilistic_event_based as prob

def _filename(job_id):
    """Return the name of the generated file."""
    return "%s-aggregate-loss-curve.svg" % job_id

def _for_plotting(loss_curve, time_span):
    """Translate a loss curve into a dictionary compatible to
    the interface defined in CurvePlot.write."""
    data = {}

    data["AggregateLossCurve"] = {}
    data["AggregateLossCurve"]["abscissa"] = tuple(loss_curve.abscissae)
    data["AggregateLossCurve"]["ordinate"] = tuple(loss_curve.ordinates)
    data["AggregateLossCurve"]["abscissa_property"] = "Economic Losses"
    data["AggregateLossCurve"]["ordinate_property"] = \
            "PoE in %s years" % (str(time_span))

    data["AggregateLossCurve"]["curve_title"] = "Aggregate Loss Curve"

    return data

def compute_aggregate_curve(job):
    """Compute and plot an aggreate loss curve.

    This function expects to find in kvs a set of pre computed
    GMFs and assets.

    This function is triggered only if the AGGREGATE_LOSS_CURVE
    parameter is specified in the configuration file.

    :param job: the job the engine is currently processing.
    :type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
    """
    print "compute_aggregate_curve"
    if not job.has("AGGREGATE_LOSS_CURVE"):
        LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
                "skipping aggregate loss curve computation...")

        return

    epsilon_provider = risk_job.EpsilonProvider(job.params)
    aggregate_loss_curve = prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)

    path = os.path.join(job.params["BASE_PATH"],
            job.params["OUTPUT_DIR"], _filename(job.id))

    plotter = curve.CurvePlot(path)
    plotter.write(_for_plotting(
            aggregate_loss_curve.compute(),
            job.params["INVESTIGATION_TIME"]), autoscale_y=False)

    plotter.close()
    LOG.debug("Aggregate loss curve stored at %s" % path)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django import template
from django.conf import settings
from django.utils import timezone
from opps.articles.models import ArticleBox

register = template.Library()

@register.simple_tag
def get_articlebox(slug, channel_slug=None, template_name=None):
    if channel_slug:
        slug = "{0}-{1}".format(slug, channel_slug)

    try:
        box = ArticleBox.objects.get(site=settings.SITE_ID, slug=slug,
                                     date_available__lte=timezone.now(),
                                     published=True)
    except ArticleBox.DoesNotExist:
        box = None

    t = template.loader.get_template('articles/articlebox_detail.html')
    if template_name:
        t = template.loader.get_template(template_name)

    return t.render(template.Context({'articlebox': box, 'slug': slug}))

@register.simple_tag
def get_all_articlebox(channel_slug, template_name=None):
    boxes = ArticleBox.objects.filter(site=settings.SITE_ID,
                                      date_available__lte=timezone.now(),
                                      published=True,
                                      channel__slug=channel_slug)

    t = template.loader.get_template('articles/articlebox_list.html')
    if template_name:
        t = template.loader.get_template(template_name)

    return t.render(template.Context({'articleboxes': boxes}))
<commit_msg>fix import ArticleBox on templatetag articles<commit_after># -*- coding: utf-8 -*-
from django import template
from django.conf import settings
from django.utils import timezone

from .models import ArticleBox

register = template.Library()

@register.simple_tag
def get_articlebox(slug, channel_slug=None, template_name=None):
    if channel_slug:
        slug = "{0}-{1}".format(slug, channel_slug)

    try:
        box = ArticleBox.objects.get(site=settings.SITE_ID, slug=slug,
                                     date_available__lte=timezone.now(),
                                     published=True)
    except ArticleBox.DoesNotExist:
        box = None

    t = template.loader.get_template('articles/articlebox_detail.html')
    if template_name:
        t = template.loader.get_template(template_name)

    return t.render(template.Context({'articlebox': box, 'slug': slug}))

@register.simple_tag
def get_all_articlebox(channel_slug, template_name=None):
    boxes = ArticleBox.objects.filter(site=settings.SITE_ID,
                                      date_available__lte=timezone.now(),
                                      published=True,
                                      channel__slug=channel_slug)

    t = template.loader.get_template('articles/articlebox_list.html')
    if template_name:
        t = template.loader.get_template(template_name)

    return t.render(template.Context({'articleboxes': boxes}))
<|endoftext|>


<commit_before>import os
from PyQt4 import QtGui

from Orange.widgets.settings import Setting
from Orange.widgets.widget import OWWidget
from Orange.widgets import gui
from Orange.widgets.data.owselectcolumns import VariablesListItemModel, VariablesListItemView
from orangecontrib.text.corpus import Corpus, get_sample_corpora_dir

class Output:
    CORPUS = "Corpus"

class OWLoadCorpus(OWWidget):
    name = "Corpus"
    description = "Load a corpus of text documents, (optionally) tagged with categories."
    icon = "icons/TextFile.svg"
    priority = 10

    outputs = [(Output.CORPUS, Corpus)]
    want_main_area = False
    resizing_enabled = False

    dlgFormats = "Only tab files (*.tab)"

    recent_files = Setting(["(none)"])

    def __init__(self):
        super().__init__()

        self.corpus = None

        # Refresh recent files
        self.recent_files = [fn for fn in self.recent_files
                             if os.path.exists(fn)]

        # Browse file box
        fbox = gui.widgetBox(self.controlArea, "Corpus file", orientation=0)

        # Drop-down for recent files
        self.file_combo = QtGui.QComboBox(fbox)
        self.file_combo.setMinimumWidth(300)
        fbox.layout().addWidget(self.file_combo)
        self.file_combo.activated[int].connect(self.select_file)

        # Browse button
        browse = gui.button(fbox, self, 'Browse', callback=self.browse_file)
        browse.setIcon(self.style().standardIcon(QtGui.QStyle.SP_DirOpenIcon))
        browse.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)

        # Reload button
        reload = gui.button(fbox, self, "Reload", callback=self.reload, default=True)
        reload.setIcon(self.style().standardIcon(QtGui.QStyle.SP_BrowserReload))
        reload.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)

        # Corpus info
        ibox = gui.widgetBox(self.controlArea, "Corpus info", addSpace=True)
        corp_info = "Corpus of 0 documents."
        self.info_label = gui.label(ibox, self, corp_info)

        # Used Text Features
        fbox = gui.widgetBox(self.controlArea, orientation=0)
        ubox = gui.widgetBox(fbox, "Used text features", addSpace=True)
        self.used_attrs = VariablesListItemModel()
        self.used_attrs_view = VariablesListItemView()
        self.used_attrs_view.setModel(self.used_attrs)
        ubox.layout().addWidget(self.used_attrs_view)

        aa = self.used_attrs
        aa.dataChanged.connect(self.update_feature_selection)
        aa.rowsInserted.connect(self.update_feature_selection)
        aa.rowsRemoved.connect(self.update_feature_selection)

        # Ignored Text Features
        ibox = gui.widgetBox(fbox, "Ignored text features", addSpace=True)
        self.unused_attrs = VariablesListItemModel()
        self.unused_attrs_view = VariablesListItemView()
        self.unused_attrs_view.setModel(self.unused_attrs)
        ibox.layout().addWidget(self.unused_attrs_view)

        # Load the most recent file
        self.set_file_list()
        if len(self.recent_files) > 0:
            self.open_file(self.recent_files[0])

    def set_file_list(self):
        self.file_combo.clear()
        if not self.recent_files:
            self.file_combo.addItem("(none)")
        for file in self.recent_files:
            if file == "(none)":
                self.file_combo.addItem("(none)")
            else:
                self.file_combo.addItem(os.path.split(file)[1])
        self.file_combo.addItem("Browse documentation corpora ...")

    def reload(self):
        if self.recent_files:
            return self.open_file(self.recent_files[0])

    def select_file(self, n):
        if n < len(self.recent_files) :
            name = self.recent_files[n]
            del self.recent_files[n]
            self.recent_files.insert(0, name)
        elif n:
            self.browse_file(True)

        if len(self.recent_files) > 0:
            self.set_file_list()
            self.open_file(self.recent_files[0])

    def browse_file(self, demos_loc=False):
        start_file = os.path.expanduser("~/")
        if demos_loc:
            start_file = get_sample_corpora_dir()
        filename = QtGui.QFileDialog.getOpenFileName(
            self, 'Open Orange Document Corpus', start_file, self.dlgFormats)
        if not filename:
            return
        if filename in self.recent_files:
            self.recent_files.remove(filename)
        self.recent_files.insert(0, filename)
        self.set_file_list()
        self.open_file(filename)

    def open_file(self, path):
        self.error(1, '')
        self.used_attrs[:] = []
        self.unused_attrs[:] = []

        try:
            self.corpus = Corpus.from_file(path)
            for f in self.corpus.domain.metas:
                if f in self.corpus.text_features:
                    self.used_attrs.append(f)
                else:
                    self.unused_attrs.append(f)
            self.info_label.setText("Corpus of {} documents.".format(len(self.corpus)))
            self.send(Output.CORPUS, self.corpus)
        except BaseException as err:
            self.error(1, str(err))

    def update_feature_selection(self):
        if self.corpus is not None:
            self.corpus.set_text_features(self.used_attrs)
            self.send(Output.CORPUS, self.corpus)
<commit_msg>OWLoadCorpus: Fixed 'Used features' loading<commit_after>import os
from PyQt4 import QtGui

from Orange.widgets.settings import Setting
from Orange.widgets.widget import OWWidget
from Orange.widgets import gui
from Orange.widgets.data.owselectcolumns import VariablesListItemModel, VariablesListItemView
from orangecontrib.text.corpus import Corpus, get_sample_corpora_dir

class Output:
    CORPUS = "Corpus"

class OWLoadCorpus(OWWidget):
    name = "Corpus"
    description = "Load a corpus of text documents, (optionally) tagged with categories."
    icon = "icons/TextFile.svg"
    priority = 10

    outputs = [(Output.CORPUS, Corpus)]
    want_main_area = False
    resizing_enabled = False

    dlgFormats = "Only tab files (*.tab)"

    recent_files = Setting(["(none)"])

    def __init__(self):
        super().__init__()

        self.corpus = None

        # Refresh recent files
        self.recent_files = [fn for fn in self.recent_files
                             if os.path.exists(fn)]

        # Browse file box
        fbox = gui.widgetBox(self.controlArea, "Corpus file", orientation=0)

        # Drop-down for recent files
        self.file_combo = QtGui.QComboBox(fbox)
        self.file_combo.setMinimumWidth(300)
        fbox.layout().addWidget(self.file_combo)
        self.file_combo.activated[int].connect(self.select_file)

        # Browse button
        browse = gui.button(fbox, self, 'Browse', callback=self.browse_file)
        browse.setIcon(self.style().standardIcon(QtGui.QStyle.SP_DirOpenIcon))
        browse.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)

        # Reload button
        reload = gui.button(fbox, self, "Reload", callback=self.reload, default=True)
        reload.setIcon(self.style().standardIcon(QtGui.QStyle.SP_BrowserReload))
        reload.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)

        # Corpus info
        ibox = gui.widgetBox(self.controlArea, "Corpus info", addSpace=True)
        corp_info = "Corpus of 0 documents."
        self.info_label = gui.label(ibox, self, corp_info)

        # Used Text Features
        fbox = gui.widgetBox(self.controlArea, orientation=0)
        ubox = gui.widgetBox(fbox, "Used text features", addSpace=True)
        self.used_attrs = VariablesListItemModel()
        self.used_attrs_view = VariablesListItemView()
        self.used_attrs_view.setModel(self.used_attrs)
        ubox.layout().addWidget(self.used_attrs_view)

        aa = self.used_attrs
        aa.dataChanged.connect(self.update_feature_selection)
        aa.rowsInserted.connect(self.update_feature_selection)
        aa.rowsRemoved.connect(self.update_feature_selection)

        # Ignored Text Features
        ibox = gui.widgetBox(fbox, "Ignored text features", addSpace=True)
        self.unused_attrs = VariablesListItemModel()
        self.unused_attrs_view = VariablesListItemView()
        self.unused_attrs_view.setModel(self.unused_attrs)
        ibox.layout().addWidget(self.unused_attrs_view)

        # Load the most recent file
        self.set_file_list()
        if len(self.recent_files) > 0:
            self.open_file(self.recent_files[0])

    def set_file_list(self):
        self.file_combo.clear()
        if not self.recent_files:
            self.file_combo.addItem("(none)")
        for file in self.recent_files:
            if file == "(none)":
                self.file_combo.addItem("(none)")
            else:
                self.file_combo.addItem(os.path.split(file)[1])
        self.file_combo.addItem("Browse documentation corpora ...")

    def reload(self):
        if self.recent_files:
            return self.open_file(self.recent_files[0])

    def select_file(self, n):
        if n < len(self.recent_files) :
            name = self.recent_files[n]
            del self.recent_files[n]
            self.recent_files.insert(0, name)
        elif n:
            self.browse_file(True)

        if len(self.recent_files) > 0:
            self.set_file_list()
            self.open_file(self.recent_files[0])

    def browse_file(self, demos_loc=False):
        start_file = os.path.expanduser("~/")
        if demos_loc:
            start_file = get_sample_corpora_dir()
        filename = QtGui.QFileDialog.getOpenFileName(
            self, 'Open Orange Document Corpus', start_file, self.dlgFormats)
        if not filename:
            return
        if filename in self.recent_files:
            self.recent_files.remove(filename)
        self.recent_files.insert(0, filename)
        self.set_file_list()
        self.open_file(filename)

    def open_file(self, path):
        self.error(1, '')
        self.used_attrs[:] = []
        self.unused_attrs[:] = []

        try:
            self.corpus = Corpus.from_file(path)
            self.info_label.setText("Corpus of {} documents.".format(len(self.corpus)))
            self.used_attrs.extend(self.corpus.text_features)
            self.unused_attrs.extend([f for f in self.corpus.domain.metas if f not in self.corpus.text_features])
        except BaseException as err:
            self.error(1, str(err))

    def update_feature_selection(self):
        if self.corpus is not None:
            self.corpus.set_text_features(self.used_attrs)
            self.send(Output.CORPUS, self.corpus)
<|endoftext|>