<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import numpy

from openquake.baselib import hdf5, datastore, parallel, performance
from openquake.baselib.general import group_array, humansize
from openquake.baselib.python3compat import zip, encode
from openquake.hazardlib.stats import set_rlzs_stats
from openquake.hazardlib.calc.stochastic import TWO32
from openquake.risklib import riskinput, scientific
from openquake.calculators import base, event_based, getters
from openquake.calculators.export.loss_curves import get_loss_builder

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
U64 = numpy.uint64

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter:
        a RuptureGetter instance
    :param srcfilter:
        a SourceFilter instance
    :param param:
        a dictionary of parameters
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :returns:
        an ArrayWrapper with shape (E, L, T, ...)
    """
    riskmodel = param['riskmodel']
    L = len(riskmodel.lti)
    with monitor('getting ruptures'):
        ebruptures = rupgetter.get_ruptures(srcfilter)
    with monitor('getting assetcol'):
        with datastore.read(rupgetter.hdf5path) as dstore:
            assetcol = dstore['assetcol']
        assets_by_site = assetcol.assets_by_site()
    getter = getters.GmfGetter(
        rupgetter.rlzs_by_gsim, ebruptures, srcfilter.sitecol,
        param['oqparam'], param['min_iml'])
    tagnames = param['aggregate_by']
    with monitor('getting hazard'):
        getter.init()  # instantiate the computers
        data = getter.get_hazard(rlzidx=False)  # (sid, eid, gmv)
        haz_by_sid = group_array(data, 'sid')
    eids = numpy.unique(data['eid'])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    shape = assetcol.agg_shape((len(eids), L), tagnames)
    acc = numpy.zeros(shape, F32)  # shape (E, L, T, ...)
    for loss_type, asset, eids_, loss_ratios in getter.gen_risk(
            assets_by_site, riskmodel, haz_by_sid):
        losses = asset.value(loss_type) * loss_ratios
        lti = riskmodel.lti[loss_type]
        tagi = assetcol.array[asset.ordinal][tagnames] if tagnames else ()
        tagidxs = tuple(idx - 1 for idx in tagi)
        for eid, loss in zip(eids_, losses):
            acc[(eid2idx[eid], lti) + tagidxs] += loss
    return hdf5.ArrayWrapper(acc, {'eids': eids})

def compute_loss_curves_maps(hdf5path, multi_index, clp, individual_curves,
                             monitor):
    with datastore.read(hdf5path) as dstore:
        oq = dstore['oqparam']
        stats = oq.risk_stats()
        builder = get_loss_builder(dstore)
        indices = dstore.get_attr('events', 'indices')
        R = len(indices)
        losses_by_event = dstore['losses_by_event']
        losses = [None] * R
        for r, (s1, s2) in enumerate(indices):
            idx = (slice(s1, s2),) + multi_index
            losses[r] = losses_by_event[idx]
    result = {'idx': multi_index}
    result['loss_curves/rlzs'], result['loss_curves/stats'] = (
        builder.build_pair(losses, stats))
    if R > 1 and individual_curves is False:
        del result['loss_curves/rlzs']
    if clp:
        result['loss_maps/rlzs'], result['loss_maps/stats'] = (
            builder.build_loss_maps(losses, clp, stats))
        if R > 1 and individual_curves is False:
            del result['loss_maps/rlzs']
    return result

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True

    def pre_execute(self, pre_calculator=None):
        super().pre_execute(pre_calculator)
        # save a copy of the assetcol in hdf5cache
        self.hdf5cache = self.datastore.hdf5cache()
        with hdf5.File(self.hdf5cache) as cache:
            cache['assetcol'] = self.assetcol
        self.param['aggregate_by'] = self.oqparam.aggregate_by
        # initialize the riskmodel
        self.riskmodel.taxonomy = self.assetcol.tagcol.taxonomy
        self.param['riskmodel'] = self.riskmodel

    def agg_dicts(self, dummy, arr):
        """
        :param dummy: unused parameter
        :param arr: ArrayWrapper with an attribute .eids
        """
        if 'losses_by_event' not in self.datastore:  # first time
            L = len(self.riskmodel.lti)
            shp = self.get_shape(len(self.eid2idx), L)
            logging.info('Creating losses_by_event of shape %s, %s', shp,
                         humansize(numpy.product(shp) * 4))
            self.datastore.create_dset('losses_by_event', F32, shp)
            self.oqparam.ground_motion_fields = False
        if len(arr):
            with self.monitor('saving losses_by_event', measuremem=True):
                lbe = self.datastore['losses_by_event']
                for eid, loss in zip(arr.eids, arr):
                    lbe[self.eid2idx[eid]] = loss
        return 1

    def get_shape(self, *sizes):
        return self.assetcol.agg_shape(sizes, self.oqparam.aggregate_by)

    def build_datasets(self, builder):
        oq = self.oqparam
        stats = oq.risk_stats()
        R = len(builder.weights)
        S = len(stats)
        L = len(self.riskmodel.lti)
        P = len(builder.return_periods)
        C = len(oq.conditional_loss_poes)
        if oq.individual_curves or R == 1:
            shp = self.get_shape(P, R, L)
            self.datastore.create_dset('loss_curves/rlzs', F32, shp)
            self.datastore.set_attrs(
                'loss_curves/rlzs', return_periods=builder.return_periods)
        if oq.conditional_loss_poes:
            shp = self.get_shape(C, R, L)
            self.datastore.create_dset('loss_maps/rlzs', F32, shp)
        if R > 1:
            shp = self.get_shape(P, S, L)
            self.datastore.create_dset('loss_curves/stats', F32, shp)
            self.datastore.set_attrs(
                'loss_curves/stats', return_periods=builder.return_periods,
                stats=[encode(name) for (name, func) in stats])
            if oq.conditional_loss_poes:
                shp = self.get_shape(C, S, L)
                self.datastore.create_dset('loss_maps/stats', F32, shp)
                self.datastore.set_attrs(
                    'loss_maps/stats',
                    stats=[encode(name) for (name, func) in stats])

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        logging.info('Building losses_by_rlz')
        with self.monitor('building losses_by_rlz', autoflush=True):
            self.build_losses_by_rlz()
        oq = self.oqparam
        builder = get_loss_builder(self.datastore)
        self.build_datasets(builder)
        mon = performance.Monitor(hdf5=hdf5.File(self.datastore.hdf5cache()))
        smap = parallel.Starmap(compute_loss_curves_maps, monitor=mon)
        first = self.datastore['losses_by_event'][0]  # to get the multi_index
        self.datastore.close()
        acc = []
        mon = self.monitor('building loss_curves and maps')
        with mon:
            for idx, _ in numpy.ndenumerate(first):
                smap.submit(self.datastore.hdf5path, idx,
                            oq.conditional_loss_poes, oq.individual_curves)
            for res in smap:
                idx = res.pop('idx')
                for name, arr in res.items():
                    if arr is not None:
                        acc.append((name, idx, arr))
        self.datastore.open('r+')  # reopen
        with mon:
            for name, idx, arr in acc:
                for ij, val in numpy.ndenumerate(arr):
                    self.datastore[name][ij + idx] = val
        mon.flush()

    def build_losses_by_rlz(self):
        indices = self.datastore.get_attr('events', 'indices')
        R = len(indices)
        dset = self.datastore['losses_by_event']
        lbr = self.datastore.create_dset(
            'avglosses_by_rlz', F32, (R,) + dset.shape[1:])
        for r, (s1, s2) in enumerate(indices):
            lbr[r] = dset[s1:s2].sum(axis=0) * self.oqparam.ses_ratio
<commit_msg>More monitoring<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import numpy

from openquake.baselib import hdf5, datastore, parallel, performance
from openquake.baselib.general import group_array, humansize
from openquake.baselib.python3compat import zip, encode
from openquake.hazardlib.stats import set_rlzs_stats
from openquake.hazardlib.calc.stochastic import TWO32
from openquake.risklib import riskinput, scientific
from openquake.calculators import base, event_based, getters
from openquake.calculators.export.loss_curves import get_loss_builder

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
U64 = numpy.uint64

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter:
        a RuptureGetter instance
    :param srcfilter:
        a SourceFilter instance
    :param param:
        a dictionary of parameters
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :returns:
        an ArrayWrapper with shape (E, L, T, ...)
    """
    riskmodel = param['riskmodel']
    L = len(riskmodel.lti)
    with monitor('getting ruptures'):
        ebruptures = rupgetter.get_ruptures(srcfilter)
    with monitor('getting assetcol'):
        with datastore.read(rupgetter.hdf5path) as dstore:
            assetcol = dstore['assetcol']
        assets_by_site = assetcol.assets_by_site()
    getter = getters.GmfGetter(
        rupgetter.rlzs_by_gsim, ebruptures, srcfilter.sitecol,
        param['oqparam'], param['min_iml'])
    tagnames = param['aggregate_by']
    with monitor('getting hazard'):
        getter.init()  # instantiate the computers
        data = getter.get_hazard(rlzidx=False)  # (sid, eid, gmv)
        haz_by_sid = group_array(data, 'sid')
    eids = numpy.unique(data['eid'])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    shape = assetcol.agg_shape((len(eids), L), tagnames)
    acc = numpy.zeros(shape, F32)  # shape (E, L, T, ...)
    for loss_type, asset, eids_, loss_ratios in getter.gen_risk(
            assets_by_site, riskmodel, haz_by_sid):
        losses = asset.value(loss_type) * loss_ratios
        lti = riskmodel.lti[loss_type]
        tagi = assetcol.array[asset.ordinal][tagnames] if tagnames else ()
        tagidxs = tuple(idx - 1 for idx in tagi)
        for eid, loss in zip(eids_, losses):
            acc[(eid2idx[eid], lti) + tagidxs] += loss
    return hdf5.ArrayWrapper(acc, {'eids': eids})

def compute_loss_curves_maps(hdf5path, multi_index, clp, individual_curves,
                             monitor):
    with datastore.read(hdf5path) as dstore:
        oq = dstore['oqparam']
        stats = oq.risk_stats()
        builder = get_loss_builder(dstore)
        indices = dstore.get_attr('events', 'indices')
        R = len(indices)
        losses_by_event = dstore['losses_by_event']
        losses = [None] * R
        for r, (s1, s2) in enumerate(indices):
            idx = (slice(s1, s2),) + multi_index
            losses[r] = losses_by_event[idx]
    result = {'idx': multi_index}
    result['loss_curves/rlzs'], result['loss_curves/stats'] = (
        builder.build_pair(losses, stats))
    if R > 1 and individual_curves is False:
        del result['loss_curves/rlzs']
    if clp:
        result['loss_maps/rlzs'], result['loss_maps/stats'] = (
            builder.build_loss_maps(losses, clp, stats))
        if R > 1 and individual_curves is False:
            del result['loss_maps/rlzs']
    return result

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True

    def pre_execute(self, pre_calculator=None):
        super().pre_execute(pre_calculator)
        # save a copy of the assetcol in hdf5cache
        self.hdf5cache = self.datastore.hdf5cache()
        with hdf5.File(self.hdf5cache) as cache:
            cache['assetcol'] = self.assetcol
        self.param['aggregate_by'] = self.oqparam.aggregate_by
        # initialize the riskmodel
        self.riskmodel.taxonomy = self.assetcol.tagcol.taxonomy
        self.param['riskmodel'] = self.riskmodel

    def agg_dicts(self, dummy, arr):
        """
        :param dummy: unused parameter
        :param arr: ArrayWrapper with an attribute .eids
        """
        if 'losses_by_event' not in self.datastore:  # first time
            L = len(self.riskmodel.lti)
            shp = self.get_shape(len(self.eid2idx), L)
            logging.info('Creating losses_by_event of shape %s, %s', shp,
                         humansize(numpy.product(shp) * 4))
            self.datastore.create_dset('losses_by_event', F32, shp)
            self.oqparam.ground_motion_fields = False
        if len(arr):
            with self.monitor('saving losses_by_event', measuremem=True):
                lbe = self.datastore['losses_by_event']
                for eid, loss in zip(arr.eids, arr):
                    lbe[self.eid2idx[eid]] = loss
        return 1

    def get_shape(self, *sizes):
        return self.assetcol.agg_shape(sizes, self.oqparam.aggregate_by)

    def build_datasets(self, builder):
        oq = self.oqparam
        stats = oq.risk_stats()
        R = len(builder.weights)
        S = len(stats)
        L = len(self.riskmodel.lti)
        P = len(builder.return_periods)
        C = len(oq.conditional_loss_poes)
        if oq.individual_curves or R == 1:
            shp = self.get_shape(P, R, L)
            self.datastore.create_dset('loss_curves/rlzs', F32, shp)
            self.datastore.set_attrs(
                'loss_curves/rlzs', return_periods=builder.return_periods)
        if oq.conditional_loss_poes:
            shp = self.get_shape(C, R, L)
            self.datastore.create_dset('loss_maps/rlzs', F32, shp)
        if R > 1:
            shp = self.get_shape(P, S, L)
            self.datastore.create_dset('loss_curves/stats', F32, shp)
            self.datastore.set_attrs(
                'loss_curves/stats', return_periods=builder.return_periods,
                stats=[encode(name) for (name, func) in stats])
            if oq.conditional_loss_poes:
                shp = self.get_shape(C, S, L)
                self.datastore.create_dset('loss_maps/stats', F32, shp)
                self.datastore.set_attrs(
                    'loss_maps/stats',
                    stats=[encode(name) for (name, func) in stats])

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        logging.info('Building losses_by_rlz')
        with self.monitor('building avglosses_by_rlz', autoflush=True):
            self.build_avglosses_by_rlz()
        oq = self.oqparam
        builder = get_loss_builder(self.datastore)
        self.build_datasets(builder)
        mon = performance.Monitor(hdf5=hdf5.File(self.datastore.hdf5cache()))
        smap = parallel.Starmap(compute_loss_curves_maps, monitor=mon)
        first = self.datastore['losses_by_event'][0]  # to get the multi_index
        self.datastore.close()
        acc = []
        for idx, _ in numpy.ndenumerate(first):
            smap.submit(self.datastore.hdf5path, idx,
                        oq.conditional_loss_poes, oq.individual_curves)
        for res in smap:
            idx = res.pop('idx')
            for name, arr in res.items():
                if arr is not None:
                    acc.append((name, idx, arr))
        # copy performance information from the cache to the datastore
        pd = mon.hdf5['performance_data'].value
        hdf5.extend3(self.datastore.hdf5path, 'performance_data', pd)
        self.datastore.open('r+')  # reopen
        self.datastore['task_info/compute_loss_curves_and_maps'] = (
            mon.hdf5['task_info/compute_loss_curves_maps'].value)
        with self.monitor('saving loss_curves and maps', autoflush=True):
            for name, idx, arr in acc:
                for ij, val in numpy.ndenumerate(arr):
                    self.datastore[name][ij + idx] = val

    def build_avglosses_by_rlz(self):
        """
        Build the dataset avglosses_by_rlz from losses_by_event
        """
        indices = self.datastore.get_attr('events', 'indices')
        R = len(indices)
        dset = self.datastore['losses_by_event']
        lbr = self.datastore.create_dset(
            'avglosses_by_rlz', F32, (R,) + dset.shape[1:])
        for r, (s1, s2) in enumerate(indices):
            lbr[r] = dset[s1:s2].sum(axis=0) * self.oqparam.ses_ratio
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.hazardlib import InvalidFile
from openquake.hazardlib.calc.filters import getdefault
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    """
    :param gmfs: an array of GMFs with fields sid, eid, gmv
    :param param: a dictionary of parameters coming from the job.ini
    :param monitor: a Monitor instance
    :returns: a dictionary of arrays with keys elt, alt, losses_by_A, ...
    """
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assets_df = dstore.read_df('assetcol/array', 'ordinal')
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    # aggkey -> eid -> loss
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    lba.alt = general.AccumDict(
        accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    lba.losses_by_E = numpy.zeros((E, L), F32)
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    for lt, lti in crmodel.lti.items():
        val = param['minimum_asset_loss'][lt]
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    haz_by_sid = general.group_array(gmfs, 'sid')
    for sid, asset_df in assets_df.groupby('site_id'):
        try:
            haz = haz_by_sid[sid]
        except KeyError:  # no hazard here
            continue
        with mon_risk:
            assets = asset_df.to_records()  # fast
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            else:
                ws = None
            assets_by_taxo = get_assets_by_taxo(assets, tempname)  # fast
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])  # fast
            out = get_output(crmodel, assets_by_taxo, haz)  # slow
        with mon_agg:
            tagidxs = assets[aggby] if aggby else None
            acc['numlosses'] += lba.aggregate(
                out, eidx, minimum_loss, tagidxs, ws)
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, lba.losses_by_E) if losses.sum()),
        elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid, ultra-fast
        ((eid, eid2rlz[eid], loss) for eid, loss in lba.alt[idx].items()),
        elt_dt) for idx in lba.alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'],
                           param['amplifier'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.rupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    alltags = [getattr(tagcol, tagname) for tagname in aggby]
    ranges = [range(1, len(tags)) for tags in alltags]
    for idxs in itertools.product(*ranges):
        d = {name: tags[idx] for idx, name, tags in zip(idxs, aggby, alltags)}
        yield idxs, d

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        mal = {lt: getdefault(oq.minimum_asset_loss, lt)
               for lt in oq.loss_names}
        logging.info('minimum_asset_loss=%s', mal)
        if oq.aggregate_by and self.E * A > oq.max_potential_gmfs and any(
                val == 0 for val in mal.values()):
            logging.warning('The calculation is really big; you should set '
                            'minimum_asset_loss')
        self.param['minimum_asset_loss'] = mal

        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs, attrs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt,
                                       attrs=attrs)
        self.param.pop('oqparam', None)  # unneeded
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        parent = self.datastore.parent
        full_lt = parent['full_lt'] if parent else self.full_lt
        self.init_logic_tree(full_lt)
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info('Sending %d ruptures', len(self.datastore['ruptures']))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(self.datastore, srcfilter):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stats'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<commit_msg>Better logging [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.hazardlib import InvalidFile
from openquake.hazardlib.calc.filters import getdefault
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    """
    :param gmfs: an array of GMFs with fields sid, eid, gmv
    :param param: a dictionary of parameters coming from the job.ini
    :param monitor: a Monitor instance
    :returns: a dictionary of arrays with keys elt, alt, losses_by_A, ...
    """
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assets_df = dstore.read_df('assetcol/array', 'ordinal')
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    # aggkey -> eid -> loss
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    lba.alt = general.AccumDict(
        accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    lba.losses_by_E = numpy.zeros((E, L), F32)
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    for lt, lti in crmodel.lti.items():
        val = param['minimum_asset_loss'][lt]
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    haz_by_sid = general.group_array(gmfs, 'sid')
    for sid, asset_df in assets_df.groupby('site_id'):
        try:
            haz = haz_by_sid[sid]
        except KeyError:  # no hazard here
            continue
        with mon_risk:
            assets = asset_df.to_records()  # fast
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            else:
                ws = None
            assets_by_taxo = get_assets_by_taxo(assets, tempname)  # fast
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])  # fast
            out = get_output(crmodel, assets_by_taxo, haz)  # slow
        with mon_agg:
            tagidxs = assets[aggby] if aggby else None
            acc['numlosses'] += lba.aggregate(
                out, eidx, minimum_loss, tagidxs, ws)
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, lba.losses_by_E) if losses.sum()),
        elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid, ultra-fast
        ((eid, eid2rlz[eid], loss) for eid, loss in lba.alt[idx].items()),
        elt_dt) for idx in lba.alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'],
                           param['amplifier'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.rupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    alltags = [getattr(tagcol, tagname) for tagname in aggby]
    ranges = [range(1, len(tags)) for tags in alltags]
    for idxs in itertools.product(*ranges):
        d = {name: tags[idx] for idx, name, tags in zip(idxs, aggby, alltags)}
        yield idxs, d

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        mal = {lt: getdefault(oq.minimum_asset_loss, lt)
               for lt in oq.loss_names}
        logging.info('minimum_asset_loss=%s', mal)
        if oq.aggregate_by and self.E * A > oq.max_potential_gmfs and any(
                val == 0 for val in mal.values()):
            logging.warning('The calculation is really big; you should set '
                            'minimum_asset_loss')
        self.param['minimum_asset_loss'] = mal

        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs, attrs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt,
                                       attrs=attrs)
        self.param.pop('oqparam', None)  # unneeded
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        parent = self.datastore.parent
        full_lt = parent['full_lt'] if parent else self.full_lt
        self.init_logic_tree(full_lt)
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info(
            'Sending {:_d} ruptures'.format(len(self.datastore['ruptures'])))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(self.datastore, srcfilter):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stats'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<|endoftext|>


<commit_before>"""
Django settings for superlists project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'oodrl8-5r$*eq839to*4vdm49d^#@elsxkz7k$5#-u=aj87ou0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'lists',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'superlists.urls'

WSGI_APPLICATION = 'superlists.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>set static root in settings and disabled admin<commit_after>"""
Django settings for superlists project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'oodrl8-5r$*eq839to*4vdm49d^#@elsxkz7k$5#-u=aj87ou0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    # 'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'lists',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'superlists.urls'

WSGI_APPLICATION = 'superlists.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.abspath(os.path.join(BASE_DIR, '../static'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assetcol = dstore['assetcol']
        assets_by_site = assetcol.assets_by_site()
        exposed_values = dstore['exposed_values/agg'][()]
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    alt = general.AccumDict(accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    # aggkey -> eid -> loss
    arr = numpy.zeros((E, L), F32)
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    fraction = param['minimum_loss_fraction'] / len(assetcol)
    for lt, lti in crmodel.lti.items():
        val = exposed_values[lti] * fraction
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    for sid, haz in general.group_array(gmfs, 'sid').items():
        assets_on_sid = assets_by_site[sid]
        if len(assets_on_sid) == 0:
            continue
        with mon_risk:
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            assets_by_taxo = get_assets_by_taxo(assets_on_sid, tempname)
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])
            out = get_output(crmodel, assets_by_taxo, haz)
        for lti, lt in enumerate(crmodel.loss_types):
            lratios = out[lt]
            if lt == 'occupants':
                field = 'occupants_None'
            else:
                field = 'value-' + lt
            for a, asset in enumerate(assets_on_sid):
                if aggby:
                    idx = ','.join(map(str, asset[aggby]))
                aid = asset['ordinal']
                ls = asset[field] * lratios[a]
                for loss_idx, losses in lba.compute(asset, ls, lt):
                    kept = 0
                    with mon_agg:
                        if aggby:
                            for loss, eid in zip(losses, out.eids):
                                if loss >= minimum_loss[loss_idx]:
                                    alt[idx][eid][loss_idx] += loss
                                    kept += 1
                        arr[eidx, loss_idx] += losses
                    if param['avg_losses']:  # this is really fast
                        lba.losses_by_A[aid, loss_idx] += losses @ ws
                    acc['numlosses'] += numpy.array([kept, len(losses)])
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, arr) if losses.sum()), elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid
        ((eid, eid2rlz[eid], loss) for eid, loss in alt[idx].items()),
        elt_dt) for idx in alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.rupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    ranges = [range(1, len(getattr(tagcol, tagname))) for tagname in aggby]
    for idxs in itertools.product(*ranges):
        yield idxs

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk', 'ucerf_hazard']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.param['minimum_loss_fraction'] = oq.minimum_loss_fraction

        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt)
        self.param.pop('oqparam', None)  # unneeded
        A = len(self.assetcol)
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        parent = self.datastore.parent
        csm_info = parent['csm_info'] if parent else self.csm_info
        self.init_logic_tree(csm_info)
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info('Sending %d ruptures', len(self.datastore['ruptures']))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(
                self.datastore, srcfilter=srcfilter):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stats'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<commit_msg>Optimized the management of the assets in ebrisk by using pandas [skip hazardlib][demos]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    """
    :param gmfs: an array of GMFs with fields sid, eid, gmv
    :param param: a dictionary of parameters coming from the job.ini
    :param monitor: a Monitor instance
    :returns: a dictionary of arrays with keys elt, alt, losses_by_A, ...
    """
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assets_df = dstore.read_df('assetcol/array', 'ordinal')
        exposed_values = dstore['exposed_values/agg'][()]
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    alt = general.AccumDict(accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    # aggkey -> eid -> loss
    arr = numpy.zeros((E, L), F32)
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    fraction = param['minimum_loss_fraction'] / len(assets_df)
    for lt, lti in crmodel.lti.items():
        val = exposed_values[lti] * fraction
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    haz_by_sid = general.group_array(gmfs, 'sid')
    for sid, asset_df in assets_df.groupby('site_id'):
        try:
            haz = haz_by_sid[sid]
        except KeyError:  # no hazard here
            continue
        assets = asset_df.to_records()
        with mon_risk:
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            assets_by_taxo = get_assets_by_taxo(assets, tempname)  # fast
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])
            out = get_output(crmodel, assets_by_taxo, haz)
        with mon_agg:
            for lti, lt in enumerate(crmodel.loss_types):
                lratios = out[lt]
                if lt == 'occupants':
                    field = 'occupants_None'
                else:
                    field = 'value-' + lt
                if aggby:
                    tagidxs = assets[aggby]
                for a, asset in enumerate(assets):
                    if aggby:
                        idx = ','.join(map(str, tagidxs[a]))
                    aid = asset['ordinal']
                    ls = asset[field] * lratios[a]
                    for loss_idx, losses in lba.compute(asset, ls, lt):
                        kept = 0
                        if aggby:
                            for loss, eid in zip(losses, out.eids):
                                if loss >= minimum_loss[loss_idx]:
                                    alt[idx][eid][loss_idx] += loss
                                    kept += 1
                        arr[eidx, loss_idx] += losses
                        if param['avg_losses']:  # this is really fast
                            lba.losses_by_A[aid, loss_idx] += losses @ ws
                        acc['numlosses'] += numpy.array([kept, len(losses)])
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, arr) if losses.sum()), elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid, ultra-fast
        ((eid, eid2rlz[eid], loss) for eid, loss in alt[idx].items()),
        elt_dt) for idx in alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.rupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    ranges = [range(1, len(getattr(tagcol, tagname))) for tagname in aggby]
    for idxs in itertools.product(*ranges):
        yield idxs

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk', 'ucerf_hazard']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.param['minimum_loss_fraction'] = oq.minimum_loss_fraction

        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt)
        self.param.pop('oqparam', None)  # unneeded
        A = len(self.assetcol)
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        parent = self.datastore.parent
        csm_info = parent['csm_info'] if parent else self.csm_info
        self.init_logic_tree(csm_info)
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info('Sending %d ruptures', len(self.datastore['ruptures']))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(
                self.datastore, srcfilter=srcfilter):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stats'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2013-2015, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import Gaffer
import GafferUI

from Qt import QtWidgets

## A simple PlugValueWidget which just displays the name of the plug,
# with the popup action menu for the plug.
#
# Supported plug metadata :
#
#  - "labelPlugValueWidget:renameable"
class LabelPlugValueWidget( GafferUI.PlugValueWidget ) :

	def __init__( self, plug, horizontalAlignment=GafferUI.Label.HorizontalAlignment.Left, verticalAlignment=GafferUI.Label.VerticalAlignment.Center, **kw ) :

		GafferUI.PlugValueWidget.__init__( self, QtWidgets.QWidget(), plug, **kw )

		layout = QtWidgets.QHBoxLayout()
		layout.setContentsMargins( 0, 0, 0, 0 )
		layout.setSizeConstraint( QtWidgets.QLayout.SetMinAndMaxSize )
		self._qtWidget().setLayout( layout )

		self.__label = GafferUI.NameLabel(
			plug,
			horizontalAlignment = horizontalAlignment,
			verticalAlignment = verticalAlignment,
		)
		self.__label._qtWidget().setObjectName( "gafferPlugLabel" )
		layout.addWidget( self.__label._qtWidget() )

		self.__editableLabel = None # we'll make this lazily as needed

		# connecting at group 0 so we're called before the slots
		# connected by the NameLabel class.
		self.__label.dragBeginSignal().connect( 0, Gaffer.WeakMethod( self.__dragBegin ), scoped = False )
		self.__label.dragEndSignal().connect( 0, Gaffer.WeakMethod( self.__dragEnd ), scoped = False )

		Gaffer.Metadata.plugValueChangedSignal().connect( Gaffer.WeakMethod( self.__plugMetadataChanged ), scoped = False )

		self._addPopupMenu( self.__label )

		self.setPlug( plug )

	def label( self ) :

		return self.__label

	def setPlug( self, plug ) :

		GafferUI.PlugValueWidget.setPlug( self, plug )

		self.__label.setGraphComponent( plug )
		if self.__editableLabel is not None :
			self.__editableLabel.setGraphComponent( plug )

		self.__updateFormatter()
		self.__updateDoubleClickConnection()

	def setHighlighted( self, highlighted ) :

		GafferUI.PlugValueWidget.setHighlighted( self, highlighted )

		self.__label.setHighlighted( highlighted )

	def getToolTip( self ) :

		result = GafferUI.PlugValueWidget.getToolTip( self )

		if self.getPlug() is not None :
			if result :
				result += "\n"
			result += "## Actions\n\n"
			result += "- Left drag to connect\n"
			if hasattr( self.getPlug(), "getValue" ) :
				result += "- Shift+left or middle drag to transfer value"

		return result

	def _updateFromPlug( self ) :

		plug = self.getPlug()

		valueChanged = plug.getInput() is not None
		if not valueChanged and isinstance( plug, Gaffer.ValuePlug ) :
			with self.getContext() :
				if Gaffer.NodeAlgo.hasUserDefault( plug ) :
					try:
						valueChanged = not Gaffer.NodeAlgo.isSetToUserDefault( plug )
					except:
						# an error here should not cause the ui to break, specially since the value widget corresponding could be indicating the error itself
						valueChanged = True
				else :
					try:
						valueChanged = not plug.isSetToDefault()
					except:
						# an error here should not cause the ui to break, specially since the value widget corresponding could be indicating the error itself
						valueChanged = True

		self.__setValueChanged( valueChanged )

	# Sets whether or not the label be rendered in a ValueChanged state.
	def __setValueChanged( self, valueChanged ) :

		if valueChanged == self.__getValueChanged() :
			return

		self.__label._qtWidget().setProperty( "gafferValueChanged", GafferUI._Variant.toVariant( valueChanged ) )
		self.__label._repolish()

	def __getValueChanged( self ) :

		if "gafferValueChanged" not in self.__label._qtWidget().dynamicPropertyNames() :
			return False

		return GafferUI._Variant.fromVariant( self.__label._qtWidget().property( "gafferValueChanged" ) )

	def __dragBegin( self, widget, event ) :

		# initiate a drag containing the value of the plug
		# for shift-left drag or a middle drag. initiate a
		# drag containing the plug for a straight left-drag.

		shift = event.modifiers & event.Modifiers.Shift
		left = event.buttons == event.Buttons.Left
		middle = event.buttons == event.Buttons.Middle
		if ( shift and left ) or middle :
			if not hasattr( self.getPlug(), "getValue" ) :
				return None
			GafferUI.Pointer.setCurrent( "values" )
			with self.getContext() :
				return self.getPlug().getValue()
		elif left :
			GafferUI.Pointer.setCurrent( "plug" )
			return self.getPlug()

	def __dragEnd( self, widget, event ) :

		GafferUI.Pointer.setCurrent( None )

	def __updateFormatter( self ) :

		plug = self.getPlug()
		label = Gaffer.Metadata.value( plug, "label" ) if plug is not None else None
		if label is not None :
			self.__label.setFormatter( lambda graphComponents : label )
		else :
			self.__label.setFormatter( self.__label.defaultFormatter )

	def __updateDoubleClickConnection( self ) :

		self.__labelDoubleClickConnection = None

		if self.getPlug() is None or not Gaffer.Metadata.value( self.getPlug(), "labelPlugValueWidget:renameable" ) :
			return

		self.__labelDoubleClickConnection = self.__label.buttonDoubleClickSignal().connect( Gaffer.WeakMethod( self.__labelDoubleClicked ) )

	def __labelDoubleClicked( self, label, event ) :

		assert( label is self.__label )

		if Gaffer.MetadataAlgo.readOnly( self.getPlug() ) :
			return

		if self.__editableLabel is None :
			self.__editableLabel = GafferUI.NameWidget( self.getPlug() )
			self.__editableLabel._qtWidget().setMinimumSize( self.label()._qtWidget().minimumSize() )
			self.__editableLabel._qtWidget().setMaximumSize( self.label()._qtWidget().maximumSize() )
			# Connect at group 0 so we're called before the NameWidget's own slots.
			self.__labelEditingFinishedConnection = self.__editableLabel.editingFinishedSignal().connect( 0, Gaffer.WeakMethod( self.__labelEditingFinished ) )
			self._qtWidget().layout().insertWidget( 0, self.__editableLabel._qtWidget() )

		self.__label.setVisible( False )
		self.__editableLabel.setVisible( True )
		self.__editableLabel.setSelection( 0, len( self.__editableLabel.getText() ) )
		self.__editableLabel.grabFocus()

	def __labelEditingFinished( self, nameWidget ) :

		with Gaffer.UndoScope( self.getPlug().ancestor( Gaffer.ScriptNode ) ) :
			# Do what the NameWidget would have done for us anyway, so we
			# can group it with the metadata deregistration in the undo queue.
			self.getPlug().setName( nameWidget.getText() )
			# Remove any metadata label which would mask the name - if a user
			# has gone to the trouble of setting a sensible name, then it should
			# take precedence.
			Gaffer.Metadata.deregisterValue( self.getPlug(), "label" )

		self.__label.setVisible( True )
		self.__editableLabel.setVisible( False )

		# Return True so that the NameWidget's handler isn't run, since we
		# did all the work ourselves.
		return True

	def __plugMetadataChanged( self, nodeTypeId, plugPath, key, plug ) :

		if self.getPlug() is None :
			return

		if key=="label" and Gaffer.MetadataAlgo.affectedByChange( self.getPlug(), nodeTypeId, plugPath, plug ) :
			self.__updateFormatter()
<commit_msg>LabelPlugValueWidget : Support dynamic "label" metadata that uses `getName()`<commit_after>##########################################################################
#
#  Copyright (c) 2013-2015, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import Gaffer
import GafferUI

from Qt import QtWidgets

## A simple PlugValueWidget which just displays the name of the plug,
# with the popup action menu for the plug.
#
# Supported plug metadata :
#
#  - "labelPlugValueWidget:renameable"
class LabelPlugValueWidget( GafferUI.PlugValueWidget ) :

	def __init__( self, plug, horizontalAlignment=GafferUI.Label.HorizontalAlignment.Left, verticalAlignment=GafferUI.Label.VerticalAlignment.Center, **kw ) :

		GafferUI.PlugValueWidget.__init__( self, QtWidgets.QWidget(), plug, **kw )

		layout = QtWidgets.QHBoxLayout()
		layout.setContentsMargins( 0, 0, 0, 0 )
		layout.setSizeConstraint( QtWidgets.QLayout.SetMinAndMaxSize )
		self._qtWidget().setLayout( layout )

		self.__label = GafferUI.NameLabel(
			plug,
			horizontalAlignment = horizontalAlignment,
			verticalAlignment = verticalAlignment,
			formatter = self.__formatter,
		)
		self.__label._qtWidget().setObjectName( "gafferPlugLabel" )
		layout.addWidget( self.__label._qtWidget() )

		self.__editableLabel = None # we'll make this lazily as needed

		# connecting at group 0 so we're called before the slots
		# connected by the NameLabel class.
		self.__label.dragBeginSignal().connect( 0, Gaffer.WeakMethod( self.__dragBegin ), scoped = False )
		self.__label.dragEndSignal().connect( 0, Gaffer.WeakMethod( self.__dragEnd ), scoped = False )

		Gaffer.Metadata.plugValueChangedSignal().connect( Gaffer.WeakMethod( self.__plugMetadataChanged ), scoped = False )

		self._addPopupMenu( self.__label )

		self.setPlug( plug )

	def label( self ) :

		return self.__label

	def setPlug( self, plug ) :

		GafferUI.PlugValueWidget.setPlug( self, plug )

		self.__label.setGraphComponent( plug )
		if self.__editableLabel is not None :
			self.__editableLabel.setGraphComponent( plug )

		self.__updateDoubleClickConnection()

	def setHighlighted( self, highlighted ) :

		GafferUI.PlugValueWidget.setHighlighted( self, highlighted )

		self.__label.setHighlighted( highlighted )

	def getToolTip( self ) :

		result = GafferUI.PlugValueWidget.getToolTip( self )

		if self.getPlug() is not None :
			if result :
				result += "\n"
			result += "## Actions\n\n"
			result += "- Left drag to connect\n"
			if hasattr( self.getPlug(), "getValue" ) :
				result += "- Shift+left or middle drag to transfer value"

		return result

	def _updateFromPlug( self ) :

		plug = self.getPlug()

		valueChanged = plug.getInput() is not None
		if not valueChanged and isinstance( plug, Gaffer.ValuePlug ) :
			with self.getContext() :
				if Gaffer.NodeAlgo.hasUserDefault( plug ) :
					try:
						valueChanged = not Gaffer.NodeAlgo.isSetToUserDefault( plug )
					except:
						# an error here should not cause the ui to break, specially since the value widget corresponding could be indicating the error itself
						valueChanged = True
				else :
					try:
						valueChanged = not plug.isSetToDefault()
					except:
						# an error here should not cause the ui to break, specially since the value widget corresponding could be indicating the error itself
						valueChanged = True

		self.__setValueChanged( valueChanged )

	# Sets whether or not the label be rendered in a ValueChanged state.
	def __setValueChanged( self, valueChanged ) :

		if valueChanged == self.__getValueChanged() :
			return

		self.__label._qtWidget().setProperty( "gafferValueChanged", GafferUI._Variant.toVariant( valueChanged ) )
		self.__label._repolish()

	def __getValueChanged( self ) :

		if "gafferValueChanged" not in self.__label._qtWidget().dynamicPropertyNames() :
			return False

		return GafferUI._Variant.fromVariant( self.__label._qtWidget().property( "gafferValueChanged" ) )

	def __dragBegin( self, widget, event ) :

		# initiate a drag containing the value of the plug
		# for shift-left drag or a middle drag. initiate a
		# drag containing the plug for a straight left-drag.

		shift = event.modifiers & event.Modifiers.Shift
		left = event.buttons == event.Buttons.Left
		middle = event.buttons == event.Buttons.Middle
		if ( shift and left ) or middle :
			if not hasattr( self.getPlug(), "getValue" ) :
				return None
			GafferUI.Pointer.setCurrent( "values" )
			with self.getContext() :
				return self.getPlug().getValue()
		elif left :
			GafferUI.Pointer.setCurrent( "plug" )
			return self.getPlug()

	def __dragEnd( self, widget, event ) :

		GafferUI.Pointer.setCurrent( None )

	def __updateDoubleClickConnection( self ) :

		self.__labelDoubleClickConnection = None

		if self.getPlug() is None or not Gaffer.Metadata.value( self.getPlug(), "labelPlugValueWidget:renameable" ) :
			return

		self.__labelDoubleClickConnection = self.__label.buttonDoubleClickSignal().connect( Gaffer.WeakMethod( self.__labelDoubleClicked ) )

	def __labelDoubleClicked( self, label, event ) :

		assert( label is self.__label )

		if Gaffer.MetadataAlgo.readOnly( self.getPlug() ) :
			return

		if self.__editableLabel is None :
			self.__editableLabel = GafferUI.NameWidget( self.getPlug() )
			self.__editableLabel._qtWidget().setMinimumSize( self.label()._qtWidget().minimumSize() )
			self.__editableLabel._qtWidget().setMaximumSize( self.label()._qtWidget().maximumSize() )
			# Connect at group 0 so we're called before the NameWidget's own slots.
			self.__labelEditingFinishedConnection = self.__editableLabel.editingFinishedSignal().connect( 0, Gaffer.WeakMethod( self.__labelEditingFinished ) )
			self._qtWidget().layout().insertWidget( 0, self.__editableLabel._qtWidget() )

		self.__label.setVisible( False )
		self.__editableLabel.setVisible( True )
		self.__editableLabel.setSelection( 0, len( self.__editableLabel.getText() ) )
		self.__editableLabel.grabFocus()

	def __labelEditingFinished( self, nameWidget ) :

		with Gaffer.UndoScope( self.getPlug().ancestor( Gaffer.ScriptNode ) ) :
			# Do what the NameWidget would have done for us anyway, so we
			# can group it with the metadata deregistration in the undo queue.
			self.getPlug().setName( nameWidget.getText() )
			# Remove any metadata label which would mask the name - if a user
			# has gone to the trouble of setting a sensible name, then it should
			# take precedence.
			Gaffer.Metadata.deregisterValue( self.getPlug(), "label" )

		self.__label.setVisible( True )
		self.__editableLabel.setVisible( False )

		# Return True so that the NameWidget's handler isn't run, since we
		# did all the work ourselves.
		return True

	def __plugMetadataChanged( self, nodeTypeId, plugPath, key, plug ) :

		if self.getPlug() is None :
			return

		if key=="label" and Gaffer.MetadataAlgo.affectedByChange( self.getPlug(), nodeTypeId, plugPath, plug ) :
			# The NameLabel doesn't know that our formatter is sensitive
			# to the metadata, so give it a little kick.
			self.__label.setFormatter( self.__formatter )

	@staticmethod
	def __formatter( graphComponents ) :

		if graphComponents :
			label = Gaffer.Metadata.value( graphComponents[-1], "label" )
			if label is not None :
				return label

		return GafferUI.NameLabel.defaultFormatter( graphComponents )
<|endoftext|>