<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (C) 2016-2017 GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import sys
import time
import socket
import os.path
import sqlite3
import logging
import threading
import subprocess

from openquake.baselib import config, sap, zeromq as z
from openquake.baselib.parallel import safely_call
from openquake.commonlib import logs
from openquake.server.db import actions
from openquake.server import dbapi
from openquake.server import __file__ as server_path
from openquake.server.settings import DATABASE

db = dbapi.Db(sqlite3.connect, DATABASE['NAME'], isolation_level=None,
              detect_types=sqlite3.PARSE_DECLTYPES, timeout=20)
# NB: I am increasing the timeout from 5 to 20 seconds to see if the random
# OperationalError: "database is locked" disappear in the WebUI tests

class DbServer(object):
    """
    A server collecting the received commands into a queue
    """
    def __init__(self, db, address, num_workers=5):
        self.db = db
        self.frontend = 'tcp://%s:%s' % address
        self.backend = 'inproc://dbworkers'
        self.num_workers = num_workers
        self.pid = os.getpid()

    def worker(self, sock):
        for cmd_ in sock:
            cmd, args = cmd_[0], cmd_[1:]
            if cmd == 'getpid':
                sock.rep((self.pid, None, None))
                continue
            try:
                func = getattr(actions, cmd)
            except AttributeError:
                sock.rep(('Invalid command ' + cmd, ValueError, None))
            else:
                sock.rep(safely_call(func, (self.db,) + args))

    def start(self):
        # start workers
        workers = []
        for _ in range(self.num_workers):
            sock = z.Socket(self.backend, z.zmq.REP, 'connect')
            threading.Thread(target=self.worker, args=(sock,)).start()
            workers.append(sock)
        logging.warn('DB server started with %s on %s, pid=%d',
                     sys.executable, self.frontend, self.pid)
        # start frontend->backend proxy
        try:
            z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER),
                        z.bind(self.backend, z.zmq.DEALER))
        except (KeyboardInterrupt, z.zmq.ZMQError):
            for sock in workers:
                sock.running = False
            logging.warn('DB server stopped')

def different_paths(path1, path2):
    path1 = os.path.realpath(path1)  # expand symlinks
    path2 = os.path.realpath(path2)  # expand symlinks
    # don't care about the extension (it may be .py or .pyc)
    return os.path.splitext(path1)[0] != os.path.splitext(path2)[0]

def get_status(address=None):
    """
    Check if the DbServer is up.

    :param address: pair (hostname, port)
    :returns: 'running' or 'not-running'
    """
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s = config.dbserver
    try:
        err = sock.connect_ex(address or (s.host, s.port))
    finally:
        sock.close()
    return 'not-running' if err else 'running'

def check_foreign():
    """
    Check if we the DbServer is the right one
    """
    if not config.dbserver.multi_user:
        remote_server_path = logs.dbcmd('get_path')
        if different_paths(server_path, remote_server_path):
            return('You are trying to contact a DbServer from another'
                   ' instance (got %s, expected %s)\n'
                   'Check the configuration or stop the foreign'
                   ' DbServer instance') % (remote_server_path, server_path)

def ensure_on():
    """
    Start the DbServer if it is off
    """
    if get_status() == 'not-running':
        if config.dbserver.multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'])

        # wait for the dbserver to start
        waiting_seconds = 10
        while get_status() == 'not-running':
            if waiting_seconds == 0:
                sys.exit('The DbServer cannot be started after 10 seconds. '
                         'Please check the configuration')
            time.sleep(1)
            waiting_seconds -= 1

@sap.Script
def run_server(dbhostport=None, dbpath=None, logfile=DATABASE['LOG'],
               loglevel='WARN'):
    """
    Run the DbServer on the given database file and port. If not given,
    use the settings in openquake.cfg.
    """
    if dbhostport:  # assume a string of the form "dbhost:port"
        dbhost, port = dbhostport.split(':')
        addr = (dbhost, int(port))
        DATABASE['PORT'] = int(port)
    else:
        addr = (config.dbserver.host, config.dbserver.port)

    if dbpath:
        DATABASE['NAME'] = dbpath

    # create the db directory if needed
    dirname = os.path.dirname(DATABASE['NAME'])
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    # create and upgrade the db if needed
    db('PRAGMA foreign_keys = ON')  # honor ON DELETE CASCADE
    actions.upgrade_db(db)
    # the line below is needed to work around a very subtle bug of sqlite;
    # we need new connections, see https://github.com/gem/oq-engine/pull/3002
    db.close()

    # configure logging and start the server
    logging.basicConfig(level=getattr(logging, loglevel), filename=logfile)
    try:
        DbServer(db, addr).start()
    finally:
        db.close()

run_server.arg('dbhostport', 'dbhost:port')
run_server.arg('dbpath', 'dbpath')
run_server.arg('logfile', 'log file')
run_server.opt('loglevel', 'WARN or INFO')

if __name__ == '__main__':
    run_server.callfunc()
<commit_msg>Fixed bug with celery [skip CI]<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (C) 2016-2017 GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import sys
import time
import socket
import os.path
import sqlite3
import logging
import threading
import subprocess

from openquake.baselib import config, sap, zeromq as z
from openquake.baselib.parallel import safely_call
from openquake.commonlib import logs
from openquake.server.db import actions
from openquake.server import dbapi
from openquake.server import __file__ as server_path
from openquake.server.settings import DATABASE

db = dbapi.Db(sqlite3.connect, DATABASE['NAME'], isolation_level=None,
              detect_types=sqlite3.PARSE_DECLTYPES, timeout=20)
# NB: I am increasing the timeout from 5 to 20 seconds to see if the random
# OperationalError: "database is locked" disappear in the WebUI tests

class DbServer(object):
    """
    A server collecting the received commands into a queue
    """
    def __init__(self, db, address, num_workers=5):
        self.db = db
        self.frontend = 'tcp://%s:%s' % address
        self.backend = 'inproc://dbworkers'
        self.num_workers = num_workers
        self.pid = os.getpid()

    def worker(self, sock):
        for cmd_ in sock:
            cmd, args = cmd_[0], cmd_[1:]
            if cmd == 'getpid':
                sock.rep((self.pid, None, None))
                continue
            try:
                func = getattr(actions, cmd)
            except AttributeError:
                sock.rep(('Invalid command ' + cmd, ValueError, None))
            else:
                sock.rep(safely_call(func, (self.db,) + args))

    def start(self):
        # start workers
        workers = []
        for _ in range(self.num_workers):
            sock = z.Socket(self.backend, z.zmq.REP, 'connect')
            threading.Thread(target=self.worker, args=(sock,)).start()
            workers.append(sock)
        logging.warn('DB server started with %s on %s, pid=%d',
                     sys.executable, self.frontend, self.pid)
        # start frontend->backend proxy
        try:
            z.zmq.proxy(z.bind(self.frontend, z.zmq.ROUTER),
                        z.bind(self.backend, z.zmq.DEALER))
        except (KeyboardInterrupt, z.zmq.ZMQError):
            for sock in workers:
                sock.running = False
            logging.warn('DB server stopped')

def different_paths(path1, path2):
    path1 = os.path.realpath(path1)  # expand symlinks
    path2 = os.path.realpath(path2)  # expand symlinks
    # don't care about the extension (it may be .py or .pyc)
    return os.path.splitext(path1)[0] != os.path.splitext(path2)[0]

def get_status(address=None):
    """
    Check if the DbServer is up.

    :param address: pair (hostname, port)
    :returns: 'running' or 'not-running'
    """
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s = config.dbserver
    try:
        err = sock.connect_ex(address or (s.host, int(s.port)))
    finally:
        sock.close()
    return 'not-running' if err else 'running'

def check_foreign():
    """
    Check if we the DbServer is the right one
    """
    if not config.dbserver.multi_user:
        remote_server_path = logs.dbcmd('get_path')
        if different_paths(server_path, remote_server_path):
            return('You are trying to contact a DbServer from another'
                   ' instance (got %s, expected %s)\n'
                   'Check the configuration or stop the foreign'
                   ' DbServer instance') % (remote_server_path, server_path)

def ensure_on():
    """
    Start the DbServer if it is off
    """
    if get_status() == 'not-running':
        if config.dbserver.multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'])

        # wait for the dbserver to start
        waiting_seconds = 10
        while get_status() == 'not-running':
            if waiting_seconds == 0:
                sys.exit('The DbServer cannot be started after 10 seconds. '
                         'Please check the configuration')
            time.sleep(1)
            waiting_seconds -= 1

@sap.Script
def run_server(dbhostport=None, dbpath=None, logfile=DATABASE['LOG'],
               loglevel='WARN'):
    """
    Run the DbServer on the given database file and port. If not given,
    use the settings in openquake.cfg.
    """
    if dbhostport:  # assume a string of the form "dbhost:port"
        dbhost, port = dbhostport.split(':')
        addr = (dbhost, int(port))
        DATABASE['PORT'] = int(port)
    else:
        addr = (config.dbserver.host, config.dbserver.port)

    if dbpath:
        DATABASE['NAME'] = dbpath

    # create the db directory if needed
    dirname = os.path.dirname(DATABASE['NAME'])
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    # create and upgrade the db if needed
    db('PRAGMA foreign_keys = ON')  # honor ON DELETE CASCADE
    actions.upgrade_db(db)
    # the line below is needed to work around a very subtle bug of sqlite;
    # we need new connections, see https://github.com/gem/oq-engine/pull/3002
    db.close()

    # configure logging and start the server
    logging.basicConfig(level=getattr(logging, loglevel), filename=logfile)
    try:
        DbServer(db, addr).start()
    finally:
        db.close()

run_server.arg('dbhostport', 'dbhost:port')
run_server.arg('dbpath', 'dbpath')
run_server.arg('logfile', 'log file')
run_server.opt('loglevel', 'WARN or INFO')

if __name__ == '__main__':
    run_server.callfunc()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
###############################################################################
#                                                                             #
# Copyright (C) 2012  Renato Lima (Akretion)                                  #
#                                                                             #
#This program is free software: you can redistribute it and/or modify         #
#it under the terms of the GNU Affero General Public License as published by  #
#the Free Software Foundation, either version 3 of the License, or            #
#(at your option) any later version.                                          #
#                                                                             #
#This program is distributed in the hope that it will be useful,              #
#but WITHOUT ANY WARRANTY; without even the implied warranty of               #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
#GNU Affero General Public License for more details.                          #
#                                                                             #
#You should have received a copy of the GNU Affero General Public License     #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.        #
###############################################################################

import re

from openerp.osv import orm, fields

class L10n_brZip(orm.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip'
    _columns = {
        'zip': fields.char('CEP', size=8, required=True),
        'street_type': fields.char('Tipo', size=26),
        'street': fields.char('Logradouro', size=72),
        'district': fields.char('Bairro', size=72),
        'country_id': fields.many2one('res.country', 'Country'),
        'state_id': fields.many2one(
            'res.country.state', 'Estado',
            domain="[('country_id','=',country_id)]"),
        'l10n_br_city_id': fields.many2one(
            'l10n_br_base.city', 'Cidade',
            required=True, domain="[('state_id','=',state_id)]"),
    }

    def set_domain(self, country_id=False, state_id=False,
                l10n_br_city_id=False, district=False,
                street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = re.sub('[^0-9]', '', zip_code or '')
            domain.append(('zip', '=', new_zip))
        else:
            if not state_id or not l10n_br_city_id or \
                len(street or '') == 0:
                raise orm.except_orm(
                    u'Parametros insuficientes',
                    u'Necessário informar Estado, município e logradouro')

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if l10n_br_city_id:
                domain.append(('l10n_br_city_id', '=', l10n_br_city_id))
            if district:
                domain.append(('district', 'like', district))
            if street:
                domain.append(('street', 'like', street))

        return domain

    def set_result(self, cr, uid, ids, context, zip_read=None):
        if zip_read:
            zip_code = zip_read['zip']
            if len(zip_code) == 8:
                zip_code = '%s-%s' % (zip_code[0:5], zip_code[5:8])
            result = {
                'country_id': zip_read['country_id'] and zip_read['country_id'][0] or False,
                'state_id': zip_read['state_id'] and zip_read['state_id'][0] or False,
                'l10n_br_city_id': zip_read['l10n_br_city_id'] and zip_read['l10n_br_city_id'][0] or False,
                'district': (zip_read['district'] or ''),
                'street': ((zip_read['street_type'] or '') + ' ' + (zip_read['street'] or '')),
                'zip': zip_code,
            }
        else:
            result = {}
        return result

    def zip_search_multi(self, cr, uid, ids, context, country_id=False,
                        state_id=False, l10n_br_city_id=False,
                        district=False, street=False, zip_code=False):
        domain = self.set_domain(
            country_id=country_id,
            state_id=state_id,
            l10n_br_city_id=l10n_br_city_id,
            district=district,
            street=street,
            zip_code=zip_code)
        return self.search(cr, uid, domain)

    def zip_search(self, cr, uid, ids, context,
                country_id=False, state_id=False,
                l10n_br_city_id=False, district=False,
                street=False, zip_code=False):
        result = self.set_result(cr, uid, ids, context)
        zip_id = self.zip_search_multi(
            cr, uid, ids, context,
            country_id, state_id,
            l10n_br_city_id, district,
            street, zip_code)
        if len(zip_id) == 1:
            result = self.set_result(cr, uid, ids, context, zip_id[0])
            return result
        else:
            return False

    def create_wizard(self, cr, uid, ids, context,
                    object_name, country_id=False,
                    state_id=False, l10n_br_city_id=False,
                    district=False, street=False, zip_code=False,
                    zip_ids=False):

        context.update({
            'zip': zip_code,
            'street': street,
            'district': district,
            'country_id': country_id,
            'state_id': state_id,
            'l10n_br_city_id': l10n_br_city_id,
            'zip_ids': zip_ids,
            'address_id': ids[0],
            'object_name': object_name})

        result = {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
        }

        return result
<commit_msg>migrate l10n_br.zip object to new API<commit_after># -*- encoding: utf-8 -*-
###############################################################################
#                                                                             #
# Copyright (C) 2012  Renato Lima (Akretion)                                  #
#                                                                             #
#This program is free software: you can redistribute it and/or modify         #
#it under the terms of the GNU Affero General Public License as published by  #
#the Free Software Foundation, either version 3 of the License, or            #
#(at your option) any later version.                                          #
#                                                                             #
#This program is distributed in the hope that it will be useful,              #
#but WITHOUT ANY WARRANTY; without even the implied warranty of               #
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
#GNU Affero General Public License for more details.                          #
#                                                                             #
#You should have received a copy of the GNU Affero General Public License     #
#along with this program.  If not, see <http://www.gnu.org/licenses/>.        #
###############################################################################

import re

from openerp import models, fields
from openerp.exceptions import except_orm

class L10n_brZip(models.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip'

    zip = fields.Char('CEP', size=8, required=True)
    street_type = fields.Char('Tipo', size=26)
    street = fields.Char('Logradouro', size=72)
    district = fields.Char('Bairro', size=72)
    country_id = fields.Many2one('res.country', 'Country')
    state_id = fields.Many2one(
        'res.country.state', 'Estado',
        domain="[('country_id','=',country_id)]")
    l10n_br_city_id = fields.Many2one(
        'l10n_br_base.city', 'Cidade',
        required=True, domain="[('state_id','=',state_id)]")

    # TODO migrate to new API
    def set_domain(self, country_id=False, state_id=False,
                l10n_br_city_id=False, district=False,
                street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = re.sub('[^0-9]', '', zip_code or '')
            domain.append(('zip', '=', new_zip))
        else:
            if not state_id or not l10n_br_city_id or \
                len(street or '') == 0:
                raise except_orm(
                    u'Parametros insuficientes',
                    u'Necessário informar Estado, município e logradouro')

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if l10n_br_city_id:
                domain.append(('l10n_br_city_id', '=', l10n_br_city_id))
            if district:
                domain.append(('district', 'like', district))
            if street:
                domain.append(('street', 'like', street))

        return domain

    def set_result(self, cr, uid, ids, context, zip_read=None):
        if zip_read:
            zip_code = zip_read['zip']
            if len(zip_code) == 8:
                zip_code = '%s-%s' % (zip_code[0:5], zip_code[5:8])
            result = {
                'country_id': zip_read['country_id'] and zip_read['country_id'][0] or False,
                'state_id': zip_read['state_id'] and zip_read['state_id'][0] or False,
                'l10n_br_city_id': zip_read['l10n_br_city_id'] and zip_read['l10n_br_city_id'][0] or False,
                'district': (zip_read['district'] or ''),
                'street': ((zip_read['street_type'] or '') + ' ' + (zip_read['street'] or '')),
                'zip': zip_code,
            }
        else:
            result = {}
        return result

    # TODO migrate to new API
    def zip_search_multi(self, cr, uid, ids, context, country_id=False,
                        state_id=False, l10n_br_city_id=False,
                        district=False, street=False, zip_code=False):
        domain = self.set_domain(
            country_id=country_id,
            state_id=state_id,
            l10n_br_city_id=l10n_br_city_id,
            district=district,
            street=street,
            zip_code=zip_code)
        return self.search(cr, uid, domain)

    def zip_search(self, cr, uid, ids, context,
                country_id=False, state_id=False,
                l10n_br_city_id=False, district=False,
                street=False, zip_code=False):
        result = self.set_result(cr, uid, ids, context)
        zip_id = self.zip_search_multi(
            cr, uid, ids, context,
            country_id, state_id,
            l10n_br_city_id, district,
            street, zip_code)
        if len(zip_id) == 1:
            result = self.set_result(cr, uid, ids, context, zip_id[0])
            return result
        else:
            return False

    # TODO migrate to new API
    def create_wizard(self, cr, uid, ids, context,
                    object_name, country_id=False,
                    state_id=False, l10n_br_city_id=False,
                    district=False, street=False, zip_code=False,
                    zip_ids=False):

        context.update({
            'zip': zip_code,
            'street': street,
            'district': district,
            'country_id': country_id,
            'state_id': state_id,
            'l10n_br_city_id': l10n_br_city_id,
            'zip_ids': zip_ids,
            'address_id': ids[0],
            'object_name': object_name})

        result = {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
        }

        return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.hazardlib.calc.filters import getdefault
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    """
    :param gmfs: an array of GMFs with fields sid, eid, gmv
    :param param: a dictionary of parameters coming from the job.ini
    :param monitor: a Monitor instance
    :returns: a dictionary of arrays with keys elt, alt, losses_by_A, ...
    """
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assets_df = dstore.read_df('assetcol/array', 'ordinal')
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    # aggkey -> eid -> loss
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    lba.alt = general.AccumDict(
        accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    lba.losses_by_E = numpy.zeros((E, L), F32)
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    for lt, lti in crmodel.lti.items():
        val = param['minimum_asset_loss'][lt]
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    haz_by_sid = general.group_array(gmfs, 'sid')
    for sid, asset_df in assets_df.groupby('site_id'):
        try:
            haz = haz_by_sid[sid]
        except KeyError:  # no hazard here
            continue
        with mon_risk:
            assets = asset_df.to_records()  # fast
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            else:
                ws = None
            assets_by_taxo = get_assets_by_taxo(assets, tempname)  # fast
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])  # fast
            out = get_output(crmodel, assets_by_taxo, haz)  # slow
        with mon_agg:
            tagidxs = assets[aggby] if aggby else None
            acc['numlosses'] += lba.aggregate(
                out, eidx, minimum_loss, tagidxs, ws)
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, lba.losses_by_E) if losses.sum()),
        elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid, ultra-fast
        ((eid, eid2rlz[eid], loss) for eid, loss in lba.alt[idx].items()),
        elt_dt) for idx in lba.alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'],
                           param['amplifier'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.ebrupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    alltags = [getattr(tagcol, tagname) for tagname in aggby]
    ranges = [range(1, len(tags)) for tags in alltags]
    for idxs in itertools.product(*ranges):
        d = {name: tags[idx] for idx, name, tags in zip(idxs, aggby, alltags)}
        yield idxs, d

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        self.check_number_loss_curves()
        mal = {lt: getdefault(oq.minimum_asset_loss, lt)
               for lt in oq.loss_names}
        logging.info('minimum_asset_loss=%s', mal)
        if (oq.aggregate_by and self.E * A > oq.max_potential_gmfs and
                any(val == 0 for val in mal.values()) and not
                sum(oq.minimum_asset_loss.values())):
            logging.warning('The calculation is really big; you should set '
                            'minimum_asset_loss')
        self.param['minimum_asset_loss'] = mal

        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs, attrs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt,
                                       attrs=attrs)
        self.param.pop('oqparam', None)  # unneeded
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def check_number_loss_curves(self):
        """
        Raise an error if generating too many loss curves (>10,000)
        """
        shp = self.assetcol.tagcol.agg_shape(
            (self.L,), aggregate_by=self.oqparam.aggregate_by)
        if numpy.prod(shp) > 10_000:
            dic = dict(loss_types=self.L)
            for aggby in self.oqparam.aggregate_by:
                dic[aggby] = len(getattr(self.assetcol.tagcol, aggby)) - 1
            tot = numpy.prod(list(dic.values()))
            msg = ' * ' .join('(%s=%d)' % item for item in dic.items())
            raise ValueError('Producing too many aggregate loss curves, please'
                             ' reduce the aggregate_by\n%s = %d' % (msg, tot))

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        parent = self.datastore.parent
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info(
            'Sending {:_d} ruptures'.format(len(self.datastore['ruptures'])))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(
                self.datastore, srcfilter, oq.concurrent_tasks):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stat'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<commit_msg>[skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator
import itertools
from datetime import datetime
import numpy

from openquake.baselib import datastore, hdf5, parallel, general
from openquake.baselib.python3compat import zip
from openquake.hazardlib.calc.filters import getdefault
from openquake.risklib import riskmodels
from openquake.risklib.scientific import LossesByAsset
from openquake.risklib.riskinput import (
    cache_epsilons, get_assets_by_taxo, get_output)
from openquake.commonlib import logs
from openquake.calculators import base, event_based, getters
from openquake.calculators.post_risk import PostRiskCalculator

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64
TWO32 = 2 ** 32
get_n_occ = operator.itemgetter(1)

gmf_info_dt = numpy.dtype([('rup_id', U32), ('task_no', U16),
                           ('nsites', U16), ('gmfbytes', F32), ('dt', F32)])

def calc_risk(gmfs, param, monitor):
    """
    :param gmfs: an array of GMFs with fields sid, eid, gmv
    :param param: a dictionary of parameters coming from the job.ini
    :param monitor: a Monitor instance
    :returns: a dictionary of arrays with keys elt, alt, losses_by_A, ...
    """
    mon_risk = monitor('computing risk', measuremem=False)
    mon_agg = monitor('aggregating losses', measuremem=False)
    eids = numpy.unique(gmfs['eid'])
    dstore = datastore.read(param['hdf5path'])
    with monitor('getting assets'):
        assets_df = dstore.read_df('assetcol/array', 'ordinal')
    with monitor('getting crmodel'):
        crmodel = riskmodels.CompositeRiskModel.read(dstore)
        events = dstore['events'][list(eids)]
        weights = dstore['weights'][()]
    E = len(eids)
    L = len(param['lba'].loss_names)
    elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
    # aggkey -> eid -> loss
    acc = dict(events_per_sid=0, numlosses=numpy.zeros(2, int))  # (kept, tot)
    lba = param['lba']
    lba.alt = general.AccumDict(
        accum=general.AccumDict(accum=numpy.zeros(L, F32)))
    lba.losses_by_E = numpy.zeros((E, L), F32)
    tempname = param['tempname']
    eid2rlz = dict(events[['id', 'rlz_id']])
    eid2idx = {eid: idx for idx, eid in enumerate(eids)}
    aggby = param['aggregate_by']

    minimum_loss = []
    for lt, lti in crmodel.lti.items():
        val = param['minimum_asset_loss'][lt]
        minimum_loss.append(val)
        if lt in lba.policy_dict:  # same order as in lba.compute
            minimum_loss.append(val)

    haz_by_sid = general.group_array(gmfs, 'sid')
    for sid, asset_df in assets_df.groupby('site_id'):
        try:
            haz = haz_by_sid[sid]
        except KeyError:  # no hazard here
            continue
        with mon_risk:
            assets = asset_df.to_records()  # fast
            acc['events_per_sid'] += len(haz)
            if param['avg_losses']:
                ws = weights[[eid2rlz[eid] for eid in haz['eid']]]
            else:
                ws = None
            assets_by_taxo = get_assets_by_taxo(assets, tempname)  # fast
            eidx = numpy.array([eid2idx[eid] for eid in haz['eid']])  # fast
            out = get_output(crmodel, assets_by_taxo, haz)  # slow
        with mon_agg:
            tagidxs = assets[aggby] if aggby else None
            acc['numlosses'] += lba.aggregate(
                out, eidx, minimum_loss, tagidxs, ws)
    if len(gmfs):
        acc['events_per_sid'] /= len(gmfs)
    acc['elt'] = numpy.fromiter(  # this is ultra-fast
        ((event['id'], event['rlz_id'], losses)
         for event, losses in zip(events, lba.losses_by_E) if losses.sum()),
        elt_dt)
    acc['alt'] = {idx: numpy.fromiter(  # already sorted by aid, ultra-fast
        ((eid, eid2rlz[eid], loss) for eid, loss in lba.alt[idx].items()),
        elt_dt) for idx in lba.alt}
    if param['avg_losses']:
        acc['losses_by_A'] = param['lba'].losses_by_A * param['ses_ratio']
        # without resetting the cache the sequential avg_losses would be wrong!
        del param['lba'].__dict__['losses_by_A']
    return acc

def ebrisk(rupgetter, srcfilter, param, monitor):
    """
    :param rupgetter: RuptureGetter with multiple ruptures
    :param srcfilter: a SourceFilter
    :param param: dictionary of parameters coming from oqparam
    :param monitor: a Monitor instance
    :returns: a dictionary with keys elt, alt, ...
    """
    mon_rup = monitor('getting ruptures', measuremem=False)
    mon_haz = monitor('getting hazard', measuremem=False)
    gmfs = []
    gmf_info = []
    gg = getters.GmfGetter(rupgetter, srcfilter, param['oqparam'],
                           param['amplifier'])
    nbytes = 0
    for c in gg.gen_computers(mon_rup):
        with mon_haz:
            data, time_by_rup = c.compute_all(gg.min_iml, gg.rlzs_by_gsim)
        if len(data):
            gmfs.append(data)
            nbytes += data.nbytes
        gmf_info.append((c.ebrupture.id, mon_haz.task_no, len(c.sids),
                         data.nbytes, mon_haz.dt))
        if nbytes > param['ebrisk_maxsize']:
            msg = 'produced subtask'
            try:
                logs.dbcmd('log', monitor.calc_id, datetime.utcnow(), 'DEBUG',
                           'ebrisk#%d' % monitor.task_no, msg)
            except Exception:  # for `oq run`
                print(msg)
            yield calc_risk, numpy.concatenate(gmfs), param
            nbytes = 0
            gmfs = []
    res = {}
    if gmfs:
        res.update(calc_risk(numpy.concatenate(gmfs), param, monitor))
    if gmf_info:
        res['gmf_info'] = numpy.array(gmf_info, gmf_info_dt)
    yield res

def gen_indices(tagcol, aggby):
    alltags = [getattr(tagcol, tagname) for tagname in aggby]
    ranges = [range(1, len(tags)) for tags in alltags]
    for idxs in itertools.product(*ranges):
        d = {name: tags[idx] for idx, name, tags in zip(idxs, aggby, alltags)}
        yield idxs, d

@base.calculators.add('ebrisk')
class EbriskCalculator(event_based.EventBasedCalculator):
    """
    Event based PSHA calculator generating event loss tables
    """
    core_task = ebrisk
    is_stochastic = True
    precalc = 'event_based'
    accept_precalc = ['event_based', 'event_based_risk']

    def pre_execute(self):
        oq = self.oqparam
        oq.ground_motion_fields = False
        super().pre_execute()
        self.param['lba'] = lba = (
            LossesByAsset(self.assetcol, oq.loss_names,
                          self.policy_name, self.policy_dict))
        self.param['ses_ratio'] = oq.ses_ratio
        self.param['aggregate_by'] = oq.aggregate_by
        self.param['ebrisk_maxsize'] = oq.ebrisk_maxsize
        self.A = A = len(self.assetcol)
        self.L = L = len(lba.loss_names)
        self.check_number_loss_curves()
        mal = {lt: getdefault(oq.minimum_asset_loss, lt)
               for lt in oq.loss_names}
        logging.info('minimum_asset_loss=%s', mal)
        if (oq.aggregate_by and self.E * A > oq.max_potential_gmfs and
                any(val == 0 for val in mal.values()) and not
                sum(oq.minimum_asset_loss.values())):
            logging.warning('The calculation is really big; you should set '
                            'minimum_asset_loss')
        self.param['minimum_asset_loss'] = mal

        elt_dt = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L,)))]
        for idxs, attrs in gen_indices(self.assetcol.tagcol, oq.aggregate_by):
            idx = ','.join(map(str, idxs))
            self.datastore.create_dset('event_loss_table/' + idx, elt_dt,
                                       attrs=attrs)
        self.param.pop('oqparam', None)  # unneeded
        self.datastore.create_dset('avg_losses-stats', F32, (A, 1, L))  # mean
        elt_nbytes = 4 * self.E * L
        if elt_nbytes / (oq.concurrent_tasks or 1) > TWO32:
            raise RuntimeError('The event loss table is too big to be transfer'
                               'red with %d tasks' % oq.concurrent_tasks)
        self.datastore.create_dset('losses_by_event', elt_dt)
        self.datastore.create_dset('gmf_info', gmf_info_dt)

    def check_number_loss_curves(self):
        """
        Raise an error if generating too many loss curves (>10,000)
        """
        shp = self.assetcol.tagcol.agg_shape(
            (self.L,), aggregate_by=self.oqparam.aggregate_by)
        if numpy.prod(shp) > 10_000:
            dic = dict(loss_types=self.L)
            for aggby in self.oqparam.aggregate_by:
                dic[aggby] = len(getattr(self.assetcol.tagcol, aggby)) - 1
            tot = numpy.prod(list(dic.values()))
            msg = ' * ' .join('(%s=%d)' % item for item in dic.items())
            raise ValueError('Producing too many aggregate loss curves, please'
                             ' reduce the aggregate_by\n%s = %d' % (msg, tot))

    def execute(self):
        self.datastore.flush()  # just to be sure
        oq = self.oqparam
        self.set_param(
            hdf5path=self.datastore.filename,
            tempname=cache_epsilons(
                self.datastore, oq, self.assetcol, self.crmodel, self.E))
        srcfilter = self.src_filter(self.datastore.tempname)
        logging.info(
            'Sending {:_d} ruptures'.format(len(self.datastore['ruptures'])))
        self.events_per_sid = []
        self.numlosses = 0
        self.datastore.swmr_on()
        self.indices = general.AccumDict(accum=[])  # rlzi -> [(start, stop)]
        smap = parallel.Starmap(
            self.core_task.__func__, h5=self.datastore.hdf5)
        for rgetter in getters.gen_rupture_getters(
                self.datastore, srcfilter, oq.concurrent_tasks):
            smap.submit((rgetter, srcfilter, self.param))
        smap.reduce(self.agg_dicts)
        if self.indices:
            self.datastore['event_loss_table/indices'] = self.indices
        gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum()
        logging.info(
            'Produced %s of GMFs', general.humansize(gmf_bytes))
        logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses))
        return 1

    def agg_dicts(self, dummy, dic):
        """
        :param dummy: unused parameter
        :param dic: dictionary with keys elt, losses_by_A
        """
        if 'gmf_info' in dic:
            hdf5.extend(self.datastore['gmf_info'], dic.pop('gmf_info'))
        if not dic:
            return
        self.oqparam.ground_motion_fields = False  # hack
        with self.monitor('saving losses_by_event and event_loss_table'):
            hdf5.extend(self.datastore['losses_by_event'], dic['elt'])
            for idx, arr in dic['alt'].items():
                hdf5.extend(self.datastore['event_loss_table/' + idx], arr)
        if self.oqparam.avg_losses:
            with self.monitor('saving avg_losses'):
                self.datastore['avg_losses-stats'][:, 0] += dic['losses_by_A']
        self.events_per_sid.append(dic['events_per_sid'])
        self.numlosses += dic['numlosses']

    def post_execute(self, dummy):
        """
        Compute and store average losses from the losses_by_event dataset,
        and then loss curves and maps.
        """
        oq = self.oqparam
        if oq.avg_losses:
            self.datastore['avg_losses-stats'].attrs['stat'] = [b'mean']
        prc = PostRiskCalculator(oq, self.datastore.calc_id)
        prc.datastore.parent = self.datastore.parent
        prc.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, orm

class it_config_settings(orm.TransientModel):
    _name = 'l10n_it.config.settings'
    _inherit = 'res.config.settings'

    _columns = {
        'module_l10n_it_pec': fields.boolean(
            'Use Pec Mail in Partner Profile',
            help="""Install l10n_it_pec module for pec mail management"""
        ),
    }

    def create(self, cr, uid, values, context=None):
        id = super(it_config_settings, self).create(cr, uid, values, context)
        # Hack: to avoid some nasty bug, related fields are not written
        # upon record creation. Hence we write on those fields here.
        vals = {}
        for fname, field in self._columns.iteritems():
            if isinstance(field, fields.related) and fname in values:
                vals[fname] = values[fname]
        self.write(cr, uid, [id], vals, context)
        return id
<commit_msg>[Fix] Fix CamelCase<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, orm

class L10nItConfigSettings(orm.TransientModel):
    _name = 'l10n_it.config.settings'
    _inherit = 'res.config.settings'

    _columns = {
        'module_l10n_it_pec': fields.boolean(
            'Use Pec Mail in Partner Profile',
            help="""Install l10n_it_pec module for pec mail management"""
        ),
    }

    def create(self, cr, uid, values, context=None):
        id = super(L10nItConfigSettings, self).create(cr, uid, values, context)
        # Hack: to avoid some nasty bug, related fields are not written
        # upon record creation. Hence we write on those fields here.
        vals = {}
        for fname, field in self._columns.iteritems():
            if isinstance(field, fields.related) and fname in values:
                vals[fname] = values[fname]
        self.write(cr, uid, [id], vals, context)
        return id
<|endoftext|>


<commit_before>import pandas as pd

from break4w.categorical import Categorical

class Bool(Categorical):
    def __init__(self, name, description, clean_name=None, bool_format=None,
                 ambiguous_values=None, mimarks=False, ontology=None,
                 missing=None, blanks=None, colormap=None):
        """A question object for boolean question

        Parameters
        ----------
        name : str
            The name of column where the group is stored
        description : str
            A brief description of the data contained in the question
        clean_name : str, optional
            A nicer version of the way the column should be named.
        bool_format: list, optional
            The format expected for the boolean values. For example, `['True',
            'False']`, `[0, 1]`, `[True, False]`. By default, data is assumed
            to be a lower case text string (`['true', 'false']`).
        ambigious_values: str, optional
            A value indicating whether the respondant was unsure about the
            response. This is a different value than the participant failing
            to answer the question. (For instance, in response to a question
            like, "Are you pregnant?", the participant may not yet known but
            may suspect it.)
        mimarks : bool, optional
            If the question was a mimarks standard field
        ontology : str, optional
            The type of ontology, if any, used to answer the question. An
            ontology provides a consistent, structured vocabulary. A list
            of ontologies can be found at https://www.ebi.ac.uk/ols/ontologies
        missing : str, list, optional
            Acceptable missing values. Missing values will be used to validate
            all values in the column. Specified missing values can also be
            ignored during analysis if correctly specified.
        blanks: str, list, optional
            Value to represent experimental blanks, if relevent.
        colormap: str, iterable, optional
            The colors to use when plotting the data. This can be a matplotlib
            colormap object, a string describing a matplotlib compatable
            colormap (i.e. `'RdBu'`), or an iterable of matplotlib compatable
            color values.

        """

        if bool_format is not None:
            t_format = bool_format[0]
            f_format = bool_format[1]
        else:
            t_format = 'true'
            f_format = 'false'

        if ambiguous_values is not None:
            adj_order = [t_format, f_format, ambiguous_values]
        else:
            adj_order = [t_format, f_format]

        Categorical.__init__(self,
                             name=name,
                             description=description,
                             dtype=bool,
                             order=adj_order,
                             extremes=[t_format, f_format],
                             clean_name=clean_name,
                             ambiguous_values=ambiguous_values,
                             mimarks=mimarks,
                             ontology=ontology,
                             missing=missing,
                             blanks=blanks,
                             colormap=colormap,
                             )
        self.type = 'Bool'

    def analysis_convert_to_word(self, map_):
        """Converts boolean values to 'yes' and 'no'

        Parameters
        ----------
        map_ : DataFrame
            A pandas object containing the data to be analyzed. The
            Question `name` should be a column in the `map_`.

        """
        self.check_map(map_)
        self.analysis_remap_dtype(map_)

        def remap_(x):
            if pd.isnull(x):
                return x
            if x:
                return 'yes'
            elif not x:
                return 'no'

        map_[self.name] = map_[self.name].apply(remap_)
        self._update_order(remap_)
        self._update_log('convert boolean', 'replace', 'standarize to yes/no')
<commit_msg>drops check_map<commit_after>import pandas as pd

from break4w.categorical import Categorical

class Bool(Categorical):
    def __init__(self, name, description, clean_name=None, bool_format=None,
                 ambiguous_values=None, mimarks=False, ontology=None,
                 missing=None, blanks=None, colormap=None):
        """A question object for boolean question

        Parameters
        ----------
        name : str
            The name of column where the group is stored
        description : str
            A brief description of the data contained in the question
        clean_name : str, optional
            A nicer version of the way the column should be named.
        bool_format: list, optional
            The format expected for the boolean values. For example, `['True',
            'False']`, `[0, 1]`, `[True, False]`. By default, data is assumed
            to be a lower case text string (`['true', 'false']`).
        ambigious_values: str, optional
            A value indicating whether the respondant was unsure about the
            response. This is a different value than the participant failing
            to answer the question. (For instance, in response to a question
            like, "Are you pregnant?", the participant may not yet known but
            may suspect it.)
        mimarks : bool, optional
            If the question was a mimarks standard field
        ontology : str, optional
            The type of ontology, if any, used to answer the question. An
            ontology provides a consistent, structured vocabulary. A list
            of ontologies can be found at https://www.ebi.ac.uk/ols/ontologies
        missing : str, list, optional
            Acceptable missing values. Missing values will be used to validate
            all values in the column. Specified missing values can also be
            ignored during analysis if correctly specified.
        blanks: str, list, optional
            Value to represent experimental blanks, if relevent.
        colormap: str, iterable, optional
            The colors to use when plotting the data. This can be a matplotlib
            colormap object, a string describing a matplotlib compatable
            colormap (i.e. `'RdBu'`), or an iterable of matplotlib compatable
            color values.

        """

        if bool_format is not None:
            t_format = bool_format[0]
            f_format = bool_format[1]
        else:
            t_format = 'true'
            f_format = 'false'

        if ambiguous_values is not None:
            adj_order = [t_format, f_format, ambiguous_values]
        else:
            adj_order = [t_format, f_format]

        Categorical.__init__(self,
                             name=name,
                             description=description,
                             dtype=bool,
                             order=adj_order,
                             extremes=[t_format, f_format],
                             clean_name=clean_name,
                             ambiguous_values=ambiguous_values,
                             mimarks=mimarks,
                             ontology=ontology,
                             missing=missing,
                             blanks=blanks,
                             colormap=colormap,
                             )
        self.type = 'Bool'

    def analysis_convert_to_word(self, map_):
        """Converts boolean values to 'yes' and 'no'

        Parameters
        ----------
        map_ : DataFrame
            A pandas object containing the data to be analyzed. The
            Question `name` should be a column in the `map_`.

        """
        self.analysis_remap_dtype(map_)

        def remap_(x):
            if pd.isnull(x):
                return x
            if x:
                return 'yes'
            elif not x:
                return 'no'

        map_[self.name] = map_[self.name].apply(remap_)
        self._update_order(remap_)
        self._update_log('convert boolean', 'replace', 'standarize to yes/no')
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

"""
WPTools Extract module.
"""

import lxml.html
import json
import re

from . import fetch
from . import utils
from collections import defaultdict
from lxml import etree

class WPToolsExtract:

    def __init__(self):
        pass

def disambig(source, data, title):
    """return DISAMBIGUATION if found"""
    if source == 'html':
        for item in lxml.html.fromstring(data).xpath("//p[1]"):
            if "may refer to:" in etree.tostring(item):
                return "DISAMBIGUATION " + title
    if source == 'parsetree':
        for item in etree.fromstring(data).xpath("//text()"):
            if 'may refer to:' in item:
                return "DISAMBIGUATION " + title

def handle_redirect(red, lead):
    title = red.split("REDIRECT")[-1].strip()
    doc  = html(fetch.get_html(title, lead))
    try:
        return doc.decode('utf-8')
    except:
        return doc

def html(data, lead=False):
    """returns HTML part of action=parse query"""
    try:
        data = json.loads(data)
        doc = data["parse"]["text"]["*"]
        title = data["parse"]["title"]
    except:
        return "NOTFOUND"
    dis = disambig('html', doc, title)
    if dis:  # Misfits
        return dis
    red = redirect('html', doc)
    if red:  # Einstein
        doc = handle_redirect(red, lead)
    if lead:
        doc = html_lead(doc)
    try:
        return doc.encode('utf-8')
    except:
        return doc

def html_lead(frag):
    """returns lead section from page HTML fragment"""
    lead = []
    start = False
    for item in lxml.html.fromstring(frag):
        if html_lead_break(item):
            break
        if not start and item.tag == "p":  # skip templates
            start = True
        if start:
            # print(etree.tostring(item))
            lead.append(etree.tostring(item))
    if not lead:
        return frag
    return "\n".join(lead)

def html_lead_break(elem):
    """returns True if element ends lead section"""
    if elem.tag == "ol":
        return True

def html_keep_tags(frag):
    """strip tags, except replace  and  with markdown"""
    from lxml.html.clean import Cleaner
    cleaner = Cleaner(allow_tags=['b', 'i'], remove_unknown_tags=False)
    text = cleaner.clean_html(frag)
    text = text.replace("
", "").replace("
", "")
    text = re.sub(r"</?b>", "**", text)
    text = re.sub(r"</?i>", "_", text)
    return text

def html_strip_tags(frag):
    """returns lead HTML fragment with tags removed"""
    html = lxml.html.fromstring(frag)
    return etree.tostring(html, method="text", encoding="utf-8")

def html_text(data, lead=False, compact=False):
    doc = html(data, lead)
    doc = html_keep_tags(doc)
    doc = utils.strip_refs(doc)
    doc = utils.single_space(doc)
    doc = plain_text_cleanup(doc)
    if compact:
        doc = utils.collapse(doc)
    try:
        return doc.encode('utf-8')
    except:
        return doc

def infobox(data):
    """returns infobox dict from parsetree"""
    ptree = parsetree(data)
    try:
        for item in etree.fromstring(ptree).xpath("//template"):
            if "box" in item.find('title').text:
                return json.dumps(template_to_dict(item))
    except:
        return ptree

def parsetree(data):
    """return parsetree XML from API JSON"""
    try:
        data = json.loads(data)
        ptree = data["parse"]["parsetree"]["*"].encode('utf-8')
        title = data["parse"]["title"].encode('utf-8')
        dis = disambig('parsetree', ptree, title)
        if dis:
            return dis
        red = redirect('parsetree', ptree)
        if red:
            return red
        return ptree
    except:
        return json.loads(data)["error"]["info"].encode('utf-8')

def redirect(source, data):
    """return #REDIRECT text from query if extant"""
    if source == 'html':
        xpath = "//div[@class=\"redirectMsg\"]//a"
        for item in lxml.html.fromstring(data).xpath(xpath):
            return "#REDIRECT %s" % item.text
    if source == 'parsetree':
        for item in etree.fromstring(data).xpath("//text()"):
            if item.startswith("#REDIRECT"):
                return item.strip()

def plain_text_cleanup(blob):
    """remove known extraneous items"""
    try:
        blob = blob.decode('utf-8')
    except:
        pass
    blob = re.sub(r'\s\( listen\)', "", blob, flags=re.UNICODE)
    return blob.encode('utf-8')

def template_to_dict(tree):
    """returns wikitext template as dict (one deep)"""
    obj = defaultdict(str)
    for item in tree:
        name = item.find('name')
        if name is not None:
            value = item.find('value').text.strip()
            tmp = item.find('value').find('template')
            if tmp is not None:
                value = etree.tostring(tmp)
            obj[name.text.strip()] = value
    return dict(obj)

def text(data, lead=False, compact=False):
    """
    returns plain text of article from HTML
    :param lead: lead section only
    :param compact: collapse newlines into pilcrows
    """
    return html_text(data, lead, compact)

def wikitext(data):
    """return wikitext from API JSON"""
    text = json.loads(data)["parse"]["wikitext"]["*"].encode('utf-8')
    if text.startswith("#REDIRECT"):
        return text.split("\n")[0]
    return text

def wikitext_from_action_query(data):
    """DEPRECATED"""
    out = []
    try:
        for page in json.loads(data)["query"]["pages"]:
            content = page["revisions"][0]["content"].encode('utf-8')
            out.append({"title": page["title"], "content": content})
            return json.dumps(out)
    except:
        return data
<commit_msg>added
 to html_lead_break() in extract.<commit_after># -*- coding:utf-8 -*-

"""
WPTools Extract module.
"""

import lxml.html
import json
import re

from . import fetch
from . import utils
from collections import defaultdict
from lxml import etree

class WPToolsExtract:

    def __init__(self):
        pass

def disambig(source, data, title):
    """return DISAMBIGUATION if found"""
    if source == 'html':
        for item in lxml.html.fromstring(data).xpath("//p[1]"):
            if "may refer to:" in etree.tostring(item):
                return "DISAMBIGUATION " + title
    if source == 'parsetree':
        for item in etree.fromstring(data).xpath("//text()"):
            if 'may refer to:' in item:
                return "DISAMBIGUATION " + title

def handle_redirect(red, lead):
    title = red.split("REDIRECT")[-1].strip()
    doc  = html(fetch.get_html(title, lead))
    try:
        return doc.decode('utf-8')
    except:
        return doc

def html(data, lead=False):
    """returns HTML part of action=parse query"""
    try:
        data = json.loads(data)
        doc = data["parse"]["text"]["*"]
        title = data["parse"]["title"]
    except:
        return "NOTFOUND"
    dis = disambig('html', doc, title)
    if dis:  # Misfits
        return dis
    red = redirect('html', doc)
    if red:  # Einstein
        doc = handle_redirect(red, lead)
    if lead:
        doc = html_lead(doc)
    try:
        return doc.encode('utf-8')
    except:
        return doc

def html_lead(frag):
    """returns lead section from page HTML fragment"""
    lead = []
    start = False
    for item in lxml.html.fromstring(frag):
        if not start and item.tag == "p":  # skip templates
            start = True
        if start:
            if html_lead_break(item):
                break
            # print(etree.tostring(item))
            lead.append(etree.tostring(item))
    if not lead:
        return frag
    return "\n".join(lead)

def html_lead_break(elem):
    """returns True if element ends lead section"""
    if elem.tag == "ol":
        return True
    if elem.tag == "table":
        return True

def html_keep_tags(frag):
    """strip tags, except replace  and  with markdown"""
    from lxml.html.clean import Cleaner
    cleaner = Cleaner(allow_tags=['b', 'i'], remove_unknown_tags=False)
    text = cleaner.clean_html(frag)
    text = text.replace("
", "").replace("
", "")
    text = re.sub(r"</?b>", "**", text)
    text = re.sub(r"</?i>", "_", text)
    return text

def html_strip_tags(frag):
    """returns lead HTML fragment with tags removed"""
    html = lxml.html.fromstring(frag)
    return etree.tostring(html, method="text", encoding="utf-8")

def html_text(data, lead=False, compact=False):
    doc = html(data, lead)
    doc = html_keep_tags(doc)
    doc = utils.strip_refs(doc)
    doc = utils.single_space(doc)
    doc = plain_text_cleanup(doc)
    if compact:
        doc = utils.collapse(doc)
    try:
        return doc.encode('utf-8')
    except:
        return doc

def infobox(data):
    """returns infobox dict from parsetree"""
    ptree = parsetree(data)
    try:
        for item in etree.fromstring(ptree).xpath("//template"):
            if "box" in item.find('title').text:
                return json.dumps(template_to_dict(item))
    except:
        return ptree

def parsetree(data):
    """return parsetree XML from API JSON"""
    try:
        data = json.loads(data)
        ptree = data["parse"]["parsetree"]["*"].encode('utf-8')
        title = data["parse"]["title"].encode('utf-8')
        dis = disambig('parsetree', ptree, title)
        if dis:
            return dis
        red = redirect('parsetree', ptree)
        if red:
            return red
        return ptree
    except:
        return json.loads(data)["error"]["info"].encode('utf-8')

def redirect(source, data):
    """return #REDIRECT text from query if extant"""
    if source == 'html':
        xpath = "//div[@class=\"redirectMsg\"]//a"
        for item in lxml.html.fromstring(data).xpath(xpath):
            return "#REDIRECT %s" % item.text
    if source == 'parsetree':
        for item in etree.fromstring(data).xpath("//text()"):
            if item.startswith("#REDIRECT"):
                return item.strip()

def plain_text_cleanup(blob):
    """remove known extraneous items"""
    try:
        blob = blob.decode('utf-8')
    except:
        pass
    blob = re.sub(r'\s\( listen\)', "", blob, flags=re.UNICODE)
    return blob.encode('utf-8')

def template_to_dict(tree):
    """returns wikitext template as dict (one deep)"""
    obj = defaultdict(str)
    for item in tree:
        name = item.find('name')
        if name is not None:
            value = item.find('value').text.strip()
            tmp = item.find('value').find('template')
            if tmp is not None:
                value = etree.tostring(tmp)
            obj[name.text.strip()] = value
    return dict(obj)

def text(data, lead=False, compact=False):
    """
    returns plain text of article from HTML
    :param lead: lead section only
    :param compact: collapse newlines into pilcrows
    """
    return html_text(data, lead, compact)

def wikitext(data):
    """return wikitext from API JSON"""
    text = json.loads(data)["parse"]["wikitext"]["*"].encode('utf-8')
    if text.startswith("#REDIRECT"):
        return text.split("\n")[0]
    return text

def wikitext_from_action_query(data):
    """DEPRECATED"""
    out = []
    try:
        for page in json.loads(data)["query"]["pages"]:
            content = page["revisions"][0]["content"].encode('utf-8')
            out.append({"title": page["title"], "content": content})
            return json.dumps(out)
    except:
        return data
<|endoftext|>


<commit_before># Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for exporting risk artifacts from the database.
"""

import os
import csv

from openquake.engine.db import models
from openquake.engine.export import core
from openquake.engine.utils import FileWrapper
from openquake.commonlib import risk_writers

LOSS_CURVE_FILENAME_FMT = 'loss-curves-%(loss_curve_id)s.xml'
LOSS_MAP_FILENAME_FMT = 'loss-maps-%(loss_map_id)s.%(file_ext)s'
LOSS_FRACTION_FILENAME_FMT = 'loss-fractions-%(loss_fraction_id)s.xml'
AGGREGATE_LOSS_FILENAME_FMT = 'aggregate-loss-%s.csv'
BCR_FILENAME_FMT = 'bcr-distribution-%(bcr_distribution_id)s.xml'
EVENT_LOSS_FILENAME_FMT = 'event-loss-%s.csv'
EVENT_LOSS_ASSET_FILENAME_FMT = 'event-loss-asset-%s.csv'

DAMAGE = dict(
    collapse_map=(models.DmgDistPerAsset,
                  risk_writers.CollapseMapXMLWriter),
    dmg_dist_per_asset=(models.DmgDistPerAsset,
                        risk_writers.DmgDistPerAssetXMLWriter),
    dmg_dist_per_taxonomy=(models.DmgDistPerTaxonomy,
                           risk_writers.DmgDistPerTaxonomyXMLWriter),
    dmg_dist_total=(models.DmgDistTotal,
                    risk_writers.DmgDistTotalXMLWriter))

def _get_result_export_dest(target, output, file_ext='xml'):
    """
    Get the full absolute path (including file name) for a given ``result``.

    As a side effect, intermediate directories are created such that the file
    can be created and written to immediately.

    :param target:
        Destination directory location for exported files OR a file-like
        object. If file-like, we just simply return it.
    :param output:
        :class:`~openquake.engine.db.models.Output`.
    :param file_ext:
        Desired file extension for the output file.
        Defaults to 'xml'.

    :returns:
        Full path (including filename) to the destination export file.
        If the ``target`` is a file-like, we don't do anything special
        and simply return it.
    """
    if not isinstance(target, basestring):
        # It's not a file path. In this case, we expect a file-like object.
        # Just return it.
        return target

    output_type = output.output_type

    filename = None

    if output_type in ('loss_curve', 'agg_loss_curve', 'event_loss_curve'):
        filename = LOSS_CURVE_FILENAME_FMT % dict(
            loss_curve_id=output.loss_curve.id,
        )
    elif output_type == 'loss_map':
        filename = LOSS_MAP_FILENAME_FMT % dict(
            loss_map_id=output.loss_map.id,
            file_ext=file_ext,
        )
    elif output_type == 'loss_fraction':
        filename = LOSS_FRACTION_FILENAME_FMT % dict(
            loss_fraction_id=output.loss_fraction.id,
        )
    elif output_type == 'bcr_distribution':
        filename = BCR_FILENAME_FMT % dict(
            bcr_distribution_id=output.bcr_distribution.id,
        )
    elif output_type == 'event_loss':
        filename = EVENT_LOSS_FILENAME_FMT % output.id
    elif output_type == 'event_loss_asset':
        filename = EVENT_LOSS_ASSET_FILENAME_FMT % output.id
    elif output_type == 'aggregate_loss':
        filename = AGGREGATE_LOSS_FILENAME_FMT % (output.aggregate_loss.id)
    elif output_type in ('dmg_dist_per_asset', 'dmg_dist_per_taxonomy',
                         'dmg_dist_total', 'collapse_map'):
        filename = '%(output_type)s-%(job_id)s.%(file_ext)s' % dict(
            output_type=output_type,
            job_id=output.oq_job.id,
            file_ext=file_ext,
        )

    return os.path.abspath(os.path.join(target, filename))

def _export_common(output, loss_type):
    """
    Returns a dict containing the output metadata which are serialized
    by nrml writers before actually writing the `output` data.
    """
    metadata = output.hazard_metadata
    if metadata.sm_path is not None:
        source_model_tree_path = core.LT_PATH_JOIN_TOKEN.join(
            metadata.sm_path)
    else:
        source_model_tree_path = None
    if metadata.gsim_path is not None:
        gsim_tree_path = core.LT_PATH_JOIN_TOKEN.join(metadata.gsim_path)
    else:
        gsim_tree_path = None

    return dict(investigation_time=metadata.investigation_time,
                statistics=metadata.statistics,
                quantile_value=metadata.quantile,
                source_model_tree_path=source_model_tree_path,
                gsim_tree_path=gsim_tree_path,
                unit=output.oq_job.risk_calculation.exposure_model.unit(
                    loss_type),
                loss_type=loss_type)

@core.export_output.add(('agg_loss_curve', 'xml'))
def export_agg_loss_curve_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss curves
    serializers
    """
    args = _export_common(output, output.loss_curve.loss_type)
    dest = _get_result_export_dest(target, output)
    risk_writers.AggregateLossCurveXMLWriter(dest, **args).serialize(
        output.loss_curve.aggregatelosscurvedata)
    return dest

@core.export_output.add(('loss_curve', 'xml'), ('event_loss_curve', 'xml'))
def export_loss_curve_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss curves
    serializer
    """
    args = _export_common(output, output.loss_curve.loss_type)
    dest = _get_result_export_dest(target, output)
    args['insured'] = output.loss_curve.insured

    data = output.loss_curve.losscurvedata_set.all().order_by('asset_ref')

    risk_writers.LossCurveXMLWriter(dest, **args).serialize(data)
    return dest

@core.export_output.add(('loss_map', 'xml'), ('loss_map', 'geojson'))
def export_loss_map(key, output, target):
    """
    General loss map export code.
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.loss_map.loss_type)
    dest = _get_result_export_dest(target, output, file_ext=key[1])
    args.update(dict(
        poe=output.loss_map.poe,
        loss_category=risk_calculation.exposure_model.category))

    writercls = (risk_writers.LossMapXMLWriter if key[1] == 'xml'
                 else risk_writers.LossMapGeoJSONWriter)
    writercls(dest, **args).serialize(
        models.order_by_location(
            output.loss_map.lossmapdata_set.all().order_by('asset_ref')))
    return dest

@core.export_output.add(('loss_fraction', 'xml'))
def export_loss_fraction_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss fractions
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.loss_fraction.loss_type)
    hazard_metadata = models.Output.HazardMetadata(
        investigation_time=args['investigation_time'],
        statistics=args['statistics'],
        quantile=args.get('quantile_value'),
        sm_path=args['source_model_tree_path'],
        gsim_path=args['gsim_tree_path'])
    dest = _get_result_export_dest(target, output)
    poe = output.loss_fraction.poe
    variable = output.loss_fraction.variable
    loss_category = risk_calculation.exposure_model.category
    risk_writers.LossFractionsWriter(
        dest, variable, args['unit'], args['loss_type'],
        loss_category, hazard_metadata, poe).serialize(
        output.loss_fraction.total_fractions(),
        output.loss_fraction)
    return dest

@core.export_output.add(('bcr_distribution', 'xml'))
def export_bcr_distribution_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml bcr distribution
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.bcr_distribution.loss_type)

    dest = _get_result_export_dest(target, output)
    args.update(
        dict(interest_rate=risk_calculation.interest_rate,
             asset_life_expectancy=risk_calculation.asset_life_expectancy))
    del args['investigation_time']

    risk_writers.BCRMapXMLWriter(dest, **args).serialize(
        output.bcr_distribution.bcrdistributiondata_set.all().order_by(
            'asset_ref'))
    return dest

# XXX: clearly this is not a good approach for large exposures
@core.export_output.add(('collapse_map', 'xml'),
                        ('dmg_dist_per_taxonomy', 'xml'),
                        ('dmg_dist_per_asset', 'xml'),
                        ('dmg_dist_total', 'xml'))
def export_dmg_dist(key, output, target):
    """
    Export the damage distribution identified
    by the given output to the `target`.

    :param output:
        DB output record which identifies the distribution. A
        :class:`openquake.engine.db.models.Output` object.
    :param target:
        Destination directory of the exported file, or a file-like object
    """
    job = output.oq_job
    dest = _get_result_export_dest(target, output)
    dmg_states = list(models.DmgState.objects.filter(
        risk_calculation__id=job.id).order_by('lsi'))
    damagecls, writercls = DAMAGE[key[0]]
    if writercls is risk_writers.CollapseMapXMLWriter:  # special case
        writer = writercls(dest)
        data = damagecls.objects.filter(dmg_state=dmg_states[-1])
    else:
        writer = writercls(dest, [ds.dmg_state for ds in dmg_states])
        data = damagecls.objects.filter(
            dmg_state__risk_calculation__id=job.id)
    writer.serialize(data.order_by('dmg_state__lsi'))
    return dest

@core.export_output.add(('aggregate_loss', 'csv'))
def export_aggregate_loss_csv(key, output, target):
    """
    Export aggregate losses in CSV
    """
    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile, delimiter='|')
        writer.writerow(['Mean', 'Standard Deviation'])
        writer.writerow([output.aggregate_loss.mean,
                        output.aggregate_loss.std_dev])
    return dest

@core.export_output.add(('event_loss', 'csv'))
def export_event_loss_csv(key, output, target):
    """
    Export Event Loss Table in CSV format
    """

    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['Rupture', 'Magnitude', 'Aggregate Loss'])

        for event_loss in models.EventLossData.objects.filter(
                event_loss__output=output).select_related().order_by(
                '-aggregate_loss'):
            writer.writerow([event_loss.rupture.tag,
                             "%.07f" % event_loss.rupture.rupture.magnitude,
                             "%.07f" % event_loss.aggregate_loss])
    return dest

@core.export_output.add(('event_loss_asset', 'csv'))
def export_event_loss_asset_csv(key, output, target):
    """
    Export Event Loss Per Asset in CSV format
    """
    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['Rupture', 'Asset', 'Magnitude', 'Loss'])

        for event_loss in models.EventLossAsset.objects.filter(
                event_loss__output=output).select_related().order_by(
                '-loss'):
            writer.writerow([event_loss.rupture.tag,
                             event_loss.asset.asset_ref,
                             "%.07f" % event_loss.rupture.rupture.magnitude,
                             "%.07f" % event_loss.loss])
    return dest
<commit_msg>Making use of the new DamageWriter<commit_after># Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for exporting risk artifacts from the database.
"""

import os
import csv

from openquake.engine.db import models
from openquake.engine.export import core
from openquake.engine.utils import FileWrapper
from openquake.commonlib import risk_writers

LOSS_CURVE_FILENAME_FMT = 'loss-curves-%(loss_curve_id)s.xml'
LOSS_MAP_FILENAME_FMT = 'loss-maps-%(loss_map_id)s.%(file_ext)s'
LOSS_FRACTION_FILENAME_FMT = 'loss-fractions-%(loss_fraction_id)s.xml'
AGGREGATE_LOSS_FILENAME_FMT = 'aggregate-loss-%s.csv'
BCR_FILENAME_FMT = 'bcr-distribution-%(bcr_distribution_id)s.xml'
EVENT_LOSS_FILENAME_FMT = 'event-loss-%s.csv'
EVENT_LOSS_ASSET_FILENAME_FMT = 'event-loss-asset-%s.csv'

DAMAGE = dict(
    collapse_map=models.DmgDistPerAsset,
    dmg_dist_per_asset=models.DmgDistPerAsset,
    dmg_dist_per_taxonomy=models.DmgDistPerTaxonomy,
    dmg_dist_total=models.DmgDistTotal)

def _get_result_export_dest(target, output, file_ext='xml'):
    """
    Get the full absolute path (including file name) for a given ``result``.

    As a side effect, intermediate directories are created such that the file
    can be created and written to immediately.

    :param target:
        Destination directory location for exported files OR a file-like
        object. If file-like, we just simply return it.
    :param output:
        :class:`~openquake.engine.db.models.Output`.
    :param file_ext:
        Desired file extension for the output file.
        Defaults to 'xml'.

    :returns:
        Full path (including filename) to the destination export file.
        If the ``target`` is a file-like, we don't do anything special
        and simply return it.
    """
    if not isinstance(target, basestring):
        # It's not a file path. In this case, we expect a file-like object.
        # Just return it.
        return target

    output_type = output.output_type

    filename = None

    if output_type in ('loss_curve', 'agg_loss_curve', 'event_loss_curve'):
        filename = LOSS_CURVE_FILENAME_FMT % dict(
            loss_curve_id=output.loss_curve.id,
        )
    elif output_type == 'loss_map':
        filename = LOSS_MAP_FILENAME_FMT % dict(
            loss_map_id=output.loss_map.id,
            file_ext=file_ext,
        )
    elif output_type == 'loss_fraction':
        filename = LOSS_FRACTION_FILENAME_FMT % dict(
            loss_fraction_id=output.loss_fraction.id,
        )
    elif output_type == 'bcr_distribution':
        filename = BCR_FILENAME_FMT % dict(
            bcr_distribution_id=output.bcr_distribution.id,
        )
    elif output_type == 'event_loss':
        filename = EVENT_LOSS_FILENAME_FMT % output.id
    elif output_type == 'event_loss_asset':
        filename = EVENT_LOSS_ASSET_FILENAME_FMT % output.id
    elif output_type == 'aggregate_loss':
        filename = AGGREGATE_LOSS_FILENAME_FMT % (output.aggregate_loss.id)
    elif output_type in ('dmg_dist_per_asset', 'dmg_dist_per_taxonomy',
                         'dmg_dist_total', 'collapse_map'):
        filename = '%(output_type)s-%(job_id)s.%(file_ext)s' % dict(
            output_type=output_type,
            job_id=output.oq_job.id,
            file_ext=file_ext,
        )

    return os.path.abspath(os.path.join(target, filename))

def _export_common(output, loss_type):
    """
    Returns a dict containing the output metadata which are serialized
    by nrml writers before actually writing the `output` data.
    """
    metadata = output.hazard_metadata
    if metadata.sm_path is not None:
        source_model_tree_path = core.LT_PATH_JOIN_TOKEN.join(
            metadata.sm_path)
    else:
        source_model_tree_path = None
    if metadata.gsim_path is not None:
        gsim_tree_path = core.LT_PATH_JOIN_TOKEN.join(metadata.gsim_path)
    else:
        gsim_tree_path = None

    return dict(investigation_time=metadata.investigation_time,
                statistics=metadata.statistics,
                quantile_value=metadata.quantile,
                source_model_tree_path=source_model_tree_path,
                gsim_tree_path=gsim_tree_path,
                unit=output.oq_job.risk_calculation.exposure_model.unit(
                    loss_type),
                loss_type=loss_type)

@core.export_output.add(('agg_loss_curve', 'xml'))
def export_agg_loss_curve_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss curves
    serializers
    """
    args = _export_common(output, output.loss_curve.loss_type)
    dest = _get_result_export_dest(target, output)
    risk_writers.AggregateLossCurveXMLWriter(dest, **args).serialize(
        output.loss_curve.aggregatelosscurvedata)
    return dest

@core.export_output.add(('loss_curve', 'xml'), ('event_loss_curve', 'xml'))
def export_loss_curve_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss curves
    serializer
    """
    args = _export_common(output, output.loss_curve.loss_type)
    dest = _get_result_export_dest(target, output)
    args['insured'] = output.loss_curve.insured

    data = output.loss_curve.losscurvedata_set.all().order_by('asset_ref')

    risk_writers.LossCurveXMLWriter(dest, **args).serialize(data)
    return dest

@core.export_output.add(('loss_map', 'xml'), ('loss_map', 'geojson'))
def export_loss_map(key, output, target):
    """
    General loss map export code.
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.loss_map.loss_type)
    dest = _get_result_export_dest(target, output, file_ext=key[1])
    args.update(dict(
        poe=output.loss_map.poe,
        loss_category=risk_calculation.exposure_model.category))

    writercls = (risk_writers.LossMapXMLWriter if key[1] == 'xml'
                 else risk_writers.LossMapGeoJSONWriter)
    writercls(dest, **args).serialize(
        models.order_by_location(
            output.loss_map.lossmapdata_set.all().order_by('asset_ref')))
    return dest

@core.export_output.add(('loss_fraction', 'xml'))
def export_loss_fraction_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml loss fractions
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.loss_fraction.loss_type)
    hazard_metadata = models.Output.HazardMetadata(
        investigation_time=args['investigation_time'],
        statistics=args['statistics'],
        quantile=args.get('quantile_value'),
        sm_path=args['source_model_tree_path'],
        gsim_path=args['gsim_tree_path'])
    dest = _get_result_export_dest(target, output)
    poe = output.loss_fraction.poe
    variable = output.loss_fraction.variable
    loss_category = risk_calculation.exposure_model.category
    risk_writers.LossFractionsWriter(
        dest, variable, args['unit'], args['loss_type'],
        loss_category, hazard_metadata, poe).serialize(
        output.loss_fraction.total_fractions(),
        output.loss_fraction)
    return dest

@core.export_output.add(('bcr_distribution', 'xml'))
def export_bcr_distribution_xml(key, output, target):
    """
    Export `output` to `target` by using a nrml bcr distribution
    serializer
    """
    risk_calculation = output.oq_job.risk_calculation
    args = _export_common(output, output.bcr_distribution.loss_type)

    dest = _get_result_export_dest(target, output)
    args.update(
        dict(interest_rate=risk_calculation.interest_rate,
             asset_life_expectancy=risk_calculation.asset_life_expectancy))
    del args['investigation_time']

    risk_writers.BCRMapXMLWriter(dest, **args).serialize(
        output.bcr_distribution.bcrdistributiondata_set.all().order_by(
            'asset_ref'))
    return dest

# clearly this is not a good approach for large exposures
@core.export_output.add(('collapse_map', 'xml'),
                        ('dmg_dist_per_taxonomy', 'xml'),
                        ('dmg_dist_per_asset', 'xml'),
                        ('dmg_dist_total', 'xml'))
def export_dmg_dist(key, output, target):
    """
    Export the damage distribution identified
    by the given output to the `target`.

    :param output:
        DB output record which identifies the distribution. A
        :class:`openquake.engine.db.models.Output` object.
    :param target:
        Destination directory of the exported file, or a file-like object
    """
    job = output.oq_job
    dest = _get_result_export_dest(target, output)
    damage_states = [
        ds.dmg_state for ds in models.DmgState.objects.filter(
            risk_calculation__id=job.id).order_by('lsi')]
    writer = risk_writers.DamageWriter(damage_states)
    damagecls = DAMAGE[key[0]]
    if key[0] == 'collapse_map':
        data = damagecls.objects.filter(dmg_state__dmg_state=damage_states[-1])
    else:
        data = damagecls.objects.filter(dmg_state__risk_calculation__id=job.id)
    writer.to_nrml(key[0], data, dest)
    return dest

@core.export_output.add(('aggregate_loss', 'csv'))
def export_aggregate_loss_csv(key, output, target):
    """
    Export aggregate losses in CSV
    """
    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile, delimiter='|')
        writer.writerow(['Mean', 'Standard Deviation'])
        writer.writerow([output.aggregate_loss.mean,
                        output.aggregate_loss.std_dev])
    return dest

@core.export_output.add(('event_loss', 'csv'))
def export_event_loss_csv(key, output, target):
    """
    Export Event Loss Table in CSV format
    """

    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['Rupture', 'Magnitude', 'Aggregate Loss'])

        for event_loss in models.EventLossData.objects.filter(
                event_loss__output=output).select_related().order_by(
                '-aggregate_loss'):
            writer.writerow([event_loss.rupture.tag,
                             "%.07f" % event_loss.rupture.rupture.magnitude,
                             "%.07f" % event_loss.aggregate_loss])
    return dest

@core.export_output.add(('event_loss_asset', 'csv'))
def export_event_loss_asset_csv(key, output, target):
    """
    Export Event Loss Per Asset in CSV format
    """
    dest = _get_result_export_dest(target, output)

    with FileWrapper(dest, mode='wb') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['Rupture', 'Asset', 'Magnitude', 'Loss'])

        for event_loss in models.EventLossAsset.objects.filter(
                event_loss__output=output).select_related().order_by(
                '-loss'):
            writer.writerow([event_loss.rupture.tag,
                             event_loss.asset.asset_ref,
                             "%.07f" % event_loss.rupture.rupture.magnitude,
                             "%.07f" % event_loss.loss])
    return dest
<|endoftext|>