<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import io
import os
import logging

from openquake.baselib import sap
from openquake.baselib import datastore, hdf5
from openquake.commonlib.writers import write_csv
from openquake.commonlib import util
from openquake.calculators.views import view, rst_table
from openquake.calculators.extract import extract

def str_or_int(calc_id):
    try:
        return int(calc_id)
    except ValueError:
        return calc_id

@sap.script
def show(what='contents', calc_id=-1, extra=()):
    """
    Show the content of a datastore (by default the last one).
    """
    datadir = datastore.get_datadir()
    if what == 'all':  # show all
        if not os.path.exists(datadir):
            return
        rows = []
        for calc_id in datastore.get_calc_ids(datadir):
            try:
                ds = util.read(calc_id)
                oq = ds['oqparam']
                cmode, descr = oq.calculation_mode, oq.description
            except Exception:
                # invalid datastore file, or missing calculation_mode
                # and description attributes, perhaps due to a manual kill
                f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
                logging.warning('Unreadable datastore %s', f)
                continue
            else:
                rows.append((calc_id, cmode, descr.encode('utf-8')))
        for row in sorted(rows, key=lambda row: row[0]):  # by calc_id
            print('#%d %s: %s' % row)
        return

    ds = util.read(calc_id)

    # this part is experimental
    if view.keyfunc(what) in view:
        print(view(what, ds))
    elif what.split('/', 1)[0] in extract:
        print(extract(ds, what, *extra))
    elif what in ds:
        obj = ds.getitem(what)
        if hasattr(obj, 'items'):  # is a group of datasets
            print(obj)
        else:  # is a single dataset
            aw = hdf5.ArrayWrapper.from_(obj)
            if hasattr(aw, 'shape_descr'):
                print(rst_table(aw.to_table()))
            else:
                print(write_csv(io.BytesIO(), aw.array).decode('utf8'))
    else:
        print('%s not found' % what)

    ds.close()

show.arg('what', 'key or view of the datastore')
show.arg('calc_id', 'calculation ID or datastore path', type=str_or_int)
show.arg('extra', 'extra arguments', nargs='*')
<commit_msg>Added a refresh in oq show [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import io
import os
import logging

from openquake.baselib import sap
from openquake.baselib import datastore, hdf5
from openquake.commonlib.writers import write_csv
from openquake.commonlib import util
from openquake.calculators.views import view, rst_table
from openquake.calculators.extract import extract

def str_or_int(calc_id):
    try:
        return int(calc_id)
    except ValueError:
        return calc_id

@sap.script
def show(what='contents', calc_id=-1, extra=()):
    """
    Show the content of a datastore (by default the last one).
    """
    datadir = datastore.get_datadir()
    if what == 'all':  # show all
        if not os.path.exists(datadir):
            return
        rows = []
        for calc_id in datastore.get_calc_ids(datadir):
            try:
                ds = util.read(calc_id)
                oq = ds['oqparam']
                cmode, descr = oq.calculation_mode, oq.description
            except Exception:
                # invalid datastore file, or missing calculation_mode
                # and description attributes, perhaps due to a manual kill
                f = os.path.join(datadir, 'calc_%s.hdf5' % calc_id)
                logging.warning('Unreadable datastore %s', f)
                continue
            else:
                rows.append((calc_id, cmode, descr.encode('utf-8')))
        for row in sorted(rows, key=lambda row: row[0]):  # by calc_id
            print('#%d %s: %s' % row)
        return

    ds = util.read(calc_id)

    # this part is experimental
    if view.keyfunc(what) in view:
        print(view(what, ds))
    elif what.split('/', 1)[0] in extract:
        print(extract(ds, what, *extra))
    elif what in ds:
        obj = ds.getitem(what)
        if hasattr(obj, 'items'):  # is a group of datasets
            print(obj)
        else:  # is a single dataset
            obj.refresh()  # for SWMR mode
            aw = hdf5.ArrayWrapper.from_(obj)
            if hasattr(aw, 'shape_descr'):
                print(rst_table(aw.to_table()))
            else:
                print(write_csv(io.BytesIO(), aw.array).decode('utf8'))
    else:
        print('%s not found' % what)

    ds.close()

show.arg('what', 'key or view of the datastore')
show.arg('calc_id', 'calculation ID or datastore path', type=str_or_int)
show.arg('extra', 'extra arguments', nargs='*')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import io
import os
import re
import sys
import json
import time
import signal
import logging
import traceback
import platform
import numpy
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.python3compat import decode
from openquake.baselib import (
    parallel, general, config, datastore, __version__, zeromq as z)
from openquake.hazardlib import nrml
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, views, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
TERMINATE = config.distribution.terminate_workers_on_revoke
OQ_DISTRIBUTE = parallel.oq_distribute()

_PID = os.getpid()  # the PID
_PPID = os.getppid()  # the controlling terminal PID

if OQ_DISTRIBUTE == 'zmq':

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the available
        worker pools .
        """
        num_workers = 0
        w = config.zworkers
        for host, _cores in [hc.split() for hc in w.host_cores.split(',')]:
            url = 'tcp://%s:%s' % (host, w.ctrl_port)
            with z.Socket(url, z.zmq.REQ, 'connect') as sock:
                if not general.socket_ready(url):
                    logs.LOG.warn('%s is not running', host)
                    continue
                num_workers += sock.send('get_num_workers')
        OqParam.concurrent_tasks.default = num_workers * 3
        logs.LOG.info('Using %d zmq workers', num_workers)

elif OQ_DISTRIBUTE.startswith('celery'):
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the number of available
        celery workers.
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
        OqParam.concurrent_tasks.default = num_cores * 3
        logs.LOG.info(
            'Using %s, %d cores', ', '.join(sorted(stats)), num_cores)

    def celery_cleanup(terminate, task_ids=()):
        """
        Release the resources used by an openquake job.
        In particular revoke the running tasks (if any).

        :param bool terminate: the celery revoke command terminate flag
        :param task_ids: celery task IDs
        """
        # Using the celery API, terminate and revoke and terminate any running
        # tasks associated with the current job.
        if task_ids:
            logs.LOG.warn('Revoking %d tasks', len(task_ids))
        else:  # this is normal when OQ_DISTRIBUTE=no
            logs.LOG.debug('No task to revoke')
        for tid in task_ids:
            celery.task.control.revoke(tid, terminate=terminate)
            logs.LOG.debug('Revoked task %s', tid)

def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    rlzs = dstore['csm_info'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    # expose gmf_data only if < 10 MB
    if oq.ground_motion_fields and calcmode == 'event_based':
        nbytes = dstore['gmf_data'].attrs['nbytes']
        if nbytes < 10 * 1024 ** 2:
            dskeys.add('gmf_data')
    if 'scenario' not in calcmode:  # export sourcegroups.csv
        dskeys.add('sourcegroups')
    hdf5 = dstore.hdf5
    if (len(rlzs) == 1 and 'poes' in hdf5) or 'hcurves' in hdf5:
        dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if 'avg_losses-stats' in dstore or (
            'avg_losses-rlzs' in dstore and len(rlzs)):
        dskeys.add('avg_losses-stats')
    if 'curves-stats' in dstore:
        logs.LOG.warn('loss curves are exportable with oq export')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'all_loss_ratios' in dskeys:
        dskeys.remove('all_loss_ratios')  # export only specific IDs
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'rup_loss_table' in dskeys:  # keep it hidden for the moment
        dskeys.remove('rup_loss_table')
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys & exportable))

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def inhibitSigInt(signum, _stack):
    logs.LOG.warn('Killing job, please wait')

def raiseMasterKilled(signum, _stack):
    """
    When a SIGTERM is received, raise the MasterKilled
    exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    # Disable further CTRL-C to allow tasks revocation when Celery is used
    if OQ_DISTRIBUTE.startswith('celery'):
        signal.signal(signal.SIGINT, inhibitSigInt)

    msg = 'Received a signal %d' % signum
    if signum in (signal.SIGTERM, signal.SIGINT):
        msg = 'The openquake master process was killed manually'

    # kill the calculation only if os.getppid() != _PPID, i.e. the controlling
    # terminal died; in the workers, do nothing
    # NB: there is no SIGHUP on Windows
    if hasattr(signal, 'SIGHUP'):
        if signum == signal.SIGHUP:
            if os.getppid() == _PPID:
                return
            else:
                msg = 'The openquake master lost its controlling terminal'

    parallel.Starmap.shutdown()
    raise MasterKilled(msg)

# register the raiseMasterKilled callback for SIGTERM
# when using the Django development server this module is imported by a thread,
# so one gets a `ValueError: signal only works in main thread` that
# can be safely ignored
try:
    signal.signal(signal.SIGTERM, raiseMasterKilled)
    signal.signal(signal.SIGINT, raiseMasterKilled)
    if hasattr(signal, 'SIGHUP'):
        signal.signal(signal.SIGHUP, raiseMasterKilled)
except ValueError:
    pass

def zip(job_ini, archive_zip, oq=None, log=logging.info):
    """
    Zip the given job.ini file into the given archive, together with all
    related files.
    """
    if not os.path.exists(job_ini):
        sys.exit('%s does not exist' % job_ini)
    if isinstance(archive_zip, str):  # actually it should be path-like
        if not archive_zip.endswith('.zip'):
            sys.exit('%s does not end with .zip' % archive_zip)
        if os.path.exists(archive_zip):
            sys.exit('%s exists already' % archive_zip)
    logging.basicConfig(level=logging.INFO)
    # do not validate to avoid permissions error on the export_dir
    oq = oq or readinput.get_oqparam(job_ini, validate=False)
    files = set()

    # collect .hdf5 tables for the GSIMs, if any
    if 'gsim_logic_tree' in oq.inputs or oq.gsim:
        gsim_lt = readinput.get_gsim_lt(oq)
        for gsims in gsim_lt.values.values():
            for gsim in gsims:
                table = getattr(gsim, 'GMPE_TABLE', None)
                if table:
                    files.add(table)

    # collect exposure.csv, if any
    exposure_xml = oq.inputs.get('exposure')
    if exposure_xml:
        dname = os.path.dirname(exposure_xml)
        expo = nrml.read(exposure_xml, stop='asset')[0]
        if not expo.assets:
            exposure_csv = (~expo.assets).strip()
            for csv in exposure_csv.split():
                if csv and os.path.exists(os.path.join(dname, csv)):
                    files.add(os.path.join(dname, csv))

    # collection .hdf5 UCERF file, if any
    if oq.calculation_mode.startswith('ucerf_'):
        sm = nrml.read(oq.inputs['source_model'])
        fname = sm.sourceModel.UCERFSource['filename']
        f = os.path.join(os.path.dirname(oq.inputs['source_model']), fname)
        files.add(os.path.normpath(f))

    # collect all other files
    for key in oq.inputs:
        fname = oq.inputs[key]
        if isinstance(fname, list):
            for f in fname:
                files.add(os.path.normpath(f))
        else:
            files.add(os.path.normpath(fname))
    general.zipfiles(files, archive_zip, log=log)

def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file, hc_id=hazard_calculation_id)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.get_datadir(),
                        hazard_calculation_id)
    return job_id, oq

def run_calc(job_id, oqparam, log_level, log_file, exports,
             hazard_calculation_id=None, **kw):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    setproctitle('oq-job-%d' % job_id)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if OQ_DISTRIBUTE.startswith(('celery', 'zmq')):
            set_concurrent_tasks_default()
        msg = check_obsolete_version(oqparam.calculation_mode)
        if msg:
            logs.LOG.warn(msg)
        calc = base.calculators(oqparam, calc_id=job_id)
        calc.from_engine = True
        job_zip = oqparam.inputs.get('job_zip')
        if job_zip:  # the input was zipped from the beginning
            calc.datastore['job_zip'] = numpy.array(open(job_zip, 'rb').read())
        else:  # zip the input
            logs.LOG.info('zipping and storing the input files')
            bio = io.BytesIO()
            zip(oqparam.inputs['job_ini'], bio, oqparam, logging.debug)
            calc.datastore['job_zip'] = numpy.array(bio.getvalue())
        tb = 'None\n'
        try:
            logs.dbcmd('update_job', job_id, {'status': 'executing',
                                              'pid': _PID})
            t0 = time.time()
            calc.run(exports=exports,
                     hazard_calculation_id=hazard_calculation_id,
                     close=False, **kw)  # don't close the datastore too soon
            duration = time.time() - t0
            expose_outputs(calc.datastore)
            calc._monitor.flush()
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, duration)
            logs.dbcmd('finish', job_id, 'complete')
        except BaseException:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except BaseException:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if OQ_DISTRIBUTE.startswith('celery'):
                    celery_cleanup(TERMINATE, parallel.Starmap.task_ids)
            except BaseException:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except Exception:  # page not available or wrong version tag
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<commit_msg>Fixed error management in engine.py [skip hazardlib]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import io
import os
import re
import sys
import json
import time
import signal
import logging
import traceback
import platform
import numpy
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.python3compat import decode
from openquake.baselib import (
    parallel, general, config, datastore, __version__, zeromq as z)
from openquake.hazardlib import nrml
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, views, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
TERMINATE = config.distribution.terminate_workers_on_revoke
OQ_DISTRIBUTE = parallel.oq_distribute()

_PID = os.getpid()  # the PID
_PPID = os.getppid()  # the controlling terminal PID

if OQ_DISTRIBUTE == 'zmq':

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the available
        worker pools .
        """
        num_workers = 0
        w = config.zworkers
        for host, _cores in [hc.split() for hc in w.host_cores.split(',')]:
            url = 'tcp://%s:%s' % (host, w.ctrl_port)
            with z.Socket(url, z.zmq.REQ, 'connect') as sock:
                if not general.socket_ready(url):
                    logs.LOG.warn('%s is not running', host)
                    continue
                num_workers += sock.send('get_num_workers')
        OqParam.concurrent_tasks.default = num_workers * 3
        logs.LOG.info('Using %d zmq workers', num_workers)

elif OQ_DISTRIBUTE.startswith('celery'):
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the number of available
        celery workers.
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
        OqParam.concurrent_tasks.default = num_cores * 3
        logs.LOG.info(
            'Using %s, %d cores', ', '.join(sorted(stats)), num_cores)

    def celery_cleanup(terminate, task_ids=()):
        """
        Release the resources used by an openquake job.
        In particular revoke the running tasks (if any).

        :param bool terminate: the celery revoke command terminate flag
        :param task_ids: celery task IDs
        """
        # Using the celery API, terminate and revoke and terminate any running
        # tasks associated with the current job.
        if task_ids:
            logs.LOG.warn('Revoking %d tasks', len(task_ids))
        else:  # this is normal when OQ_DISTRIBUTE=no
            logs.LOG.debug('No task to revoke')
        for tid in task_ids:
            celery.task.control.revoke(tid, terminate=terminate)
            logs.LOG.debug('Revoked task %s', tid)

def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    rlzs = dstore['csm_info'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    # expose gmf_data only if < 10 MB
    if oq.ground_motion_fields and calcmode == 'event_based':
        nbytes = dstore['gmf_data'].attrs['nbytes']
        if nbytes < 10 * 1024 ** 2:
            dskeys.add('gmf_data')
    if 'scenario' not in calcmode:  # export sourcegroups.csv
        dskeys.add('sourcegroups')
    hdf5 = dstore.hdf5
    if (len(rlzs) == 1 and 'poes' in hdf5) or 'hcurves' in hdf5:
        dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if 'avg_losses-stats' in dstore or (
            'avg_losses-rlzs' in dstore and len(rlzs)):
        dskeys.add('avg_losses-stats')
    if 'curves-stats' in dstore:
        logs.LOG.warn('loss curves are exportable with oq export')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'all_loss_ratios' in dskeys:
        dskeys.remove('all_loss_ratios')  # export only specific IDs
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'rup_loss_table' in dskeys:  # keep it hidden for the moment
        dskeys.remove('rup_loss_table')
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys & exportable))

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def inhibitSigInt(signum, _stack):
    logs.LOG.warn('Killing job, please wait')

def raiseMasterKilled(signum, _stack):
    """
    When a SIGTERM is received, raise the MasterKilled
    exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    # Disable further CTRL-C to allow tasks revocation when Celery is used
    if OQ_DISTRIBUTE.startswith('celery'):
        signal.signal(signal.SIGINT, inhibitSigInt)

    msg = 'Received a signal %d' % signum
    if signum in (signal.SIGTERM, signal.SIGINT):
        msg = 'The openquake master process was killed manually'

    # kill the calculation only if os.getppid() != _PPID, i.e. the controlling
    # terminal died; in the workers, do nothing
    # NB: there is no SIGHUP on Windows
    if hasattr(signal, 'SIGHUP'):
        if signum == signal.SIGHUP:
            if os.getppid() == _PPID:
                return
            else:
                msg = 'The openquake master lost its controlling terminal'

    parallel.Starmap.shutdown()
    raise MasterKilled(msg)

# register the raiseMasterKilled callback for SIGTERM
# when using the Django development server this module is imported by a thread,
# so one gets a `ValueError: signal only works in main thread` that
# can be safely ignored
try:
    signal.signal(signal.SIGTERM, raiseMasterKilled)
    signal.signal(signal.SIGINT, raiseMasterKilled)
    if hasattr(signal, 'SIGHUP'):
        signal.signal(signal.SIGHUP, raiseMasterKilled)
except ValueError:
    pass

def zip(job_ini, archive_zip, oq=None, log=logging.info):
    """
    Zip the given job.ini file into the given archive, together with all
    related files.
    """
    if not os.path.exists(job_ini):
        sys.exit('%s does not exist' % job_ini)
    if isinstance(archive_zip, str):  # actually it should be path-like
        if not archive_zip.endswith('.zip'):
            sys.exit('%s does not end with .zip' % archive_zip)
        if os.path.exists(archive_zip):
            sys.exit('%s exists already' % archive_zip)
    logging.basicConfig(level=logging.INFO)
    # do not validate to avoid permissions error on the export_dir
    oq = oq or readinput.get_oqparam(job_ini, validate=False)
    files = set()

    # collect .hdf5 tables for the GSIMs, if any
    if 'gsim_logic_tree' in oq.inputs or oq.gsim:
        gsim_lt = readinput.get_gsim_lt(oq)
        for gsims in gsim_lt.values.values():
            for gsim in gsims:
                table = getattr(gsim, 'GMPE_TABLE', None)
                if table:
                    files.add(table)

    # collect exposure.csv, if any
    exposure_xml = oq.inputs.get('exposure')
    if exposure_xml:
        dname = os.path.dirname(exposure_xml)
        expo = nrml.read(exposure_xml, stop='asset')[0]
        if not expo.assets:
            exposure_csv = (~expo.assets).strip()
            for csv in exposure_csv.split():
                if csv and os.path.exists(os.path.join(dname, csv)):
                    files.add(os.path.join(dname, csv))

    # collection .hdf5 UCERF file, if any
    if oq.calculation_mode.startswith('ucerf_'):
        sm = nrml.read(oq.inputs['source_model'])
        fname = sm.sourceModel.UCERFSource['filename']
        f = os.path.join(os.path.dirname(oq.inputs['source_model']), fname)
        files.add(os.path.normpath(f))

    # collect all other files
    for key in oq.inputs:
        fname = oq.inputs[key]
        if isinstance(fname, list):
            for f in fname:
                files.add(os.path.normpath(f))
        else:
            files.add(os.path.normpath(fname))
    general.zipfiles(files, archive_zip, log=log)

def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file, hc_id=hazard_calculation_id)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.get_datadir(),
                        hazard_calculation_id)
    return job_id, oq

def run_calc(job_id, oqparam, log_level, log_file, exports,
             hazard_calculation_id=None, **kw):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    setproctitle('oq-job-%d' % job_id)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if OQ_DISTRIBUTE.startswith(('celery', 'zmq')):
            set_concurrent_tasks_default()
        msg = check_obsolete_version(oqparam.calculation_mode)
        if msg:
            logs.LOG.warn(msg)
        calc = base.calculators(oqparam, calc_id=job_id)
        calc.from_engine = True
        job_zip = oqparam.inputs.get('job_zip')
        tb = 'None\n'
        try:
            if job_zip:  # the input was zipped from the beginning
                calc.datastore['job_zip'] = numpy.array(
                    open(job_zip, 'rb').read())
            else:  # zip the input
                logs.LOG.info('zipping and storing the input files')
                bio = io.BytesIO()
                zip(oqparam.inputs['job_ini'], bio, oqparam, logging.debug)
                calc.datastore['job_zip'] = numpy.array(bio.getvalue())
            logs.dbcmd('update_job', job_id, {'status': 'executing',
                                              'pid': _PID})
            t0 = time.time()
            calc.run(exports=exports,
                     hazard_calculation_id=hazard_calculation_id,
                     close=False, **kw)  # don't close the datastore too soon
            duration = time.time() - t0
            expose_outputs(calc.datastore)
            calc._monitor.flush()
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, duration)
            logs.dbcmd('finish', job_id, 'complete')
        except BaseException:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except BaseException:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if OQ_DISTRIBUTE.startswith('celery'):
                    celery_cleanup(TERMINATE, parallel.Starmap.task_ids)
            except BaseException:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except Exception:  # page not available or wrong version tag
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<|endoftext|>


<commit_before>print "Loading ", __name__

import poser
import Numeric as num

from Poser.unimesh import Unimesh

import geometry
reload(geometry)
from geometry import Geometry

import submesh
reload(submesh)
from submesh import Submesh

class GeometryExporter(Geometry):
    def __init__(self, actor_or_figure, material_converter = None, options = {}):
        self.convert_material = material_converter or (
            lambda mat, key: "# Material \'%s/%s\'" % (key, mat.Name()))

        if isinstance(actor_or_figure, poser.ActorType):
            self.actor = actor_or_figure
            self.figure = self.actor.ItsFigure()
            data = self.get_data_from_actor()
        elif isinstance(actor_or_figure, poser.FigureType):
            self.figure = actor_or_figure
            data = self.get_data_from_unimesh()
        else:
            raise RuntimeError("Argument must be an actor or figure.")

        verts, polys, poly_mats, tverts, tpolys, self.materials = data
        Geometry.__init__(self, verts, polys, poly_mats, None, tverts, tpolys)

        self.empty = not self.polys

        if not self.empty:
            if options.get('compute_normals', True):
                self.compute_normals()
            for i in xrange(options.get('subdivisionlevel', 0)):
                self.subdivide()

    def get_data_from_actor(self):
        geom = self.actor.Geometry()
        if not (geom and self.actor.Visible()):
            return [None] * 6

        sets = geom.Sets()
        tsets = geom.TexSets()

        verts = num.array([[v.X(), v.Y(), v.Z()]
                           for v in geom.WorldVertices()], 'd')
        polys = [sets[p.Start() : p.Start() + p.NumVertices()]
                 for p in geom.Polygons()]
        poly_mats = [p.MaterialIndex() for p in geom.Polygons()]
        if geom.TexVertices():
            tverts = num.array([[v.U(), v.V()]
                                for v in geom.TexVertices()], 'd')
            tpolys = [tsets[p.Start() : p.Start() + p.NumTexVertices()]
                      for p in geom.TexPolygons()]
        else:
            tverts = tpolys = []

        if self.figure:
            key = self.figure.Name()
        else:
            key = self.actor.Name()
        materials = [self.convert_material(mat, key)
                     for mat in geom.Materials()]

        return verts, polys, poly_mats, tverts, tpolys, materials

    def get_data_from_unimesh(self):
        uni = Unimesh(self.figure)
        key = self.figure.Name()
        materials = [self.convert_material(mat, key) for mat in uni.materials]

        return (uni.verts, uni.polys, uni.poly_mats,
                uni.tverts, uni.tpolys, materials)

    def write_submesh(self, file, sub):
        print >>file, 'Shape "mesh"'

        print >>file, ' "integer triindices" ['
        for u, v, w in sub.triangles:
            print >>file, u, v, w
        print >>file, ']\n'

        print >>file, ' "point P" ['
        for x, y, z in sub.points:
            print >>file, "%.8f %.8f %.8f" % (x, y, z)
        print >>file, ']\n'

        if sub.has_normals:
            print >>file, ' "normal N" ['
            for nx, ny, nz in sub.normals:
                print >>file, "%.8f %.8f %.8f" % (nx, ny, nz)
            print >>file, ']\n'

        if sub.has_texture_points:
            print >>file, ' "float uv" ['
            for u, v in sub.texture_points:
                print >>file, "%.8f %.8f" % (u, v)
            print >>file, ']\n'

    def write(self, file):
        if self.empty: return

        for mat_idx, material in enumerate(self.materials):
            indices = [i for i,n in enumerate(self.poly_mats) if n == mat_idx]

            if indices:
                print >>file, 'AttributeBegin'
                print >>file, material
                self.write_submesh(file, Submesh(self, indices))
                print >>file, 'AttributeEnd\n'
<commit_msg>GeometryExporter no longer inherits from Geometry<commit_after>print "Loading ", __name__

import poser
import Numeric as num

from Poser.unimesh import Unimesh

import geometry
reload(geometry)
from geometry import Geometry

import submesh
reload(submesh)
from submesh import Submesh

class GeometryExporter(object):
    def __init__(self, actor_or_figure, material_converter = None, options = {}):
        self.convert_material = material_converter or (
            lambda mat, key: "# Material \'%s/%s\'" % (key, mat.Name()))

        if isinstance(actor_or_figure, poser.ActorType):
            self.actor = actor_or_figure
            self.figure = self.actor.ItsFigure()
            data = self.get_data_from_actor()
        elif isinstance(actor_or_figure, poser.FigureType):
            self.figure = actor_or_figure
            data = self.get_data_from_unimesh()
        else:
            raise TypeError("Argument must be an actor or figure.")

        verts, polys, poly_mats, tverts, tpolys, self.materials = data
        self.geom = Geometry(verts, polys, poly_mats, None, tverts, tpolys)

        self.empty = not polys

        if not self.empty:
            if options.get('compute_normals', True):
                self.geom.compute_normals()
            for i in xrange(options.get('subdivisionlevel', 0)):
                self.geom.subdivide()

    def get_data_from_actor(self):
        geom = self.actor.Geometry()
        if not (geom and self.actor.Visible()):
            return [None] * 6

        sets = geom.Sets()
        tsets = geom.TexSets()

        verts = num.array([[v.X(), v.Y(), v.Z()]
                           for v in geom.WorldVertices()], 'd')
        polys = [sets[p.Start() : p.Start() + p.NumVertices()]
                 for p in geom.Polygons()]
        poly_mats = [p.MaterialIndex() for p in geom.Polygons()]
        if geom.TexVertices():
            tverts = num.array([[v.U(), v.V()]
                                for v in geom.TexVertices()], 'd')
            tpolys = [tsets[p.Start() : p.Start() + p.NumTexVertices()]
                      for p in geom.TexPolygons()]
        else:
            tverts = tpolys = []

        if self.figure:
            key = self.figure.Name()
        else:
            key = self.actor.Name()
        materials = [self.convert_material(mat, key)
                     for mat in geom.Materials()]

        return verts, polys, poly_mats, tverts, tpolys, materials

    def get_data_from_unimesh(self):
        uni = Unimesh(self.figure)
        key = self.figure.Name()
        materials = [self.convert_material(mat, key) for mat in uni.materials]

        return (uni.verts, uni.polys, uni.poly_mats,
                uni.tverts, uni.tpolys, materials)

    def write_submesh(self, file, sub):
        print >>file, 'Shape "mesh"'

        print >>file, ' "integer triindices" ['
        for u, v, w in sub.triangles:
            print >>file, u, v, w
        print >>file, ']\n'

        print >>file, ' "point P" ['
        for x, y, z in sub.points:
            print >>file, "%.8f %.8f %.8f" % (x, y, z)
        print >>file, ']\n'

        if sub.has_normals:
            print >>file, ' "normal N" ['
            for nx, ny, nz in sub.normals:
                print >>file, "%.8f %.8f %.8f" % (nx, ny, nz)
            print >>file, ']\n'

        if sub.has_texture_points:
            print >>file, ' "float uv" ['
            for u, v in sub.texture_points:
                print >>file, "%.8f %.8f" % (u, v)
            print >>file, ']\n'

    def write(self, file):
        if self.empty: return

        for mat_idx, material in enumerate(self.materials):
            indices = [i for i,n in enumerate(self.geom.poly_mats)
                       if n == mat_idx]

            if indices:
                print >>file, 'AttributeBegin'
                print >>file, material
                self.write_submesh(file, Submesh(self.geom, indices))
                print >>file, 'AttributeEnd\n'
<|endoftext|>


<commit_before>__author__ = 'ian'
import pandas as pd
import numpy as np
from scipy import stats
import sys
import matplotlib.pyplot as plt
import plotly.plotly as py
import plotly.graph_objs as go

#import data
data = pd.read_csv('masterdataNumerosity_sailfins.csv')
data['condition']=np.where(data['fishName'].str[:1]=='H','High','Low')
#
#
#
##############################
# side bias
tankRight = data.loc[data.tank=='R', ['propTimeRight', 'propTimeLeft']]
tankLeft = data.loc[data.tank=='L', ['propTimeRight', 'propTimeLeft']]
trace3 = go.Box(y=tankRight.propTimeRight,name='Right Tank Right Side')
trace2 = go.Box(y=tankRight.propTimeLeft,name='Right Tank Left Side')
trace1 = go.Box(y=tankLeft.propTimeRight,name='Left Tank Right Side')
trace0 = go.Box(y=tankLeft.propTimeLeft,name='Left Tank Left Side')
datum = [trace0, trace1, trace2, trace3]
sidebiasanovaF, sidebiasanovaP = stats.f_oneway(tankLeft.propTimeLeft, tankLeft.propTimeRight, tankRight.propTimeLeft, tankRight.propTimeRight)
sidebiasLeftLeftvLeftRightT, sidebiasLeftLeftvLeftRightP = stats.ttest_rel(tankLeft.propTimeLeft,tankLeft.propTimeRight)
sidebiasLeftLeftvRightRightT, sidebiasLeftLeftvRightRightP = stats.ttest_ind(tankLeft.propTimeLeft,tankRight.propTimeRight)

fig1 = go.Figure(data=datum)
plot_url = py.plot(fig1, filename='tank side bias')
print 'sidebiasanovaF: ', sidebiasanovaF
print 'sidebiasanovaP: ', sidebiasanovaP
print 'sidebiasLeftLeftvLeftRightT', sidebiasLeftLeftvLeftRightT
print 'sidebiasLeftLeftvLeftRightP', sidebiasLeftLeftvLeftRightP
###we see a bias in the left tank towards the right side/away from left side
### is this side bias correlated with thigmotaxis (our measure of stress)?
##############################
#
#
##############################
#stress levels (thigmotaxis)
righttank = data.loc[data.tank=='R',['propTimeEdge']]
lefttank = data.loc[data.tank=='L',['propTimeEdge']]
tr = go.Box(y=righttank.propTimeEdge,name='righttankstress')
tr2 = go.Box(y=lefttank.propTimeEdge, name='lefttankstress')
stressdata = [tr,tr2]
stressfig = go.Figure(data=stressdata)
stressplot = py.plot(stressfig, filename='stress by tank')
stresbytankU, stressbytankP = stats.ttest_ind(righttank.propTimeEdge, lefttank.propTimeEdge)
print 'stressbytankT',stressbytankT
print 'stressbytankP', stressbytankP
###stress levels are similar in both tanks

hightrainers = data.loc[data.condition=='High',['propTimeEdge']]
lowtrainers = data.loc[data.condition=='Low',['propTimeEdge']]
tr222 = go.Box(y=hightrainers.propTimeEdge,name='hightrainerstress')
tr2222 = go.Box(y=lowtrainers.propTimeEdge, name='lowtrainerstress')
stressdatahighlow = [tr222,tr2222]
stressfighl = go.Figure(data=stressdatahighlow)
stressplothl = py.plot(stressfighl, filename='stress by trained condition')
stressbytrainerT, stressbytrainerP = stats.ttest_ind(hightrainers.propTimeEdge, lowtrainers.propTimeEdge)
print 'stressbytrainerT',stressbytrainerT
print 'stressbytrainerP', stressbytrainerP
###stress levels are similar across training groups
##############################
#
#
##############################
# performance by ratio on testing days
'''
# individual trials
testing_days = data.loc[data["day"]>=5, ["fishName", "propTimeStim", "day", "ratio"]]

testdays5 = testing_days.loc[testing_days['ratio']==0.5, ["fishName", "propTimeStim"]]
testdays6 = testing_days.loc[testing_days['ratio']==0.67, ["fishName", "propTimeStim"]]
testdays7 = testing_days.loc[testing_days['ratio']==0.75, ["fishName", "propTimeStim"]]

x1 = testdays5.propTimeStim
x2 = testdays7.propTimeStim

trace5 = go.Scatter(y=testdays6.propTimeStim,x=x1,mode='markers',name='0.67v0.50',text=testdays6.index)
trace6 = go.Scatter(y=testdays7.propTimeStim,x=x1,mode='markers',name='0.75v0.50',text=testdays7.index)
trace7 = go.Scatter(y=testdays6.propTimeStim,x=x2,mode='markers',name='0.67v0.75',text=testdays6.index)

datuma = [trace5, trace6, trace7]
fig2 = go.Figure(data=datuma)
plot_url1 = py.plot(fig2, filename='cross-ratio performance (inc. reinf.)')

# averages
testdays5avgs = testdays5.groupby('fishName').propTimeStim.mean()
testdays6avgs = testdays6.groupby('fishName').propTimeStim.mean()
testdays7avgs = testdays7.groupby('fishName').propTimeStim.mean()

traceA = go.Scatter(y=testdays6avgs,x=testdays5avgs,mode='markers',name='0.67v0.50',text=testdays6avgs.index)
traceB = go.Scatter(y=testdays7avgs,x=testdays5avgs,mode='markers',name='0.75v0.50',text=testdays7avgs.index)
traceC = go.Scatter(y=testdays6avgs,x=testdays7avgs,mode='markers',name='0.67v0.75',text=testdays6avgs.index)

datumavgs = [traceA, traceB, traceC]
figavg = go.Figure(data=datumavgs)
plot_url2 = py.plot(figavg, filename='cross-ratio performance averages ALL (inc. reinf.)')
testdayavg6v7R, testdayavg6v7P = stats.pearsonr(testdays6avgs,testdays7avgs)
print 'testdayavg6v7R',testdayavg6v7R
print'testdayavg6v7P', testdayavg6v7P
###we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75
'''
#excluding reinfocement
testing_daysexc = data.loc[(data["day"]>=5)&(data["session"]<=3), ["fishName", "propTimeStim", "day", "session", "ratio"]]

testdays5exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.5, ["fishName", "propTimeStim", "condition"]]
testdays6exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.67, ["fishName", "propTimeStim", "condition"]]
testdays7exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.75, ["fishName", "propTimeStim", "condition"]]

x1exc = testdays5exc.propTimeStim
x2exc = testdays7exc.propTimeStim

trace5exc = go.Scatter(y=testdays6exc.propTimeStim,x=x1exc,mode='markers',name='0.67v0.50',text=testdays6exc.index)
trace6exc = go.Scatter(y=testdays7exc.propTimeStim,x=x1exc,mode='markers',name='0.75v0.50',text=testdays7exc.index)
trace7exc = go.Scatter(y=testdays6exc.propTimeStim,x=x2exc,mode='markers',name='0.67v0.75',text=testdays6exc.index)

datumaexc = [trace5exc, trace6exc, trace7exc]
fig2exc = go.Figure(data=datumaexc)
plot_url3 = py.plot(fig2exc, filename='cross-ratio performance')

#exclusding reinforcenment averages
testdays5avgsexc = testdays5exc.groupby('fishName').propTimeStim.mean()
testdays6avgsexc = testdays6exc.groupby('fishName').propTimeStim.mean()
testdays7avgsexc = testdays7exc.groupby('fishName').propTimeStim.mean()

traceAexc = go.Scatter(y=testdays6avgsexc,x=testdays5avgsexc,mode='markers',name='0.67v0.50',text=testdays6avgsexc.index)
traceBexc = go.Scatter(y=testdays7avgsexc,x=testdays5avgsexc,mode='markers',name='0.75v0.50',text=testdays7avgsexc.index)
traceCexc = go.Scatter(y=testdays6avgsexc,x=testdays7avgsexc,mode='markers',name='0.67v0.75',text=testdays6avgsexc.index)

datumavgsexc = [traceAexc, traceBexc, traceCexc]
figavgexc = go.Figure(data=datumavgsexc)
plot_url4 = py.plot(figavgexc, filename='cross-ratio performance averages')
testdayavg6v7excR, testdayavg6v7excP = stats.pearsonr(testdays6avgsexc,testdays7avgsexc)
print 'testdayavg6v7excR',testdayavg6v7excR
print'testdayavg6v7excP', testdayavg6v7excP
###excluding reinforcement: we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75
################################
#

#
################################

'''
testing_days = data.loc[(data["day"]>=5)&(data["tank"]=='R'), ["fishName", "propTimeStim", "day", "ratio"]]

testdays5 = testing_days.loc[testing_days['ratio']==0.5, ["fishName", "propTimeStim"]]
testdays6 = testing_days.loc[testing_days['ratio']==0.67, ["fishName", "propTimeStim"]]
testdays7 = testing_days.loc[testing_days['ratio']==0.75, ["fishName", "propTimeStim"]]

x1 = testdays5.propTimeStim
x2 = testdays7.propTimeStim

trace5 = go.Scatter(y=testdays6.propTimeStim,x=x1,mode='markers',name='0.67v0.50')
trace6 = go.Scatter(y=testdays7.propTimeStim,x=x1,mode='markers',name='0.75v0.50')
trace7 = go.Scatter(y=testdays6.propTimeStim,x=x2,mode='markers',name='0.67v0.75')

datuma = [trace5, trace6, trace7]
fig2 = go.Figure(data=datuma)
plot_url1 = py.plot(fig2, filename='individual testing days across ratios')

# averages
testdays5avgs = testdays5.groupby('fishName').propTimeStim.mean()
testdays6avgs = testdays6.groupby('fishName').propTimeStim.mean()
testdays7avgs = testdays7.groupby('fishName').propTimeStim.mean()

traceA = go.Scatter(y=testdays6avgs,x=testdays5avgs,mode='markers',name='0.67v0.50')
traceB = go.Scatter(y=testdays7avgs,x=testdays5avgs,mode='markers',name='0.75v0.50')
traceC = go.Scatter(y=testdays6avgs,x=testdays7avgs,mode='markers',name='0.67v0.75')

datumavgs = [traceA, traceB, traceC]
figavg = go.Figure(data=datumavgs)
plot_url2 = py.plot(figavg, filename='average performancea across ratios')
testdayavg6v7R, testdayavg6v7P = stats.pearsonr(testdays6avgs,testdays7avgs)
print 'testdayavg6v7R',testdayavg6v7R
print'testdayavg6v7P', testdayavg6v7P
# we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75

'''

'''

#define subsets
sexgroup = data.groupby('sex')
speciesgroup = data.groupby('species')
daygroup = data.groupby('day')

learners = data[(data.day>4) & (data.propTimeStim>0.60)]
nonlearners = data[(data.day>4) & (data.propTimeStim<0.60)]

learnersbyday = learners.groupby('day')
nonlearnersbyday = nonlearners.groupby('day')

learnersbysex = learners.groupby('sex')
nonlearnersbysex = nonlearners.groupby('sex')

learnersbyspecies = learners.groupby('species')
nonlearnersbyspecies = nonlearners.groupby('species')

'''

'''

'species', 'sex', 'round', 'day',
                                                                                'session',
                                                                                'standardLength', 'fishID',
                                                                                'fishName', 'timeEdge',
                                                                                'propTimeEdge',
                                                                                'propTimeStim', 'propActivityStim',
                                                                                'stimulus', 'stimSide', 'timeStim',
                                                                                'activityStim',
                                                                                'activityTotal', 'activityLeft',
                                                                                'activityRight', 'activityMiddle',
                                                                                'propActivityLeft',
                                                                                'propActivityRight',
                                                                                'propActivityMiddle', 'timeLeft',
                                                                                'timeRight', 'timeMiddle',
                                                                                'propTimeLeft', 'propTimeRight',
                                                                                'propTimeMiddle', 'survivalMetric']

'''
<commit_msg>testing atom git control<commit_after>__author__ = 'ian'
import pandas as pd
import numpy as np
from scipy import stats
import sys
import matplotlib.pyplot as plt
import plotly.plotly as py
import plotly.graph_objs as go

#import data
data = pd.read_csv('masterdataNumerosity_sailfins.csv')
#add high/low condition
data['condition']=np.where(data['fishName'].str[:1]=='H','High','Low')
#
#
#
##############################
# side bias
tankRight = data.loc[data.tank=='R', ['propTimeRight', 'propTimeLeft']]
tankLeft = data.loc[data.tank=='L', ['propTimeRight', 'propTimeLeft']]
trace3 = go.Box(y=tankRight.propTimeRight,name='Right Tank Right Side')
trace2 = go.Box(y=tankRight.propTimeLeft,name='Right Tank Left Side')
trace1 = go.Box(y=tankLeft.propTimeRight,name='Left Tank Right Side')
trace0 = go.Box(y=tankLeft.propTimeLeft,name='Left Tank Left Side')
datum = [trace0, trace1, trace2, trace3]
sidebiasanovaF, sidebiasanovaP = stats.f_oneway(tankLeft.propTimeLeft, tankLeft.propTimeRight, tankRight.propTimeLeft, tankRight.propTimeRight)
sidebiasLeftLeftvLeftRightT, sidebiasLeftLeftvLeftRightP = stats.ttest_rel(tankLeft.propTimeLeft,tankLeft.propTimeRight)
sidebiasLeftLeftvRightRightT, sidebiasLeftLeftvRightRightP = stats.ttest_ind(tankLeft.propTimeLeft,tankRight.propTimeRight)

fig1 = go.Figure(data=datum)
plot_url = py.plot(fig1, filename='tank side bias')
print 'sidebiasanovaF: ', sidebiasanovaF
print 'sidebiasanovaP: ', sidebiasanovaP
print 'sidebiasLeftLeftvLeftRightT', sidebiasLeftLeftvLeftRightT
print 'sidebiasLeftLeftvLeftRightP', sidebiasLeftLeftvLeftRightP
###we see a bias in the left tank towards the right side/away from left side
### is this side bias correlated with thigmotaxis (our measure of stress)?
##############################
#
#
##############################
#stress levels (thigmotaxis)
righttank = data.loc[data.tank=='R',['propTimeEdge']]
lefttank = data.loc[data.tank=='L',['propTimeEdge']]
tr = go.Box(y=righttank.propTimeEdge,name='righttankstress')
tr2 = go.Box(y=lefttank.propTimeEdge, name='lefttankstress')
stressdata = [tr,tr2]
stressfig = go.Figure(data=stressdata)
stressplot = py.plot(stressfig, filename='stress by tank')
stresbytankU, stressbytankP = stats.ttest_ind(righttank.propTimeEdge, lefttank.propTimeEdge)
print 'stressbytankT',stressbytankT
print 'stressbytankP', stressbytankP
###stress levels are similar in both tanks
hightrainers = data.loc[data.condition=='High',['propTimeEdge']]
lowtrainers = data.loc[data.condition=='Low',['propTimeEdge']]
tr222 = go.Box(y=hightrainers.propTimeEdge,name='hightrainerstress')
tr2222 = go.Box(y=lowtrainers.propTimeEdge, name='lowtrainerstress')
stressdatahighlow = [tr222,tr2222]
stressfighl = go.Figure(data=stressdatahighlow)
stressplothl = py.plot(stressfighl, filename='stress by trained condition')
stressbytrainerT, stressbytrainerP = stats.ttest_ind(hightrainers.propTimeEdge, lowtrainers.propTimeEdge)
print 'stressbytrainerT',stressbytrainerT
print 'stressbytrainerP', stressbytrainerP
###stress levels are similar across training groups
##############################
#
#
##############################
# performance by ratio on testing days
'''
# individual trials
testing_days = data.loc[data["day"]>=5, ["fishName", "propTimeStim", "day", "ratio"]]

testdays5 = testing_days.loc[testing_days['ratio']==0.5, ["fishName", "propTimeStim"]]
testdays6 = testing_days.loc[testing_days['ratio']==0.67, ["fishName", "propTimeStim"]]
testdays7 = testing_days.loc[testing_days['ratio']==0.75, ["fishName", "propTimeStim"]]

x1 = testdays5.propTimeStim
x2 = testdays7.propTimeStim

trace5 = go.Scatter(y=testdays6.propTimeStim,x=x1,mode='markers',name='0.67v0.50',text=testdays6.index)
trace6 = go.Scatter(y=testdays7.propTimeStim,x=x1,mode='markers',name='0.75v0.50',text=testdays7.index)
trace7 = go.Scatter(y=testdays6.propTimeStim,x=x2,mode='markers',name='0.67v0.75',text=testdays6.index)

datuma = [trace5, trace6, trace7]
fig2 = go.Figure(data=datuma)
plot_url1 = py.plot(fig2, filename='cross-ratio performance (inc. reinf.)')

# averages
testdays5avgs = testdays5.groupby('fishName').propTimeStim.mean()
testdays6avgs = testdays6.groupby('fishName').propTimeStim.mean()
testdays7avgs = testdays7.groupby('fishName').propTimeStim.mean()

traceA = go.Scatter(y=testdays6avgs,x=testdays5avgs,mode='markers',name='0.67v0.50',text=testdays6avgs.index)
traceB = go.Scatter(y=testdays7avgs,x=testdays5avgs,mode='markers',name='0.75v0.50',text=testdays7avgs.index)
traceC = go.Scatter(y=testdays6avgs,x=testdays7avgs,mode='markers',name='0.67v0.75',text=testdays6avgs.index)

datumavgs = [traceA, traceB, traceC]
figavg = go.Figure(data=datumavgs)
plot_url2 = py.plot(figavg, filename='cross-ratio performance averages ALL (inc. reinf.)')
testdayavg6v7R, testdayavg6v7P = stats.pearsonr(testdays6avgs,testdays7avgs)
print 'testdayavg6v7R',testdayavg6v7R
print'testdayavg6v7P', testdayavg6v7P
###we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75
'''
##excluding reinfocement
testing_daysexc = data.loc[(data["day"]>=5)&(data["session"]<=3), ["fishName", "propTimeStim", "day", "session", "ratio"]]
testdays5exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.5, ["fishName", "propTimeStim"]]
testdays6exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.67, ["fishName", "propTimeStim"]]
testdays7exc = testing_daysexc.loc[testing_daysexc["ratio"]==0.75, ["fishName", "propTimeStim"]]
x1exc = testdays5exc.propTimeStim
x2exc = testdays7exc.propTimeStim
trace5exc = go.Scatter(y=testdays6exc.propTimeStim,x=x1exc,mode='markers',name='0.67v0.50',text=testdays6exc.index)
trace6exc = go.Scatter(y=testdays7exc.propTimeStim,x=x1exc,mode='markers',name='0.75v0.50',text=testdays7exc.index)
trace7exc = go.Scatter(y=testdays6exc.propTimeStim,x=x2exc,mode='markers',name='0.67v0.75',text=testdays6exc.index)
datumaexc = [trace5exc, trace6exc, trace7exc]
fig2exc = go.Figure(data=datumaexc)
plot_url3 = py.plot(fig2exc, filename='cross-ratio performance')
##exclusding reinforcenment averages
testdays5avgsexc = testdays5exc.groupby('fishName').propTimeStim.mean()
testdays6avgsexc = testdays6exc.groupby('fishName').propTimeStim.mean()
testdays7avgsexc = testdays7exc.groupby('fishName').propTimeStim.mean()
traceAexc = go.Scatter(y=testdays6avgsexc,x=testdays5avgsexc,mode='markers',name='0.67v0.50',text=testdays6avgsexc.index)
traceBexc = go.Scatter(y=testdays7avgsexc,x=testdays5avgsexc,mode='markers',name='0.75v0.50',text=testdays7avgsexc.index)
traceCexc = go.Scatter(y=testdays6avgsexc,x=testdays7avgsexc,mode='markers',name='0.67v0.75',text=testdays6avgsexc.index)
datumavgsexc = [traceAexc, traceBexc, traceCexc]
figavgexc = go.Figure(data=datumavgsexc)
plot_url4 = py.plot(figavgexc, filename='cross-ratio performance averages')
testdayavg6v7excR, testdayavg6v7excP = stats.pearsonr(testdays6avgsexc,testdays7avgsexc)
print 'testdayavg6v7excR',testdayavg6v7excR
print'testdayavg6v7excP', testdayavg6v7excP
##ratio performance by trained condition
testing_dayscond = data.loc[(data["day"]>=5)&(data["session"]<=3), ["fishName", "propTimeStim", "day", "session", "ratio", "condition"]]
testdays5cond = testing_daysexc.loc[testing_daysexc["ratio"]==0.5, ["fishName", "propTimeStim", "condition"]]
testdays6cond = testing_daysexc.loc[testing_daysexc["ratio"]==0.67, ["fishName", "propTimeStim", "condition"]]
testdays7cond = testing_daysexc.loc[testing_daysexc["ratio"]==0.75, ["fishName", "propTimeStim", "condition"]]

###excluding reinforcement: we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75
################################
#

#
################################

'''
testing_days = data.loc[(data["day"]>=5)&(data["tank"]=='R'), ["fishName", "propTimeStim", "day", "ratio"]]

testdays5 = testing_days.loc[testing_days['ratio']==0.5, ["fishName", "propTimeStim"]]
testdays6 = testing_days.loc[testing_days['ratio']==0.67, ["fishName", "propTimeStim"]]
testdays7 = testing_days.loc[testing_days['ratio']==0.75, ["fishName", "propTimeStim"]]

x1 = testdays5.propTimeStim
x2 = testdays7.propTimeStim

trace5 = go.Scatter(y=testdays6.propTimeStim,x=x1,mode='markers',name='0.67v0.50')
trace6 = go.Scatter(y=testdays7.propTimeStim,x=x1,mode='markers',name='0.75v0.50')
trace7 = go.Scatter(y=testdays6.propTimeStim,x=x2,mode='markers',name='0.67v0.75')

datuma = [trace5, trace6, trace7]
fig2 = go.Figure(data=datuma)
plot_url1 = py.plot(fig2, filename='individual testing days across ratios')

# averages
testdays5avgs = testdays5.groupby('fishName').propTimeStim.mean()
testdays6avgs = testdays6.groupby('fishName').propTimeStim.mean()
testdays7avgs = testdays7.groupby('fishName').propTimeStim.mean()

traceA = go.Scatter(y=testdays6avgs,x=testdays5avgs,mode='markers',name='0.67v0.50')
traceB = go.Scatter(y=testdays7avgs,x=testdays5avgs,mode='markers',name='0.75v0.50')
traceC = go.Scatter(y=testdays6avgs,x=testdays7avgs,mode='markers',name='0.67v0.75')

datumavgs = [traceA, traceB, traceC]
figavg = go.Figure(data=datumavgs)
plot_url2 = py.plot(figavg, filename='average performancea across ratios')
testdayavg6v7R, testdayavg6v7P = stats.pearsonr(testdays6avgs,testdays7avgs)
print 'testdayavg6v7R',testdayavg6v7R
print'testdayavg6v7P', testdayavg6v7P
# we see no significant relationship with individual tests but on averaged scores
### inividuals that perform well on the 0.67 ratio also perform well on 0.75

'''

'''

#define subsets
sexgroup = data.groupby('sex')
speciesgroup = data.groupby('species')
daygroup = data.groupby('day')

learners = data[(data.day>4) & (data.propTimeStim>0.60)]
nonlearners = data[(data.day>4) & (data.propTimeStim<0.60)]

learnersbyday = learners.groupby('day')
nonlearnersbyday = nonlearners.groupby('day')

learnersbysex = learners.groupby('sex')
nonlearnersbysex = nonlearners.groupby('sex')

learnersbyspecies = learners.groupby('species')
nonlearnersbyspecies = nonlearners.groupby('species')

'''

'''

'species', 'sex', 'round', 'day',
                                                                                'session',
                                                                                'standardLength', 'fishID',
                                                                                'fishName', 'timeEdge',
                                                                                'propTimeEdge',
                                                                                'propTimeStim', 'propActivityStim',
                                                                                'stimulus', 'stimSide', 'timeStim',
                                                                                'activityStim',
                                                                                'activityTotal', 'activityLeft',
                                                                                'activityRight', 'activityMiddle',
                                                                                'propActivityLeft',
                                                                                'propActivityRight',
                                                                                'propActivityMiddle', 'timeLeft',
                                                                                'timeRight', 'timeMiddle',
                                                                                'propTimeLeft', 'propTimeRight',
                                                                                'propTimeMiddle', 'survivalMetric']

'''
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2021 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import os
import re
import sys
import json
import time
import signal
import getpass
import logging
import itertools
import platform
from os.path import getsize
import psutil
import numpy
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.python3compat import decode
from openquake.baselib import parallel, general, config, __version__
from openquake.hazardlib import valid
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
OQ_DISTRIBUTE = parallel.oq_distribute()

MB = 1024 ** 2
_PID = os.getpid()  # the PID
_PPID = os.getppid()  # the controlling terminal PID

GET_JOBS = '''--- executing or submitted
SELECT * FROM job WHERE status IN ('executing', 'submitted')
AND is_running=1 AND pid > 0 ORDER BY id'''

def get_zmq_ports():
    """
    :returns: an array with the receiver ports
    """
    start, stop = config.dbserver.receiver_ports.split('-')
    return numpy.arange(int(start), int(stop))

def set_concurrent_tasks_default(calc):
    """
    Look at the number of available workers and update the parameter
    OqParam.concurrent_tasks.default. Abort the calculations if no
    workers are available. Do nothing for trivial distributions.
    """
    if OQ_DISTRIBUTE in 'no processpool':  # do nothing
        num_workers = 0 if OQ_DISTRIBUTE == 'no' else parallel.Starmap.CT // 2
        logging.warning('Using %d cores on %s', num_workers, platform.node())
        return

    num_workers = sum(total for host, running, total
                      in parallel.workers_wait())
    if num_workers == 0:
        logging.critical("No live compute nodes, aborting calculation")
        logs.dbcmd('finish', calc.datastore.calc_id, 'failed')
        sys.exit(1)

    parallel.Starmap.CT = num_workers * 2
    parallel.Starmap.num_cores = num_workers
    OqParam.concurrent_tasks.default = num_workers * 2
    logging.warning('Using %d %s workers', num_workers, OQ_DISTRIBUTE)

def expose_outputs(dstore, owner=getpass.getuser(), status='complete'):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    if 'avg_gmf' in dskeys:
        dskeys.remove('avg_gmf')  # hide
    rlzs = dstore['full_lt'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    hdf5 = dstore.hdf5
    if 'hcurves-stats' in hdf5 or 'hcurves-rlzs' in hdf5:
        if oq.hazard_stats() or oq.individual_curves or len(rlzs) == 1:
            dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if len(rlzs) > 1 and not oq.individual_curves and not oq.collect_rlzs:
        for out in ['avg_losses-rlzs', 'agg_losses-rlzs', 'agg_curves-rlzs']:
            if out in dskeys:
                dskeys.remove(out)
    if 'curves-rlzs' in dstore and len(rlzs) == 1:
        dskeys.add('loss_curves-rlzs')
    if 'curves-stats' in dstore and len(rlzs) > 1:
        dskeys.add('loss_curves-stats')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'hmaps' in dskeys and not oq.hazard_maps:
        dskeys.remove('hmaps')  # do not export the hazard maps
    if logs.dbcmd('get_job', dstore.calc_id) is None:
        # the calculation has not been imported in the db yet
        logs.dbcmd('import_job', dstore.calc_id, oq.calculation_mode,
                   oq.description + ' [parent]', owner, status,
                   oq.hazard_calculation_id, dstore.datadir)
    keysize = []
    for key in sorted(dskeys & exportable):
        try:
            size_mb = dstore.getsize(key) / MB
        except (KeyError, AttributeError):
            size_mb = -1
        if size_mb:
            keysize.append((key, size_mb))
    ds_size = dstore.getsize() / MB
    logs.dbcmd('create_outputs', dstore.calc_id, keysize, ds_size)

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def inhibitSigInt(signum, _stack):
    logging.warning('Killing job, please wait')

def manage_signals(signum, _stack):
    """
    Convert a SIGTERM into a SystemExit exception and a SIGINT/SIGHUP into
    a MasterKilled exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    # Disable further CTRL-C to allow tasks revocation when Celery is used
    if OQ_DISTRIBUTE == 'celery':
        signal.signal(signal.SIGINT, inhibitSigInt)

    if signum == signal.SIGINT:
        parallel.workers_stop()
        raise MasterKilled('The openquake master process was killed manually')

    if signum == signal.SIGTERM:
        parallel.workers_stop()
        raise SystemExit('Terminated')

    if hasattr(signal, 'SIGHUP'):  # there is no SIGHUP on Windows
        # kill the calculation only if os.getppid() != _PPID, i.e. the
        # controlling terminal died; in the workers, do nothing
        if signum == signal.SIGHUP and os.getppid() != _PPID:
            parallel.workers_stop()
            raise MasterKilled(
                'The openquake master lost its controlling terminal')

def register_signals():
    # register the manage_signals callback for SIGTERM, SIGINT, SIGHUP
    # when using the Django development server this module is imported by a
    # thread, so one gets a `ValueError: signal only works in main thread` that
    # can be safely ignored
    try:
        signal.signal(signal.SIGTERM, manage_signals)
        signal.signal(signal.SIGINT, manage_signals)
        if hasattr(signal, 'SIGHUP'):
            # Do not register our SIGHUP handler if running with 'nohup'
            if signal.getsignal(signal.SIGHUP) != signal.SIG_IGN:
                signal.signal(signal.SIGHUP, manage_signals)
    except ValueError:
        pass

def poll_queue(job_id, poll_time):
    """
    Check the queue of executing/submitted jobs and exit when there is
    a free slot.
    """
    offset = config.distribution.serialize_jobs - 1
    if offset >= 0:
        first_time = True
        while True:
            jobs = logs.dbcmd(GET_JOBS)
            failed = [job.id for job in jobs if not psutil.pid_exists(job.pid)]
            if failed:
                for job in failed:
                    logs.dbcmd('update_job', job,
                               {'status': 'failed', 'is_running': 0})
            elif any(j.id < job_id - offset for j in jobs):
                if first_time:
                    print('Waiting for jobs %s' % [j.id for j in jobs
                                                   if j.id < job_id - offset])
                    logs.dbcmd('update_job', job_id,
                               {'status': 'submitted', 'pid': _PID})
                    first_time = False
                time.sleep(poll_time)
            else:
                break

def run_calc(log):
    """
    Run a calculation.

    :param log:
        LogContext of the current job
    """
    register_signals()
    setproctitle('oq-job-%d' % log.calc_id)
    with log:
        oqparam = log.get_oqparam()
        calc = base.calculators(oqparam, log.calc_id)
        logging.info('%s running %s [--hc=%s]',
                     getpass.getuser(),
                     calc.oqparam.inputs['job_ini'],
                     calc.oqparam.hazard_calculation_id)
        logging.info('Using engine version %s', __version__)
        msg = check_obsolete_version(oqparam.calculation_mode)
        if msg:
            logging.warning(msg)
        calc.from_engine = True
        if config.zworkers['host_cores']:
            set_concurrent_tasks_default(calc)
        else:
            logging.warning('Assuming %d %s workers',
                            parallel.Starmap.num_cores, OQ_DISTRIBUTE)
        t0 = time.time()
        calc.run()
        logging.info('Exposing the outputs to the database')
        expose_outputs(calc.datastore)
        path = calc.datastore.filename
        size = general.humansize(getsize(path))
        logging.info('Stored %s on %s in %d seconds',
                     size, path, time.time() - t0)
        calc.datastore.close()
        for line in logs.dbcmd('list_outputs', log.calc_id, False):
            general.safeprint(line)
        # sanity check to make sure that the logging on file is working
        if (log.log_file and log.log_file != os.devnull and
                getsize(log.log_file) == 0):
            logging.warning('The log file %s is empty!?' % log.log_file)
    return calc

def create_jobs(job_inis, log_level=logging.INFO, log_file=None,
                user_name=None, hc_id=None, multi=False):
    """
    Create job records on the database.

    :param job_inis: a list of pathnames or a list of dictionaries
    :returns: a list of LogContext objects
    """
    if len(job_inis) > 1 and not hc_id and not multi:  # first job as hc
        job = logs.init("job", job_inis[0], log_level, log_file,
                        user_name, hc_id)
        hc_id = job.calc_id
        jobs = [job]
        job_inis = job_inis[1:]
    else:
        jobs = []
    for job_ini in job_inis:
        if isinstance(job_ini, dict):
            dic = job_ini
        else:
            # NB: `get_params` must NOT log, since the logging is not
            # configured yet, otherwise the log will disappear :-(
            dic = readinput.get_params(job_ini)
        dic['hazard_calculation_id'] = hc_id
        if 'sensitivity_analysis' in dic:
            analysis = valid.dictionary(dic['sensitivity_analysis'])
            for values in itertools.product(*analysis.values()):
                new = logs.init('job', dic.copy(), log_level, None,
                                user_name, hc_id)
                pars = dict(zip(analysis, values))
                for param, value in pars.items():
                    new.params[param] = str(value)
                new.params['description'] = '%s %s' % (
                    dic['description'], pars)
                jobs.append(new)
        else:
            jobs.append(
                logs.init('job', dic, log_level, None, user_name, hc_id))
    if multi:
        for job in jobs:
            job.multi = True
    return jobs

def run_jobs(jobs):
    """
    Run jobs using the specified config file and other options.

    :param jobs:
        List of LogContexts
    """
    jobarray = len(jobs) > 1 and jobs[0].multi
    try:
        poll_queue(jobs[0].calc_id, poll_time=15)
        # wait for an empty slot or a CTRL-C
    except BaseException:
        # the job aborted even before starting
        for job in jobs:
            logs.dbcmd('finish', job.calc_id, 'aborted')
        return jobs
    else:
        for job in jobs:
            dic = {'status': 'executing', 'pid': _PID}
            logs.dbcmd('update_job', job.calc_id, dic)
    try:
        if config.zworkers['host_cores'] and parallel.workers_status() == []:
            print('Asking the DbServer to start the workers')
            logs.dbcmd('workers_start')  # start the workers
        allargs = [(job,) for job in jobs]
        if jobarray:
            with general.start_many(run_calc, allargs):
                pass
        else:
            for job in jobs:
                run_calc(job)
    finally:
        if config.zworkers['host_cores']:
            print('Stopping the workers')
            parallel.workers_stop()
    return jobs

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except KeyError:  # 'tag_name' not found
        # NOTE: for unauthenticated requests, the rate limit allows for up
        # to 60 requests per hour. Therefore, sometimes the api returns the
        # following message:
        # b'{"message":"API rate limit exceeded for aaa.aaa.aaa.aaa. (But'
        # ' here\'s the good news: Authenticated requests get a higher rate'
        # ' limit. Check out the documentation for more details.)",'
        # ' "documentation_url":'
        # ' "https://developer.github.com/v3/#rate-limiting"}'
        msg = ('An error occurred while calling %s/engine/latest to check'
               ' if the installed version of the engine is up to date.\n'
               '%s' % (OQ_API, data.decode('utf8')))
        logging.warning(msg)
        return
    except Exception:  # page not available or wrong version tag
        msg = ('An error occurred while calling %s/engine/latest to check'
               ' if the installed version of the engine is up to date.' %
               OQ_API)
        logging.warning(msg, exc_info=True)
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<commit_msg>Disabled version check<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2021 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import os
import re
import sys
import json
import time
import signal
import getpass
import logging
import itertools
import platform
from os.path import getsize
import psutil
import numpy
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.python3compat import decode
from openquake.baselib import parallel, general, config, __version__
from openquake.hazardlib import valid
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
OQ_DISTRIBUTE = parallel.oq_distribute()

MB = 1024 ** 2
_PID = os.getpid()  # the PID
_PPID = os.getppid()  # the controlling terminal PID

GET_JOBS = '''--- executing or submitted
SELECT * FROM job WHERE status IN ('executing', 'submitted')
AND is_running=1 AND pid > 0 ORDER BY id'''

def get_zmq_ports():
    """
    :returns: an array with the receiver ports
    """
    start, stop = config.dbserver.receiver_ports.split('-')
    return numpy.arange(int(start), int(stop))

def set_concurrent_tasks_default(calc):
    """
    Look at the number of available workers and update the parameter
    OqParam.concurrent_tasks.default. Abort the calculations if no
    workers are available. Do nothing for trivial distributions.
    """
    if OQ_DISTRIBUTE in 'no processpool':  # do nothing
        num_workers = 0 if OQ_DISTRIBUTE == 'no' else parallel.Starmap.CT // 2
        logging.warning('Using %d cores on %s', num_workers, platform.node())
        return

    num_workers = sum(total for host, running, total
                      in parallel.workers_wait())
    if num_workers == 0:
        logging.critical("No live compute nodes, aborting calculation")
        logs.dbcmd('finish', calc.datastore.calc_id, 'failed')
        sys.exit(1)

    parallel.Starmap.CT = num_workers * 2
    parallel.Starmap.num_cores = num_workers
    OqParam.concurrent_tasks.default = num_workers * 2
    logging.warning('Using %d %s workers', num_workers, OQ_DISTRIBUTE)

def expose_outputs(dstore, owner=getpass.getuser(), status='complete'):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    if 'avg_gmf' in dskeys:
        dskeys.remove('avg_gmf')  # hide
    rlzs = dstore['full_lt'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    hdf5 = dstore.hdf5
    if 'hcurves-stats' in hdf5 or 'hcurves-rlzs' in hdf5:
        if oq.hazard_stats() or oq.individual_curves or len(rlzs) == 1:
            dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if len(rlzs) > 1 and not oq.individual_curves and not oq.collect_rlzs:
        for out in ['avg_losses-rlzs', 'agg_losses-rlzs', 'agg_curves-rlzs']:
            if out in dskeys:
                dskeys.remove(out)
    if 'curves-rlzs' in dstore and len(rlzs) == 1:
        dskeys.add('loss_curves-rlzs')
    if 'curves-stats' in dstore and len(rlzs) > 1:
        dskeys.add('loss_curves-stats')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'hmaps' in dskeys and not oq.hazard_maps:
        dskeys.remove('hmaps')  # do not export the hazard maps
    if logs.dbcmd('get_job', dstore.calc_id) is None:
        # the calculation has not been imported in the db yet
        logs.dbcmd('import_job', dstore.calc_id, oq.calculation_mode,
                   oq.description + ' [parent]', owner, status,
                   oq.hazard_calculation_id, dstore.datadir)
    keysize = []
    for key in sorted(dskeys & exportable):
        try:
            size_mb = dstore.getsize(key) / MB
        except (KeyError, AttributeError):
            size_mb = -1
        if size_mb:
            keysize.append((key, size_mb))
    ds_size = dstore.getsize() / MB
    logs.dbcmd('create_outputs', dstore.calc_id, keysize, ds_size)

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def inhibitSigInt(signum, _stack):
    logging.warning('Killing job, please wait')

def manage_signals(signum, _stack):
    """
    Convert a SIGTERM into a SystemExit exception and a SIGINT/SIGHUP into
    a MasterKilled exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    # Disable further CTRL-C to allow tasks revocation when Celery is used
    if OQ_DISTRIBUTE == 'celery':
        signal.signal(signal.SIGINT, inhibitSigInt)

    if signum == signal.SIGINT:
        parallel.workers_stop()
        raise MasterKilled('The openquake master process was killed manually')

    if signum == signal.SIGTERM:
        parallel.workers_stop()
        raise SystemExit('Terminated')

    if hasattr(signal, 'SIGHUP'):  # there is no SIGHUP on Windows
        # kill the calculation only if os.getppid() != _PPID, i.e. the
        # controlling terminal died; in the workers, do nothing
        if signum == signal.SIGHUP and os.getppid() != _PPID:
            parallel.workers_stop()
            raise MasterKilled(
                'The openquake master lost its controlling terminal')

def register_signals():
    # register the manage_signals callback for SIGTERM, SIGINT, SIGHUP
    # when using the Django development server this module is imported by a
    # thread, so one gets a `ValueError: signal only works in main thread` that
    # can be safely ignored
    try:
        signal.signal(signal.SIGTERM, manage_signals)
        signal.signal(signal.SIGINT, manage_signals)
        if hasattr(signal, 'SIGHUP'):
            # Do not register our SIGHUP handler if running with 'nohup'
            if signal.getsignal(signal.SIGHUP) != signal.SIG_IGN:
                signal.signal(signal.SIGHUP, manage_signals)
    except ValueError:
        pass

def poll_queue(job_id, poll_time):
    """
    Check the queue of executing/submitted jobs and exit when there is
    a free slot.
    """
    offset = config.distribution.serialize_jobs - 1
    if offset >= 0:
        first_time = True
        while True:
            jobs = logs.dbcmd(GET_JOBS)
            failed = [job.id for job in jobs if not psutil.pid_exists(job.pid)]
            if failed:
                for job in failed:
                    logs.dbcmd('update_job', job,
                               {'status': 'failed', 'is_running': 0})
            elif any(j.id < job_id - offset for j in jobs):
                if first_time:
                    print('Waiting for jobs %s' % [j.id for j in jobs
                                                   if j.id < job_id - offset])
                    logs.dbcmd('update_job', job_id,
                               {'status': 'submitted', 'pid': _PID})
                    first_time = False
                time.sleep(poll_time)
            else:
                break

def run_calc(log):
    """
    Run a calculation.

    :param log:
        LogContext of the current job
    """
    register_signals()
    setproctitle('oq-job-%d' % log.calc_id)
    with log:
        oqparam = log.get_oqparam()
        calc = base.calculators(oqparam, log.calc_id)
        logging.info('%s running %s [--hc=%s]',
                     getpass.getuser(),
                     calc.oqparam.inputs['job_ini'],
                     calc.oqparam.hazard_calculation_id)
        logging.info('Using engine version %s', __version__)
        msg = check_obsolete_version(oqparam.calculation_mode)
        # NB: disabling the warning should be done only for users with
        # an updated LTS version, but we are doing it for all users
        # if msg:
        #    logging.warning(msg)
        calc.from_engine = True
        if config.zworkers['host_cores']:
            set_concurrent_tasks_default(calc)
        else:
            logging.warning('Assuming %d %s workers',
                            parallel.Starmap.num_cores, OQ_DISTRIBUTE)
        t0 = time.time()
        calc.run()
        logging.info('Exposing the outputs to the database')
        expose_outputs(calc.datastore)
        path = calc.datastore.filename
        size = general.humansize(getsize(path))
        logging.info('Stored %s on %s in %d seconds',
                     size, path, time.time() - t0)
        calc.datastore.close()
        for line in logs.dbcmd('list_outputs', log.calc_id, False):
            general.safeprint(line)
        # sanity check to make sure that the logging on file is working
        if (log.log_file and log.log_file != os.devnull and
                getsize(log.log_file) == 0):
            logging.warning('The log file %s is empty!?' % log.log_file)
    return calc

def create_jobs(job_inis, log_level=logging.INFO, log_file=None,
                user_name=None, hc_id=None, multi=False):
    """
    Create job records on the database.

    :param job_inis: a list of pathnames or a list of dictionaries
    :returns: a list of LogContext objects
    """
    if len(job_inis) > 1 and not hc_id and not multi:  # first job as hc
        job = logs.init("job", job_inis[0], log_level, log_file,
                        user_name, hc_id)
        hc_id = job.calc_id
        jobs = [job]
        job_inis = job_inis[1:]
    else:
        jobs = []
    for job_ini in job_inis:
        if isinstance(job_ini, dict):
            dic = job_ini
        else:
            # NB: `get_params` must NOT log, since the logging is not
            # configured yet, otherwise the log will disappear :-(
            dic = readinput.get_params(job_ini)
        dic['hazard_calculation_id'] = hc_id
        if 'sensitivity_analysis' in dic:
            analysis = valid.dictionary(dic['sensitivity_analysis'])
            for values in itertools.product(*analysis.values()):
                new = logs.init('job', dic.copy(), log_level, None,
                                user_name, hc_id)
                pars = dict(zip(analysis, values))
                for param, value in pars.items():
                    new.params[param] = str(value)
                new.params['description'] = '%s %s' % (
                    dic['description'], pars)
                jobs.append(new)
        else:
            jobs.append(
                logs.init('job', dic, log_level, None, user_name, hc_id))
    if multi:
        for job in jobs:
            job.multi = True
    return jobs

def run_jobs(jobs):
    """
    Run jobs using the specified config file and other options.

    :param jobs:
        List of LogContexts
    """
    jobarray = len(jobs) > 1 and jobs[0].multi
    try:
        poll_queue(jobs[0].calc_id, poll_time=15)
        # wait for an empty slot or a CTRL-C
    except BaseException:
        # the job aborted even before starting
        for job in jobs:
            logs.dbcmd('finish', job.calc_id, 'aborted')
        return jobs
    else:
        for job in jobs:
            dic = {'status': 'executing', 'pid': _PID}
            logs.dbcmd('update_job', job.calc_id, dic)
    try:
        if config.zworkers['host_cores'] and parallel.workers_status() == []:
            print('Asking the DbServer to start the workers')
            logs.dbcmd('workers_start')  # start the workers
        allargs = [(job,) for job in jobs]
        if jobarray:
            with general.start_many(run_calc, allargs):
                pass
        else:
            for job in jobs:
                run_calc(job)
    finally:
        if config.zworkers['host_cores']:
            print('Stopping the workers')
            parallel.workers_stop()
    return jobs

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except Exception:  # page not available or wrong version tag
        msg = ('An error occurred while calling %s/engine/latest to check'
               ' if the installed version of the engine is up to date.' %
               OQ_API)
        logging.warning(msg, exc_info=True)
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<|endoftext|>