<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import os
import re
import sys
import json
import signal
import traceback
import platform
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.performance import Monitor
from openquake.baselib.python3compat import decode
from openquake.baselib import (
    parallel, general, config, datastore, __version__, zeromq as z)
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, views, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
TERMINATE = config.distribution.terminate_workers_on_revoke
OQ_DISTRIBUTE = parallel.oq_distribute()

_PPID = os.getppid()  # the controlling terminal PID

if OQ_DISTRIBUTE == 'zmq':

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the available
        worker pools .
        """
        num_workers = 0
        w = config.zworkers
        for host, _cores in [hc.split() for hc in w.host_cores.split(',')]:
            url = 'tcp://%s:%s' % (host, w.ctrl_port)
            with z.Socket(url, z.zmq.REQ, 'connect') as sock:
                if not general.socket_ready(url):
                    logs.LOG.warn('%s is not running', host)
                    continue
                num_workers += sock.send('get_num_workers')
        OqParam.concurrent_tasks.default = num_workers * 3
        logs.LOG.info('Using %d zmq workers', num_workers)

elif OQ_DISTRIBUTE.startswith('celery'):
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the number of available
        celery workers.
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
        OqParam.concurrent_tasks.default = num_cores * 3
        logs.LOG.info(
            'Using %s, %d cores', ', '.join(sorted(stats)), num_cores)

    def celery_cleanup(terminate, task_ids=()):
        """
        Release the resources used by an openquake job.
        In particular revoke the running tasks (if any).

        :param bool terminate: the celery revoke command terminate flag
        :param task_ids: celery task IDs
        """
        # Using the celery API, terminate and revoke and terminate any running
        # tasks associated with the current job.
        if task_ids:
            logs.LOG.warn('Revoking %d tasks', len(task_ids))
        else:  # this is normal when OQ_DISTRIBUTE=no
            logs.LOG.debug('No task to revoke')
        for tid in task_ids:
            celery.task.control.revoke(tid, terminate=terminate)
            logs.LOG.debug('Revoked task %s', tid)

def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    rlzs = dstore['csm_info'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    # expose gmf_data only if < 10 MB
    if oq.ground_motion_fields and calcmode == 'event_based':
        nbytes = dstore['gmf_data'].attrs['nbytes']
        if nbytes < 10 * 1024 ** 2:
            dskeys.add('gmf_data')
    if 'scenario' not in calcmode:  # export sourcegroups.csv
        dskeys.add('sourcegroups')
    hdf5 = dstore.hdf5
    if (len(rlzs) == 1 and 'poes' in hdf5) or 'hcurves' in hdf5:
        dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if 'avg_losses-stats' in dstore or (
            'avg_losses-rlzs' in dstore and len(rlzs)):
        dskeys.add('avg_losses-stats')
    if 'curves-stats' in dstore:
        logs.LOG.warn('loss curves are exportable with oq export')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'all_loss_ratios' in dskeys:
        dskeys.remove('all_loss_ratios')  # export only specific IDs
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'rup_loss_table' in dskeys:  # keep it hidden for the moment
        dskeys.remove('rup_loss_table')
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys & exportable))

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def raiseMasterKilled(signum, _stack):
    """
    When a SIGTERM is received, raise the MasterKilled
    exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    # kill the calculation only if os.getppid() != _PPID, i.e. the controlling
    # terminal died; in the workers, do nothing
    # NB: there is no SIGHUP on Windows
    if signum == signal.SIGHUP and os.getppid() == _PPID:
        return

    parallel.Starmap.shutdown()
    if signum in (signal.SIGTERM, signal.SIGINT):
        msg = 'The openquake master process was killed manually'
    elif signum == signal.SIGHUP:
        msg = 'The openquake master lost its controlling terminal'
    else:
        msg = 'Received a signal %d' % signum
    raise MasterKilled(msg)

# register the raiseMasterKilled callback for SIGTERM
# when using the Django development server this module is imported by a thread,
# so one gets a `ValueError: signal only works in main thread` that
# can be safely ignored
try:
    signal.signal(signal.SIGTERM, raiseMasterKilled)
    signal.signal(signal.SIGINT, raiseMasterKilled)
    signal.signal(signal.SIGHUP, raiseMasterKilled)
except ValueError:
    pass

def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file, hc_id=hazard_calculation_id)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.get_datadir(),
                        hazard_calculation_id)
    return job_id, oq

def run_calc(job_id, oqparam, log_level, log_file, exports,
             hazard_calculation_id=None, **kw):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    setproctitle('oq-job-%d' % job_id)
    monitor = Monitor('total runtime', measuremem=True)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if OQ_DISTRIBUTE.startswith(('celery', 'zmq')):
            set_concurrent_tasks_default()
        msg = check_obsolete_version(oqparam.calculation_mode)
        if msg:
            logs.LOG.warn(msg)
        calc = base.calculators(oqparam, monitor, calc_id=job_id)
        monitor.hdf5path = calc.datastore.hdf5path
        calc.from_engine = True
        tb = 'None\n'
        try:
            logs.dbcmd('set_status', job_id, 'executing')
            _do_run_calc(calc, exports, hazard_calculation_id, **kw)
            duration = monitor.duration
            expose_outputs(calc.datastore)
            monitor.flush()
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, duration)
            logs.dbcmd('finish', job_id, 'complete')
        except BaseException:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except BaseException:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if OQ_DISTRIBUTE.startswith('celery'):
                    celery_cleanup(TERMINATE, parallel.Starmap.task_ids)
            except BaseException:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc

def _do_run_calc(calc, exports, hazard_calculation_id, **kw):
    with calc._monitor:
        calc.run(exports=exports, hazard_calculation_id=hazard_calculation_id,
                 close=False, **kw)  # don't close the datastore too soon

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except:  # page not available or wrong version tag
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<commit_msg>Fix SIGHUP signal on Windows<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import os
import re
import sys
import json
import signal
import traceback
import platform
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        "Do nothing"
from urllib.request import urlopen, Request
from openquake.baselib.performance import Monitor
from openquake.baselib.python3compat import decode
from openquake.baselib import (
    parallel, general, config, datastore, __version__, zeromq as z)
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import readinput
from openquake.calculators import base, views, export
from openquake.commonlib import logs

OQ_API = 'https://api.openquake.org'
TERMINATE = config.distribution.terminate_workers_on_revoke
OQ_DISTRIBUTE = parallel.oq_distribute()

_PPID = os.getppid()  # the controlling terminal PID

if OQ_DISTRIBUTE == 'zmq':

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the available
        worker pools .
        """
        num_workers = 0
        w = config.zworkers
        for host, _cores in [hc.split() for hc in w.host_cores.split(',')]:
            url = 'tcp://%s:%s' % (host, w.ctrl_port)
            with z.Socket(url, z.zmq.REQ, 'connect') as sock:
                if not general.socket_ready(url):
                    logs.LOG.warn('%s is not running', host)
                    continue
                num_workers += sock.send('get_num_workers')
        OqParam.concurrent_tasks.default = num_workers * 3
        logs.LOG.info('Using %d zmq workers', num_workers)

elif OQ_DISTRIBUTE.startswith('celery'):
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks based on the number of available
        celery workers.
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
        OqParam.concurrent_tasks.default = num_cores * 3
        logs.LOG.info(
            'Using %s, %d cores', ', '.join(sorted(stats)), num_cores)

    def celery_cleanup(terminate, task_ids=()):
        """
        Release the resources used by an openquake job.
        In particular revoke the running tasks (if any).

        :param bool terminate: the celery revoke command terminate flag
        :param task_ids: celery task IDs
        """
        # Using the celery API, terminate and revoke and terminate any running
        # tasks associated with the current job.
        if task_ids:
            logs.LOG.warn('Revoking %d tasks', len(task_ids))
        else:  # this is normal when OQ_DISTRIBUTE=no
            logs.LOG.debug('No task to revoke')
        for tid in task_ids:
            celery.task.control.revoke(tid, terminate=terminate)
            logs.LOG.debug('Revoked task %s', tid)

def expose_outputs(dstore):
    """
    Build a correspondence between the outputs in the datastore and the
    ones in the database.

    :param dstore: datastore
    """
    oq = dstore['oqparam']
    exportable = set(ekey[0] for ekey in export.export)
    calcmode = oq.calculation_mode
    dskeys = set(dstore) & exportable  # exportable datastore keys
    dskeys.add('fullreport')
    rlzs = dstore['csm_info'].rlzs
    if len(rlzs) > 1:
        dskeys.add('realizations')
    # expose gmf_data only if < 10 MB
    if oq.ground_motion_fields and calcmode == 'event_based':
        nbytes = dstore['gmf_data'].attrs['nbytes']
        if nbytes < 10 * 1024 ** 2:
            dskeys.add('gmf_data')
    if 'scenario' not in calcmode:  # export sourcegroups.csv
        dskeys.add('sourcegroups')
    hdf5 = dstore.hdf5
    if (len(rlzs) == 1 and 'poes' in hdf5) or 'hcurves' in hdf5:
        dskeys.add('hcurves')
        if oq.uniform_hazard_spectra:
            dskeys.add('uhs')  # export them
        if oq.hazard_maps:
            dskeys.add('hmaps')  # export them
    if 'avg_losses-stats' in dstore or (
            'avg_losses-rlzs' in dstore and len(rlzs)):
        dskeys.add('avg_losses-stats')
    if 'curves-stats' in dstore:
        logs.LOG.warn('loss curves are exportable with oq export')
    if oq.conditional_loss_poes:  # expose loss_maps outputs
        if 'loss_curves-stats' in dstore:
            dskeys.add('loss_maps-stats')
    if 'all_loss_ratios' in dskeys:
        dskeys.remove('all_loss_ratios')  # export only specific IDs
    if 'ruptures' in dskeys and 'scenario' in calcmode:
        exportable.remove('ruptures')  # do not export, as requested by Vitor
    if 'rup_loss_table' in dskeys:  # keep it hidden for the moment
        dskeys.remove('rup_loss_table')
    logs.dbcmd('create_outputs', dstore.calc_id, sorted(dskeys & exportable))

class MasterKilled(KeyboardInterrupt):
    "Exception raised when a job is killed manually"

def raiseMasterKilled(signum, _stack):
    """
    When a SIGTERM is received, raise the MasterKilled
    exception with an appropriate error message.

    :param int signum: the number of the received signal
    :param _stack: the current frame object, ignored
    """
    msg = 'Received a signal %d' % signum

    if signum in (signal.SIGTERM, signal.SIGINT):
        msg = 'The openquake master process was killed manually'

    # kill the calculation only if os.getppid() != _PPID, i.e. the controlling
    # terminal died; in the workers, do nothing
    # NB: there is no SIGHUP on Windows
    if hasattr(signal, 'SIGHUP'):
        if signum == signal.SIGHUP:
            if os.getppid() == _PPID:
                return
            else:
                msg = 'The openquake master lost its controlling terminal'

    parallel.Starmap.shutdown()
    raise MasterKilled(msg)

# register the raiseMasterKilled callback for SIGTERM
# when using the Django development server this module is imported by a thread,
# so one gets a `ValueError: signal only works in main thread` that
# can be safely ignored
try:
    signal.signal(signal.SIGTERM, raiseMasterKilled)
    signal.signal(signal.SIGINT, raiseMasterKilled)
    if hasattr(signal, 'SIGHUP'):
        signal.signal(signal.SIGHUP, raiseMasterKilled)
except ValueError:
    pass

def job_from_file(cfg_file, username, hazard_calculation_id=None):
    """
    Create a full job profile from a job config file.

    :param str cfg_file:
        Path to a job.ini file.
    :param str username:
        The user who will own this job profile and all results
    :param str datadir:
        Data directory of the user
    :param hazard_calculation_id:
        ID of a previous calculation or None
    :returns:
        a pair (job_id, oqparam)
    """
    oq = readinput.get_oqparam(cfg_file, hc_id=hazard_calculation_id)
    job_id = logs.dbcmd('create_job', oq.calculation_mode, oq.description,
                        username, datastore.get_datadir(),
                        hazard_calculation_id)
    return job_id, oq

def run_calc(job_id, oqparam, log_level, log_file, exports,
             hazard_calculation_id=None, **kw):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    setproctitle('oq-job-%d' % job_id)
    monitor = Monitor('total runtime', measuremem=True)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if OQ_DISTRIBUTE.startswith(('celery', 'zmq')):
            set_concurrent_tasks_default()
        msg = check_obsolete_version(oqparam.calculation_mode)
        if msg:
            logs.LOG.warn(msg)
        calc = base.calculators(oqparam, monitor, calc_id=job_id)
        monitor.hdf5path = calc.datastore.hdf5path
        calc.from_engine = True
        tb = 'None\n'
        try:
            logs.dbcmd('set_status', job_id, 'executing')
            _do_run_calc(calc, exports, hazard_calculation_id, **kw)
            duration = monitor.duration
            expose_outputs(calc.datastore)
            monitor.flush()
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, duration)
            logs.dbcmd('finish', job_id, 'complete')
        except BaseException:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except BaseException:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if OQ_DISTRIBUTE.startswith('celery'):
                    celery_cleanup(TERMINATE, parallel.Starmap.task_ids)
            except BaseException:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc

def _do_run_calc(calc, exports, hazard_calculation_id, **kw):
    with calc._monitor:
        calc.run(exports=exports, hazard_calculation_id=hazard_calculation_id,
                 close=False, **kw)  # don't close the datastore too soon

def version_triple(tag):
    """
    returns: a triple of integers from a version tag
    """
    groups = re.match(r'v?(\d+)\.(\d+)\.(\d+)', tag).groups()
    return tuple(int(n) for n in groups)

def check_obsolete_version(calculation_mode='WebUI'):
    """
    Check if there is a newer version of the engine.

    :param calculation_mode:
         - the calculation mode when called from the engine
         - an empty string when called from the WebUI
    :returns:
        - a message if the running version of the engine is obsolete
        - the empty string if the engine is updated
        - None if the check could not be performed (i.e. github is down)
    """
    if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
        # avoid flooding our API server with requests from CI systems
        return

    headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
               (__version__, calculation_mode, platform.platform(),
                config.distribution.oq_distribute)}
    try:
        req = Request(OQ_API + '/engine/latest', headers=headers)
        # NB: a timeout < 1 does not work
        data = urlopen(req, timeout=1).read()  # bytes
        tag_name = json.loads(decode(data))['tag_name']
        current = version_triple(__version__)
        latest = version_triple(tag_name)
    except:  # page not available or wrong version tag
        return
    if current < latest:
        return ('Version %s of the engine is available, but you are '
                'still using version %s' % (tag_name, __version__))
    else:
        return ''
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""
Base classes for the output methods of the various codecs.
"""

import logging
from os.path import basename

from django.db import transaction
from django.db import connections
from django.db import router
from django.contrib.gis.db import models as gis_models

from openquake.engine.db import models

LOGGER = logging.getLogger('serializer')

# pylint: disable=W0212
class BulkInserter(object):
    """Handle bulk object insertion"""

    def __init__(self, dj_model, max_cache_size=None):
        """
        Create a new bulk inserter for a Django model class

        :param dj_model:
            Django model class
        :param int max_cache_size:
            The number of entries to cache before flushing/inserting. This
            helps to limit memory consumption for large sets of inserts.

            The default value is `None`, which means there is no maximum.
        """
        self.table = dj_model
        self.max_cache_size = max_cache_size
        self.fields = None
        self.values = []
        self.count = 0

    def add_entry(self, **kwargs):
        """
        Add a new entry to be inserted

        The first time the method is called the field list is stored;
        subsequent add_entry() calls must provide the same set of
        keyword arguments.

        Handles PostGIS/GeoDjango types.
        """
        if not self.fields:
            self.fields = kwargs.keys()
        assert set(self.fields) == set(kwargs.keys())
        for k in self.fields:
            self.values.append(kwargs[k])
        self.count += 1

        # If we have hit the `max_cache_size` is set,
        if self.max_cache_size is not None:
            # check if we have hit the maximum insert the current batch.
            if len(self.values) >= self.max_cache_size:
                self.flush()

    def flush(self):
        """Inserts the entries in the database using a bulk insert query"""
        if not self.values:
            return

        alias = router.db_for_write(self.table)
        cursor = connections[alias].cursor()
        value_args = []

        field_map = dict()
        for f in self.table._meta.fields:
            field_map[f.column] = f

        for f in self.fields:
            col = field_map[f]
            if isinstance(col, gis_models.GeometryField):
                value_args.append('GeomFromText(%%s, %d)' % col.srid)
            else:
                value_args.append('%s')

        sql = "INSERT INTO \"%s\" (%s) VALUES " % (
            self.table._meta.db_table, ", ".join(self.fields)) + \
            ", ".join(["(" + ", ".join(value_args) + ")"] * self.count)
        cursor.execute(sql, self.values)
        transaction.set_dirty(using=alias)

        self.fields = None
        self.values = []
        self.count = 0
<commit_msg>writer:<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""
Base classes for the output methods of the various codecs.
"""

import logging

from django.db import transaction
from django.db import connections
from django.db import router
from django.contrib.gis.db import models as gis_models

LOGGER = logging.getLogger('serializer')

# pylint: disable=W0212
class BulkInserter(object):
    """Handle bulk object insertion"""

    def __init__(self, dj_model, max_cache_size=None):
        """
        Create a new bulk inserter for a Django model class

        :param dj_model:
            Django model class
        :param int max_cache_size:
            The number of entries to cache before flushing/inserting. This
            helps to limit memory consumption for large sets of inserts.

            The default value is `None`, which means there is no maximum.
        """
        self.table = dj_model
        self.max_cache_size = max_cache_size
        self.fields = None
        self.values = []
        self.count = 0

    def add_entry(self, **kwargs):
        """
        Add a new entry to be inserted

        The first time the method is called the field list is stored;
        subsequent add_entry() calls must provide the same set of
        keyword arguments.

        Handles PostGIS/GeoDjango types.
        """
        if not self.fields:
            self.fields = kwargs.keys()
        assert set(self.fields) == set(kwargs.keys())
        for k in self.fields:
            self.values.append(kwargs[k])
        self.count += 1

        # If we have hit the `max_cache_size` is set,
        if self.max_cache_size is not None:
            # check if we have hit the maximum insert the current batch.
            if len(self.values) >= self.max_cache_size:
                self.flush()

    def flush(self):
        """Inserts the entries in the database using a bulk insert query"""
        if not self.values:
            return

        alias = router.db_for_write(self.table)
        cursor = connections[alias].cursor()
        value_args = []

        field_map = dict()
        for f in self.table._meta.fields:
            field_map[f.column] = f

        for f in self.fields:
            col = field_map[f]
            if isinstance(col, gis_models.GeometryField):
                value_args.append('GeomFromText(%%s, %d)' % col.srid)
            else:
                value_args.append('%s')

        sql = "INSERT INTO \"%s\" (%s) VALUES " % (
            self.table._meta.db_table, ", ".join(self.fields)) + \
            ", ".join(["(" + ", ".join(value_args) + ")"] * self.count)
        cursor.execute(sql, self.values)
        transaction.set_dirty(using=alias)

        self.fields = None
        self.values = []
        self.count = 0
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""
Base classes for the output methods of the various codecs.
"""

import logging
from os.path import basename

from django.db import transaction
from django.db import connections
from django.db import router
from django.contrib.gis.db import models as gis_models

from openquake.engine.db import models

LOGGER = logging.getLogger('serializer')

class FileWriter(object):
    """Simple output half of the codec process."""

    def __init__(self, path):
        self.path = path
        self.file = None
        self.root_node = None

    def initialize(self):
        """Initialization hook for derived classes."""
        pass

    def open(self):
        """Get the file handle open for writing"""
        self.file = open(self.path, "w")

    def write(self, point, value):
        """
        Write out an individual point (unimplemented).

        :param point: location associated with the data to be written
        :type point: should be a shapes.Site object
            Note(LB): Some subclasses override this behavior in their
            write() methods. Be careful.

        :param value: some value to be written to the file
        :type value: determined by concrete class implementation
        """
        raise NotImplementedError

    def close(self):
        """Close and flush the file. Send finished messages."""
        if self.file:
            self.file.close()

    def serialize(self, iterable):
        """Wrapper for writing all items in an iterable object."""
        if isinstance(iterable, dict):
            iterable = iterable.items()
        self.initialize()
        self.open()
        for key, val in iterable:
            self.write(key, val)
        self.close()

class XMLFileWriter(FileWriter):
    """
    Base class for writing XML files.
    """

    def write_header(self):
        """
        Write out the file header.
        """
        raise NotImplementedError

    def write_footer(self):
        """
        Write out the file footer.
        """
        raise NotImplementedError

    def serialize(self, iterable):
        """
        Wrapper for writing all items in an iterable object.
        """
        if isinstance(iterable, dict):
            iterable = iterable.items()
        self.initialize()
        self.open()
        self.write_header()
        for key, val in iterable:
            self.write(key, val)
        self.write_footer()
        self.close()

class DBWriter(object):
    """
    Abstact class implementing the "serialize" interface to output an iterable
    to the database.

    Subclasses must either implement get_output_type() and insert_datum() or
    override serialize().
    """

    def __init__(self, nrml_path, oq_job_id):
        self.nrml_path = nrml_path
        self.oq_job_id = oq_job_id
        self.output = None
        self.bulk_inserter = None

    def insert_output(self, output_type):
        """Insert an `uiapi.output` record for the job at hand."""

        assert self.output is None

        job = models.OqJob.objects.get(id=self.oq_job_id)

        # al-maisan, Fri, 29 Jun 2012 17:36:35 +0200
        # https://bugs.launchpad.net/openquake/+bug/1019317
        # figure out why using a single output record for gmf_data breaks the
        # probablistic event-based risk calculator.
        if output_type == "gmf":
            one_or_none = []
        else:
            one_or_none = models.Output.objects.filter(
                oq_job=job, display_name=basename(self.nrml_path),
                output_type=output_type)
        if len(one_or_none) == 1:
            self.output = one_or_none[0]
            LOGGER.info("using output = '%s'", self.output)
        else:
            self.output = models.Output(
                owner=job.owner, oq_job=job,
                display_name=basename(self.nrml_path), output_type=output_type)
            self.output.save()
            LOGGER.info("creating output = '%s'", self.output)

    def get_output_type(self):
        """
        The type of the output record as a string
        """
        raise NotImplementedError()

    def insert_datum(self, key, values):
        """
        Called for each item of the iterable during serialize.
        """
        raise NotImplementedError()

    @transaction.commit_on_success('reslt_writer')
    def serialize(self, iterable):
        """
        Implementation of the "serialize" interface.

        An Output record with type get_output_type() will be created, then
        each item of the iterable will be serialized in turn to the database.
        """
        LOGGER.info("serializing %s points", len(iterable))

        if not self.output:
            self.insert_output(self.get_output_type())
        LOGGER.info("output = '%s'", self.output)

        if isinstance(iterable, dict):
            items = iterable.iteritems()
        else:
            items = iterable

        for key, values in items:
            self.insert_datum(key, values)

        if self.bulk_inserter:
            self.bulk_inserter.flush()

        LOGGER.info("serialized %s points", len(iterable))

class CompositeWriter(object):
    """A writer that outputs to multiple writers"""

    def __init__(self, *writers):
        self.writers = writers

    def serialize(self, iterable):
        """Implementation of the "serialize" interface."""

        for writer in self.writers:
            if writer:
                writer.serialize(iterable)

def compose_writers(writers):
    """
    Takes a list of writers (the list can be empty or contain None items) and
    returns a single writer or None if the list didn't contain any writer.
    """

    if all(writer == None for writer in writers):  # True if the list is empty
        return None
    elif len(writers) == 1:
        return writers[0]
    else:
        return CompositeWriter(*writers)

# pylint: disable=W0212
class BulkInserter(object):
    """Handle bulk object insertion"""

    def __init__(self, dj_model, max_cache_size=None):
        """
        Create a new bulk inserter for a Django model class

        :param dj_model:
            Django model class
        :param int max_cache_size:
            The number of entries to cache before flushing/inserting. This
            helps to limit memory consumption for large sets of inserts.

            The default value is `None`, which means there is no maximum.
        """
        self.table = dj_model
        self.max_cache_size = max_cache_size
        self.fields = None
        self.values = []
        self.count = 0

    def add_entry(self, **kwargs):
        """
        Add a new entry to be inserted

        The first time the method is called the field list is stored;
        subsequent add_entry() calls must provide the same set of
        keyword arguments.

        Handles PostGIS/GeoDjango types.
        """
        if not self.fields:
            self.fields = kwargs.keys()
        assert set(self.fields) == set(kwargs.keys())
        for k in self.fields:
            self.values.append(kwargs[k])
        self.count += 1

        # If we have hit the `max_cache_size` is set,
        if self.max_cache_size is not None:
            # check if we have hit the maximum insert the current batch.
            if len(self.values) >= self.max_cache_size:
                self.flush()

    def flush(self):
        """Inserts the entries in the database using a bulk insert query"""
        if not self.values:
            return

        alias = router.db_for_write(self.table)
        cursor = connections[alias].cursor()
        value_args = []

        field_map = dict()
        for f in self.table._meta.fields:
            field_map[f.column] = f

        for f in self.fields:
            col = field_map[f]
            if isinstance(col, gis_models.GeometryField):
                value_args.append('GeomFromText(%%s, %d)' % col.srid)
            else:
                value_args.append('%s')

        sql = "INSERT INTO \"%s\" (%s) VALUES " % (
            self.table._meta.db_table, ", ".join(self.fields)) + \
            ", ".join(["(" + ", ".join(value_args) + ")"] * self.count)
        cursor.execute(sql, self.values)
        transaction.set_dirty(using=alias)

        self.fields = None
        self.values = []
        self.count = 0
<commit_msg>writer:<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""
Base classes for the output methods of the various codecs.
"""

import logging
from os.path import basename

from django.db import transaction
from django.db import connections
from django.db import router
from django.contrib.gis.db import models as gis_models

from openquake.engine.db import models

LOGGER = logging.getLogger('serializer')

# pylint: disable=W0212
class BulkInserter(object):
    """Handle bulk object insertion"""

    def __init__(self, dj_model, max_cache_size=None):
        """
        Create a new bulk inserter for a Django model class

        :param dj_model:
            Django model class
        :param int max_cache_size:
            The number of entries to cache before flushing/inserting. This
            helps to limit memory consumption for large sets of inserts.

            The default value is `None`, which means there is no maximum.
        """
        self.table = dj_model
        self.max_cache_size = max_cache_size
        self.fields = None
        self.values = []
        self.count = 0

    def add_entry(self, **kwargs):
        """
        Add a new entry to be inserted

        The first time the method is called the field list is stored;
        subsequent add_entry() calls must provide the same set of
        keyword arguments.

        Handles PostGIS/GeoDjango types.
        """
        if not self.fields:
            self.fields = kwargs.keys()
        assert set(self.fields) == set(kwargs.keys())
        for k in self.fields:
            self.values.append(kwargs[k])
        self.count += 1

        # If we have hit the `max_cache_size` is set,
        if self.max_cache_size is not None:
            # check if we have hit the maximum insert the current batch.
            if len(self.values) >= self.max_cache_size:
                self.flush()

    def flush(self):
        """Inserts the entries in the database using a bulk insert query"""
        if not self.values:
            return

        alias = router.db_for_write(self.table)
        cursor = connections[alias].cursor()
        value_args = []

        field_map = dict()
        for f in self.table._meta.fields:
            field_map[f.column] = f

        for f in self.fields:
            col = field_map[f]
            if isinstance(col, gis_models.GeometryField):
                value_args.append('GeomFromText(%%s, %d)' % col.srid)
            else:
                value_args.append('%s')

        sql = "INSERT INTO \"%s\" (%s) VALUES " % (
            self.table._meta.db_table, ", ".join(self.fields)) + \
            ", ".join(["(" + ", ".join(value_args) + ")"] * self.count)
        cursor.execute(sql, self.values)
        transaction.set_dirty(using=alias)

        self.fields = None
        self.values = []
        self.count = 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""NRML base path"""

import os

NRML_SCHEMA_FILE = 'nrml.xsd'

def nrml_schema_file():
    """Returns the absolute path to the NRML schema file"""
    return os.path.join(
        os.path.abspath(os.path.dirname(__file__)),
        'schema', NRML_SCHEMA_FILE)
<commit_msg>Finished moving everything to the 0.2 folder (and all associated references)<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""NRML base path"""

import os

NRML_SCHEMA_FILE = 'nrml.xsd'

def nrml_schema_file():
    """Returns the absolute path to the NRML schema file"""
    return os.path.join(
        os.path.abspath(os.path.dirname(__file__)),
        'schema', '0.2', NRML_SCHEMA_FILE)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from googl.short import GooglUrlShort
from django.conf import settings

GENERATE_SHORT_URL = getattr(settings, 'GENERATE_SHORT_URL', False)

def shorturl_generate(sender, instance, created, **kwargs):
    if GENERATE_SHORT_URL:
        if not instance.short_url:
            instance.short_url = GooglUrlShort(
                instance.get_http_absolute_url()
            ).short()
            instance.save()

def delete_container(sender, instance, using, **kwargs):
    """
    Ensure objects are deleted from mother class Article
    when deleted from child class
    """
    try:
        instance.__class__.objects.filter(
            child_class=instance.child_class,
            slug=instance.slug,
            channel=instance.channel,
            site=instance.site
        ).delete()
    except instance.__class__.DoesNotExist:
        # object not exists
        pass
<commit_msg>Fix #301<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from googl.short import GooglUrlShort
from django.conf import settings

from opps.channels.models import Channel

GENERATE_SHORT_URL = getattr(settings, 'GENERATE_SHORT_URL', False)

def shorturl_generate(sender, instance, created, **kwargs):
    if GENERATE_SHORT_URL:
        if not instance.short_url:
            instance.short_url = GooglUrlShort(
                instance.get_http_absolute_url()
            ).short()
            instance.save()

def delete_container(sender, instance, using, **kwargs):
    """
    Ensure objects are deleted from mother class Article
    when deleted from child class
    """
    try:
        instance.__class__.objects.filter(
            child_class=instance.child_class,
            slug=instance.slug,
            channel=instance.channel,
            site=instance.site
        ).delete()
    except (instance.__class__.DoesNotExist, Channel.DoesNotExist):
        # object not exists
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django import template
from django.conf import settings
from django.contrib.sites.models import get_current_site

from opps.articles.models import Album
from opps.containers.models import Container, ContainerBox
from opps.channels.models import Channel

class View(object):

    context_object_name = "context"
    paginate_by = settings.OPPS_PAGINATE_BY
    limit = settings.OPPS_VIEWS_LIMIT
    page_kwarg = 'page'

    def __init__(self):
        self.slug = None
        self.channel = None
        self.long_slug = None
        self.channel_long_slug = []
        self.article = None
        self.excluded_ids = set()
        self.child_class = u'container'

    def get_paginate_by(self, queryset):
        queryset = self.get_queryset()

        setting_name = 'OPPS_{}_{}_PAGINATE_BY'.format(queryset.
                                                       model._meta.app_label,
                                                       queryset.model.
                                                       __name__).upper()

        paginate_by = getattr(settings, setting_name, self.paginate_by)

        return paginate_by

    def get_context_data(self, **kwargs):
        if not self.long_slug:
            context = []
            return context
        context = super(View, self).get_context_data(**kwargs)

        # channel is needed everywhere
        self.channel = self.channel or Channel.objects.get_homepage(
            site=get_current_site(self.request)
        )

        if hasattr(self, 'articleboxes'):
            context['articleboxes'] = self.articleboxes
        else:
            context['articleboxes'] = ContainerBox.objects.filter(
                channel__long_slug=self.long_slug)
            self.excluded_ids = []
            for box in context['articleboxes']:
                self.excluded_ids += [a.pk for a in box.ordered_containers()]

        obj_filter = {}
        obj_filter['site_domain'] = self.site.domain
        obj_filter['date_available__lte'] = timezone.now()

        obj_filter['published'] = True

        filters = obj_filter
        filters['channel_long_slug__in'] = self.channel_long_slug

        is_paginated = self.page_kwarg in self.request.GET
        if self.channel and self.channel.is_root_node() and not is_paginated:
            filters['show_on_root_channel'] = True
        article = Container.objects.filter(**filters)

        context['posts'] = article.filter(
            child_class='Post'
        ).exclude(pk__in=self.excluded_ids)[:self.limit]

        context['albums'] = Album.objects.filter(
            **filters
        ).exclude(pk__in=self.excluded_ids)[:self.limit]

        context['channel'] = {}
        context['channel']['long_slug'] = self.long_slug
        if self.channel:
            context['channel'] = self.channel

        # this is a palliative measure as long as we don't fix the real
        # problem
        context['breadcrumb'] = getattr(self, 'channel_descendants', [])

        if self.slug:
            try:
                context['next'] = self.get_object()\
                    .get_next_by_date_insert(**obj_filter)
            except self.get_object().DoesNotExist:
                pass
            try:
                context['prev'] = self.get_object()\
                    .get_previous_by_date_insert(**obj_filter)
            except self.get_object().DoesNotExist:
                pass

            context['articleboxes'] = context['articleboxes'].filter(
                containers__slug=self.slug)

            if self.get_object().child_class == 'Mirror':
                context['context'] = self.get_object().container

        if self.request.META.get('HTTP_X_PJAX', False) or\
           self.request.is_ajax():
            context['extends_parent'] = 'base_ajax.html'

        return context

    def get_template_folder(self):
        domain_folder = "containers"
        if self.site.id > 1:
            domain_folder = "{}/containers".format(self.site.domain)
        return domain_folder

    def get_long_slug(self):
        self.long_slug = self.kwargs.get('channel__long_slug', None)
        try:
            if not self.long_slug:
                self.long_slug = Channel.objects.get_homepage(
                    site=self.site).long_slug
        except AttributeError:
            pass
        return self.long_slug

    def set_channel_rules(self):
        self.fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        filters = dict(
            site__domain=self.site.domain,
            long_slug=self.long_slug,
            date_available__lte=timezone.now(),
            published=True
        )

        try:
            self.channel = Channel.objects.get(**filters)
        except Channel.DoesNotExist:
            if not self.fallback or self.site == self.site_master:
                raise Http404('Channel not found and fallback disabled')
            filters['site__domain'] = self.site_master.domain
            self.channel = get_object_or_404(Channel, **filters)

        self.long_slug = self.channel.long_slug

        self.channel_long_slug = [self.long_slug]
        self.channel_descendants = self.channel.get_descendants(
            include_self=False)
        for children in self.channel_descendants:
            self.channel_long_slug.append(children.long_slug)

    def check_template(self, _template):
        try:
            template.loader.get_template(_template)
            return True
        except template.TemplateDoesNotExist:
            return False
<commit_msg>Fix breadcrumb #291<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django import template
from django.conf import settings
from django.contrib.sites.models import get_current_site

from opps.articles.models import Album
from opps.containers.models import Container, ContainerBox
from opps.channels.models import Channel

class View(object):

    context_object_name = "context"
    paginate_by = settings.OPPS_PAGINATE_BY
    limit = settings.OPPS_VIEWS_LIMIT
    page_kwarg = 'page'

    def __init__(self):
        self.slug = None
        self.channel = None
        self.long_slug = None
        self.channel_long_slug = []
        self.article = None
        self.excluded_ids = set()
        self.child_class = u'container'

    def get_paginate_by(self, queryset):
        queryset = self.get_queryset()

        setting_name = 'OPPS_{}_{}_PAGINATE_BY'.format(queryset.
                                                       model._meta.app_label,
                                                       queryset.model.
                                                       __name__).upper()

        paginate_by = getattr(settings, setting_name, self.paginate_by)

        return paginate_by

    def get_context_data(self, **kwargs):
        if not self.long_slug:
            context = []
            return context
        context = super(View, self).get_context_data(**kwargs)

        # channel is needed everywhere
        self.channel = self.channel or Channel.objects.get_homepage(
            site=get_current_site(self.request)
        )

        if hasattr(self, 'articleboxes'):
            context['articleboxes'] = self.articleboxes
        else:
            context['articleboxes'] = ContainerBox.objects.filter(
                channel__long_slug=self.long_slug)
            self.excluded_ids = []
            for box in context['articleboxes']:
                self.excluded_ids += [a.pk for a in box.ordered_containers()]

        obj_filter = {}
        obj_filter['site_domain'] = self.site.domain
        obj_filter['date_available__lte'] = timezone.now()

        obj_filter['published'] = True

        filters = obj_filter
        filters['channel_long_slug__in'] = self.channel_long_slug

        is_paginated = self.page_kwarg in self.request.GET
        if self.channel and self.channel.is_root_node() and not is_paginated:
            filters['show_on_root_channel'] = True
        article = Container.objects.filter(**filters)

        context['posts'] = article.filter(
            child_class='Post'
        ).exclude(pk__in=self.excluded_ids)[:self.limit]

        context['albums'] = Album.objects.filter(
            **filters
        ).exclude(pk__in=self.excluded_ids)[:self.limit]

        context['channel'] = {}
        context['channel']['long_slug'] = self.long_slug
        if self.channel:
            context['channel'] = self.channel

        context['breadcrumb'] = self.get_breadcrumb()

        if self.slug:
            try:
                context['next'] = self.get_object()\
                    .get_next_by_date_insert(**obj_filter)
            except self.get_object().DoesNotExist:
                pass
            try:
                context['prev'] = self.get_object()\
                    .get_previous_by_date_insert(**obj_filter)
            except self.get_object().DoesNotExist:
                pass

            context['articleboxes'] = context['articleboxes'].filter(
                containers__slug=self.slug)

            if self.get_object().child_class == 'Mirror':
                context['context'] = self.get_object().container

        if self.request.META.get('HTTP_X_PJAX', False) or\
           self.request.is_ajax():
            context['extends_parent'] = 'base_ajax.html'

        return context

    def get_template_folder(self):
        domain_folder = "containers"
        if self.site.id > 1:
            domain_folder = "{}/containers".format(self.site.domain)
        return domain_folder

    def get_long_slug(self):
        self.long_slug = self.kwargs.get('channel__long_slug', None)
        try:
            if not self.long_slug:
                self.long_slug = Channel.objects.get_homepage(
                    site=self.site).long_slug
        except AttributeError:
            pass
        return self.long_slug

    def set_channel_rules(self):
        self.fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        filters = dict(
            site__domain=self.site.domain,
            long_slug=self.long_slug,
            date_available__lte=timezone.now(),
            published=True
        )

        try:
            self.channel = Channel.objects.get(**filters)
        except Channel.DoesNotExist:
            if not self.fallback or self.site == self.site_master:
                raise Http404('Channel not found and fallback disabled')
            filters['site__domain'] = self.site_master.domain
            self.channel = get_object_or_404(Channel, **filters)

        self.long_slug = self.channel.long_slug

        self.channel_long_slug = [self.long_slug]
        self.channel_descendants = self.channel.get_descendants(
            include_self=False)
        for children in self.channel_descendants:
            self.channel_long_slug.append(children.long_slug)

    def get_breadcrumb(self):
        if self.channel.is_root_node():
            return []

        return self.channel.get_ancestors(include_self=True)

    def check_template(self, _template):
        try:
            template.loader.get_template(_template)
            return True
        except template.TemplateDoesNotExist:
            return False
<|endoftext|>


<commit_before>import os
import yaml
import json
import re
import requests
import logging
import socket
from datetime import datetime

import teuthology
from .config import config
from .contextutil import safe_while

def init_logging():
    """
    Set up logging for the module

    :returns: a logger
    """
    # Don't need to see connection pool INFO messages
    logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
        logging.WARNING)

    log = logging.getLogger(__name__)
    return log

def main(args):
    run = args['--run']
    job = args['--job']
    dead = args['--dead']
    refresh = dead or args['--refresh']
    server = args['--server']
    if server:
        config.results_server = server
    if args['--verbose']:
        teuthology.log.setLevel(logging.DEBUG)

    archive_base = os.path.abspath(os.path.expanduser(args['--archive']))
    save = not args['--no-save']

    log = init_logging()
    reporter = ResultsReporter(archive_base, save=save, refresh=refresh,
                               log=log)
    if dead and not job:
        for run_name in run:
            reporter.report_run(run[0], dead=True)
    elif dead and len(run) == 1 and job:
        reporter.report_jobs(run[0], job, dead=True)
    elif len(run) == 1 and job:
        reporter.report_jobs(run[0], job)
    elif run and len(run) > 1:
        reporter.report_runs(run)
    elif run:
        reporter.report_run(run[0])
    elif args['--all-runs']:
        reporter.report_all_runs()

class ResultsSerializer(object):
    """
    This class exists to poke around in the archive directory doing things like
    assembling lists of test runs, lists of their jobs, and merging sets of job
    YAML files together to form JSON objects.
    """
    yamls = ('orig.config.yaml', 'config.yaml', 'info.yaml', 'summary.yaml')

    def __init__(self, archive_base, log=None):
        self.archive_base = archive_base
        self.log = log or init_logging()

    def job_info(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A dict.
        """
        job_archive_dir = os.path.join(self.archive_base,
                                       run_name,
                                       job_id)
        job_info = {}
        for yaml_name in self.yamls:
            yaml_path = os.path.join(job_archive_dir, yaml_name)
            if not os.path.exists(yaml_path):
                continue
            with file(yaml_path) as yaml_file:
                partial_info = yaml.safe_load(yaml_file)
                if partial_info is not None:
                    job_info.update(partial_info)

        log_path = os.path.join(job_archive_dir, 'teuthology.log')
        if os.path.exists(log_path):
            mtime = int(os.path.getmtime(log_path))
            mtime_dt = datetime.fromtimestamp(mtime)
            job_info['updated'] = str(mtime_dt)

        if 'job_id' not in job_info:
            job_info['job_id'] = job_id

        return job_info

    def json_for_job(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together to
        create a JSON object.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A JSON object.
        """
        job_info = self.job_info(run_name, job_id, pretty)
        if pretty:
            job_json = json.dumps(job_info, sort_keys=True, indent=4)
        else:
            job_json = json.dumps(job_info)

        return job_json

    def jobs_for_run(self, run_name):
        """
        Given a run name, look on the filesystem for directories containing job
        information, and return a dict mapping job IDs to job directories.

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        archive_dir = os.path.join(self.archive_base, run_name)
        if not os.path.isdir(archive_dir):
            return {}
        jobs = {}
        for item in os.listdir(archive_dir):
            if not re.match('\d+$', item):
                continue
            job_id = item
            job_dir = os.path.join(archive_dir, job_id)
            if os.path.isdir(job_dir):
                jobs[job_id] = job_dir
        return jobs

    def running_jobs_for_run(self, run_name):
        """
        Like jobs_for_run(), but only returns jobs with no summary.yaml

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        jobs = self.jobs_for_run(run_name)
        for job_id in jobs.keys():
            if os.path.exists(os.path.join(jobs[job_id], 'summary.yaml')):
                jobs.pop(job_id)
        return jobs

    @property
    def all_runs(self):
        """
        Look in the base archive directory for all test runs. Return a list of
        their names.
        """
        archive_base = self.archive_base
        if not os.path.isdir(archive_base):
            return []
        runs = []
        for run_name in os.listdir(archive_base):
            if not os.path.isdir(os.path.join(archive_base, run_name)):
                continue
            runs.append(run_name)
        return runs

class ResultsReporter(object):
    last_run_file = 'last_successful_run'

    def __init__(self, archive_base=None, base_uri=None, save=False,
                 refresh=False, timeout=20, log=None):
        self.log = log or init_logging()
        self.archive_base = archive_base or config.archive_base
        self.base_uri = base_uri or config.results_server
        if self.base_uri:
            self.base_uri = self.base_uri.rstrip('/')
        self.serializer = ResultsSerializer(archive_base, log=self.log)
        self.save_last_run = save
        self.refresh = refresh
        self.timeout = timeout

    def report_all_runs(self):
        """
        Report *all* runs in self.archive_dir to the results server.
        """
        all_runs = self.serializer.all_runs
        last_run = self.last_run
        if self.save_last_run and last_run and last_run in all_runs:
            next_index = all_runs.index(last_run) + 1
            runs = all_runs[next_index:]
        else:
            runs = all_runs
        return self.report_runs(runs)

    def report_runs(self, run_names):
        """
        Report several runs to the results server.

        :param run_names: The names of the runs.
        """
        num_runs = len(run_names)
        num_jobs = 0
        self.log.info("Posting %s runs", num_runs)
        for run in run_names:
            job_count = self.report_run(run)
            num_jobs += job_count
            if self.save_last_run:
                self.last_run = run
        del self.last_run
        self.log.info("Total: %s jobs in %s runs", num_jobs, len(run_names))

    def report_run(self, run_name, dead=False):
        """
        Report a single run to the results server.

        :param run_name: The name of the run.
        :returns:        The number of jobs reported.
        """
        jobs = self.serializer.jobs_for_run(run_name)
        self.log.info("{name} {jobs} jobs dead={dead}".format(
            name=run_name,
            jobs=len(jobs),
            dead=str(dead),
        ))
        if jobs:
            if not self.refresh:
                response = requests.head("{base}/runs/{name}/".format(
                    base=self.base_uri, name=run_name))
                if response.status_code == 200:
                    self.log.info("    already present; skipped")
                    return 0
            self.report_jobs(run_name, jobs.keys(), dead=dead)
        elif not jobs:
            self.log.debug("    no jobs; skipped")
        return len(jobs)

    def report_jobs(self, run_name, job_ids, dead=False):
        """
        Report several jobs to the results server.

        :param run_name: The name of the run.
        :param job_ids:  The jobs' ids
        """
        for job_id in job_ids:
            self.report_job(run_name, job_id, dead=dead)

    def report_job(self, run_name, job_id, job_info=None, dead=False):
        """
        Report a single job to the results server.

        :param run_name: The name of the run. The run must already exist.
        :param job_id:   The job's id
        :param job_info: The job's info dict. Optional - if not present, we
                         look at the archive.
        """
        if job_info is not None and not isinstance(job_info, dict):
            raise TypeError("job_info must be a dict")
        run_uri = "{base}/runs/{name}/jobs/".format(
            base=self.base_uri, name=run_name,)
        if job_info is None:
            job_info = self.serializer.job_info(run_name, job_id)
        if dead and job_info.get('success') is None:
            job_info['status'] = 'dead'
        job_json = json.dumps(job_info)
        headers = {'content-type': 'application/json'}
        response = requests.post(run_uri, data=job_json, headers=headers)

        if response.status_code == 200:
            return job_id

        if response.json:
            msg = response.json.get('message', '')
        else:
            msg = response.text

        if msg and msg.endswith('already exists'):
            job_uri = os.path.join(run_uri, job_id, '')
            response = requests.put(job_uri, data=job_json, headers=headers)
        elif msg:
            self.log.error(
                "POST to {uri} failed with status {status}: {msg}".format(
                    uri=run_uri,
                    status=response.status_code,
                    msg=msg,
                ))
        response.raise_for_status()

        return job_id

    @property
    def last_run(self):
        """
        The last run to be successfully reported.
        """
        if hasattr(self, '__last_run'):
            return self.__last_run
        elif os.path.exists(self.last_run_file):
            with file(self.last_run_file) as f:
                self.__last_run = f.read().strip()
            return self.__last_run

    @last_run.setter
    def last_run(self, run_name):
        self.__last_run = run_name
        with file(self.last_run_file, 'w') as f:
            f.write(run_name)

    @last_run.deleter
    def last_run(self):
        self.__last_run = None
        if os.path.exists(self.last_run_file):
            os.remove(self.last_run_file)

    def get_jobs(self, run_name, fields=None):
        """
        Query the results server for jobs in a run

        :param run_name: The name of the run
        :param fields:   Optional. A list of fields to include in the result.
                         Defaults to returning all fields.
        """
        uri = "{base}/runs/{name}/jobs/".format(base=self.base_uri,
                                                name=run_name)
        if fields:
            if not 'job_id' in fields:
                fields.append('job_id')
            uri += "?fields=" + ','.join(fields)
        response = requests.get(uri)
        response.raise_for_status()
        return response.json

    def delete_job(self, run_name, job_id):
        """
        Delete a job from the results server.

        :param run_name: The name of the run
        :param job_id:   The job's id
        """
        uri = "{base}/runs/{name}/jobs/{job_id}/".format(
            base=self.base_uri, name=run_name, job_id=job_id)
        response = requests.delete(uri)
        response.raise_for_status()

    def delete_jobs(self, run_name, job_ids):
        """
        Delete multiple jobs from the results server.

        :param run_name: The name of the run
        :param job_ids:  A list of job ids
        """
        for job_id in job_ids:
            self.delete_job(self, run_name, job_id)

def push_job_info(run_name, job_id, job_info, base_uri=None):
    """
    Push a job's info (example: ctx.config) to the results server.

    :param run_name: The name of the run.
    :param job_id:   The job's id
    :param job_info: A dict containing the job's information.
    :param base_uri: The endpoint of the results server. If you leave it out
                     ResultsReporter will ask teuthology.config.
    """
    reporter = ResultsReporter()
    reporter.report_job(run_name, job_id, job_info)

def try_push_job_info(job_config, extra_info=None):
    """
    Wrap push_job_info, gracefully doing nothing if:
        Anything inheriting from requests.exceptions.RequestException is raised
        A socket.error is raised
        config.results_server is not set
        config['job_id'] is not present or is None

    :param job_config: The ctx.config object to push
    :param extra_info: Optional second dict to push
    """
    log = init_logging()

    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to push results"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return
    elif job_config.get('job_id') is None:
        log.warning('No job_id found; not reporting results')
        return

    run_name = job_config['name']
    job_id = job_config['job_id']

    if extra_info is not None:
        job_info = extra_info.copy()
        job_info.update(job_config)
    else:
        job_info = job_config

    with safe_while(_raise=False) as proceed:
        while proceed():
            try:
                log.info("Pushing job info to %s", config.results_server)
                push_job_info(run_name, job_id, job_info)
                return
            except (requests.exceptions.RequestException, socket.error):
                log.exception("Could not report results to %s",
                              config.results_server)

def try_delete_jobs(run_name, job_ids):
    """
    Using the same error checking and retry mechanism as try_push_job_info(),
    delete one or more jobs

    :param run_name: The name of the run.
    :param job_ids:  Either a single job_id, or a list of job_ids
    """
    log = init_logging()

    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to delete job"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return

    if isinstance(job_ids, int):
        job_ids = [str(job_ids)]
    elif isinstance(job_ids, basestring):
        job_ids = [job_ids]

    reporter = ResultsReporter()
    for job_id in job_ids:
        with safe_while(_raise=False) as proceed:
            while proceed():
                try:
                    log.info("Pushing job info to %s", config.results_server)
                    reporter.delete_job(run_name, job_id)
                    return
                except (requests.exceptions.RequestException, socket.error):
                    log.exception("Could not report results to %s",
                                  config.results_server)
<commit_msg>Be slightly less verbose about logging<commit_after>import os
import yaml
import json
import re
import requests
import logging
import socket
from datetime import datetime

import teuthology
from .config import config
from .contextutil import safe_while

def init_logging():
    """
    Set up logging for the module

    :returns: a logger
    """
    # Don't need to see connection pool INFO messages
    logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
        logging.WARNING)

    log = logging.getLogger(__name__)
    return log

def main(args):
    run = args['--run']
    job = args['--job']
    dead = args['--dead']
    refresh = dead or args['--refresh']
    server = args['--server']
    if server:
        config.results_server = server
    if args['--verbose']:
        teuthology.log.setLevel(logging.DEBUG)

    archive_base = os.path.abspath(os.path.expanduser(args['--archive']))
    save = not args['--no-save']

    log = init_logging()
    reporter = ResultsReporter(archive_base, save=save, refresh=refresh,
                               log=log)
    if dead and not job:
        for run_name in run:
            reporter.report_run(run[0], dead=True)
    elif dead and len(run) == 1 and job:
        reporter.report_jobs(run[0], job, dead=True)
    elif len(run) == 1 and job:
        reporter.report_jobs(run[0], job)
    elif run and len(run) > 1:
        reporter.report_runs(run)
    elif run:
        reporter.report_run(run[0])
    elif args['--all-runs']:
        reporter.report_all_runs()

class ResultsSerializer(object):
    """
    This class exists to poke around in the archive directory doing things like
    assembling lists of test runs, lists of their jobs, and merging sets of job
    YAML files together to form JSON objects.
    """
    yamls = ('orig.config.yaml', 'config.yaml', 'info.yaml', 'summary.yaml')

    def __init__(self, archive_base, log=None):
        self.archive_base = archive_base
        self.log = log or init_logging()

    def job_info(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A dict.
        """
        job_archive_dir = os.path.join(self.archive_base,
                                       run_name,
                                       job_id)
        job_info = {}
        for yaml_name in self.yamls:
            yaml_path = os.path.join(job_archive_dir, yaml_name)
            if not os.path.exists(yaml_path):
                continue
            with file(yaml_path) as yaml_file:
                partial_info = yaml.safe_load(yaml_file)
                if partial_info is not None:
                    job_info.update(partial_info)

        log_path = os.path.join(job_archive_dir, 'teuthology.log')
        if os.path.exists(log_path):
            mtime = int(os.path.getmtime(log_path))
            mtime_dt = datetime.fromtimestamp(mtime)
            job_info['updated'] = str(mtime_dt)

        if 'job_id' not in job_info:
            job_info['job_id'] = job_id

        return job_info

    def json_for_job(self, run_name, job_id, pretty=False):
        """
        Given a run name and job id, merge the job's YAML files together to
        create a JSON object.

        :param run_name: The name of the run.
        :param job_id:   The job's id.
        :returns:        A JSON object.
        """
        job_info = self.job_info(run_name, job_id, pretty)
        if pretty:
            job_json = json.dumps(job_info, sort_keys=True, indent=4)
        else:
            job_json = json.dumps(job_info)

        return job_json

    def jobs_for_run(self, run_name):
        """
        Given a run name, look on the filesystem for directories containing job
        information, and return a dict mapping job IDs to job directories.

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        archive_dir = os.path.join(self.archive_base, run_name)
        if not os.path.isdir(archive_dir):
            return {}
        jobs = {}
        for item in os.listdir(archive_dir):
            if not re.match('\d+$', item):
                continue
            job_id = item
            job_dir = os.path.join(archive_dir, job_id)
            if os.path.isdir(job_dir):
                jobs[job_id] = job_dir
        return jobs

    def running_jobs_for_run(self, run_name):
        """
        Like jobs_for_run(), but only returns jobs with no summary.yaml

        :param run_name: The name of the run.
        :returns:        A dict like: {'1': '/path/to/1', '2': 'path/to/2'}
        """
        jobs = self.jobs_for_run(run_name)
        for job_id in jobs.keys():
            if os.path.exists(os.path.join(jobs[job_id], 'summary.yaml')):
                jobs.pop(job_id)
        return jobs

    @property
    def all_runs(self):
        """
        Look in the base archive directory for all test runs. Return a list of
        their names.
        """
        archive_base = self.archive_base
        if not os.path.isdir(archive_base):
            return []
        runs = []
        for run_name in os.listdir(archive_base):
            if not os.path.isdir(os.path.join(archive_base, run_name)):
                continue
            runs.append(run_name)
        return runs

class ResultsReporter(object):
    last_run_file = 'last_successful_run'

    def __init__(self, archive_base=None, base_uri=None, save=False,
                 refresh=False, timeout=20, log=None):
        self.log = log or init_logging()
        self.archive_base = archive_base or config.archive_base
        self.base_uri = base_uri or config.results_server
        if self.base_uri:
            self.base_uri = self.base_uri.rstrip('/')
        self.serializer = ResultsSerializer(archive_base, log=self.log)
        self.save_last_run = save
        self.refresh = refresh
        self.timeout = timeout

    def report_all_runs(self):
        """
        Report *all* runs in self.archive_dir to the results server.
        """
        all_runs = self.serializer.all_runs
        last_run = self.last_run
        if self.save_last_run and last_run and last_run in all_runs:
            next_index = all_runs.index(last_run) + 1
            runs = all_runs[next_index:]
        else:
            runs = all_runs
        return self.report_runs(runs)

    def report_runs(self, run_names):
        """
        Report several runs to the results server.

        :param run_names: The names of the runs.
        """
        num_runs = len(run_names)
        num_jobs = 0
        self.log.info("Posting %s runs", num_runs)
        for run in run_names:
            job_count = self.report_run(run)
            num_jobs += job_count
            if self.save_last_run:
                self.last_run = run
        del self.last_run
        self.log.info("Total: %s jobs in %s runs", num_jobs, len(run_names))

    def report_run(self, run_name, dead=False):
        """
        Report a single run to the results server.

        :param run_name: The name of the run.
        :returns:        The number of jobs reported.
        """
        jobs = self.serializer.jobs_for_run(run_name)
        self.log.info("{name} {jobs} jobs dead={dead}".format(
            name=run_name,
            jobs=len(jobs),
            dead=str(dead),
        ))
        if jobs:
            if not self.refresh:
                response = requests.head("{base}/runs/{name}/".format(
                    base=self.base_uri, name=run_name))
                if response.status_code == 200:
                    self.log.info("    already present; skipped")
                    return 0
            self.report_jobs(run_name, jobs.keys(), dead=dead)
        elif not jobs:
            self.log.debug("    no jobs; skipped")
        return len(jobs)

    def report_jobs(self, run_name, job_ids, dead=False):
        """
        Report several jobs to the results server.

        :param run_name: The name of the run.
        :param job_ids:  The jobs' ids
        """
        for job_id in job_ids:
            self.report_job(run_name, job_id, dead=dead)

    def report_job(self, run_name, job_id, job_info=None, dead=False):
        """
        Report a single job to the results server.

        :param run_name: The name of the run. The run must already exist.
        :param job_id:   The job's id
        :param job_info: The job's info dict. Optional - if not present, we
                         look at the archive.
        """
        if job_info is not None and not isinstance(job_info, dict):
            raise TypeError("job_info must be a dict")
        run_uri = "{base}/runs/{name}/jobs/".format(
            base=self.base_uri, name=run_name,)
        if job_info is None:
            job_info = self.serializer.job_info(run_name, job_id)
        if dead and job_info.get('success') is None:
            job_info['status'] = 'dead'
        job_json = json.dumps(job_info)
        headers = {'content-type': 'application/json'}
        response = requests.post(run_uri, data=job_json, headers=headers)

        if response.status_code == 200:
            return job_id

        if response.json:
            msg = response.json.get('message', '')
        else:
            msg = response.text

        if msg and msg.endswith('already exists'):
            job_uri = os.path.join(run_uri, job_id, '')
            response = requests.put(job_uri, data=job_json, headers=headers)
        elif msg:
            self.log.error(
                "POST to {uri} failed with status {status}: {msg}".format(
                    uri=run_uri,
                    status=response.status_code,
                    msg=msg,
                ))
        response.raise_for_status()

        return job_id

    @property
    def last_run(self):
        """
        The last run to be successfully reported.
        """
        if hasattr(self, '__last_run'):
            return self.__last_run
        elif os.path.exists(self.last_run_file):
            with file(self.last_run_file) as f:
                self.__last_run = f.read().strip()
            return self.__last_run

    @last_run.setter
    def last_run(self, run_name):
        self.__last_run = run_name
        with file(self.last_run_file, 'w') as f:
            f.write(run_name)

    @last_run.deleter
    def last_run(self):
        self.__last_run = None
        if os.path.exists(self.last_run_file):
            os.remove(self.last_run_file)

    def get_jobs(self, run_name, fields=None):
        """
        Query the results server for jobs in a run

        :param run_name: The name of the run
        :param fields:   Optional. A list of fields to include in the result.
                         Defaults to returning all fields.
        """
        uri = "{base}/runs/{name}/jobs/".format(base=self.base_uri,
                                                name=run_name)
        if fields:
            if not 'job_id' in fields:
                fields.append('job_id')
            uri += "?fields=" + ','.join(fields)
        response = requests.get(uri)
        response.raise_for_status()
        return response.json

    def delete_job(self, run_name, job_id):
        """
        Delete a job from the results server.

        :param run_name: The name of the run
        :param job_id:   The job's id
        """
        uri = "{base}/runs/{name}/jobs/{job_id}/".format(
            base=self.base_uri, name=run_name, job_id=job_id)
        response = requests.delete(uri)
        response.raise_for_status()

    def delete_jobs(self, run_name, job_ids):
        """
        Delete multiple jobs from the results server.

        :param run_name: The name of the run
        :param job_ids:  A list of job ids
        """
        for job_id in job_ids:
            self.delete_job(self, run_name, job_id)

def push_job_info(run_name, job_id, job_info, base_uri=None):
    """
    Push a job's info (example: ctx.config) to the results server.

    :param run_name: The name of the run.
    :param job_id:   The job's id
    :param job_info: A dict containing the job's information.
    :param base_uri: The endpoint of the results server. If you leave it out
                     ResultsReporter will ask teuthology.config.
    """
    reporter = ResultsReporter()
    reporter.report_job(run_name, job_id, job_info)

def try_push_job_info(job_config, extra_info=None):
    """
    Wrap push_job_info, gracefully doing nothing if:
        Anything inheriting from requests.exceptions.RequestException is raised
        A socket.error is raised
        config.results_server is not set
        config['job_id'] is not present or is None

    :param job_config: The ctx.config object to push
    :param extra_info: Optional second dict to push
    """
    log = init_logging()

    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to push results"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return
    elif job_config.get('job_id') is None:
        log.warning('No job_id found; not reporting results')
        return

    run_name = job_config['name']
    job_id = job_config['job_id']

    if extra_info is not None:
        job_info = extra_info.copy()
        job_info.update(job_config)
    else:
        job_info = job_config

    with safe_while(_raise=False) as proceed:
        while proceed():
            try:
                log.debug("Pushing job info to %s", config.results_server)
                push_job_info(run_name, job_id, job_info)
                return
            except (requests.exceptions.RequestException, socket.error):
                log.exception("Could not report results to %s",
                              config.results_server)

def try_delete_jobs(run_name, job_ids):
    """
    Using the same error checking and retry mechanism as try_push_job_info(),
    delete one or more jobs

    :param run_name: The name of the run.
    :param job_ids:  Either a single job_id, or a list of job_ids
    """
    log = init_logging()

    if not config.results_server:
        msg = "No results_server set in {yaml}; not attempting to delete job"
        log.debug(msg.format(yaml=config.teuthology_yaml))
        return

    if isinstance(job_ids, int):
        job_ids = [str(job_ids)]
    elif isinstance(job_ids, basestring):
        job_ids = [job_ids]

    reporter = ResultsReporter()
    log.debug("Deleting jobs from {server}: {jobs}".format(
        server=config.results_server, jobs=str(job_ids)))
    for job_id in job_ids:
        with safe_while(_raise=False) as proceed:
            while proceed():
                try:
                    reporter.delete_job(run_name, job_id)
                    return
                except (requests.exceptions.RequestException, socket.error):
                    log.exception("Could not report results to %s",
                                  config.results_server)
<|endoftext|>