<commit_before>import os
import sys
import fcntl
import errno
import atexit
import logging
import socket
import ssl
import time
import datetime
import calendar
import re
import string
if sys.version_info[0] < 3:
    from Queue import Queue, Full
else:
    from queue import Queue, Full

from threading import Thread
import pkg_resources

def normalize_time(time_like):
    """Returns unix timestamp integer for all common time/duration formats."""
    if isinstance(time_like, time.struct_time):
        time_like = time.mktime(time_like)
    if isinstance(time_like, datetime.datetime):
        time_like = calendar.timegm(time_like.utctimetuple())
    if isinstance(time_like, datetime.timedelta):
        time_like = time_like.total_seconds()
    return int(time_like)

def is_valid(metric, value, time, count):
    """Returns True/False if a metric/value/time/count is valid"""
    valid_metric = re.search(r"^([\d\w\-_]+\.)*[\d\w\-_]+$", metric)

    valid_value = re.search(r"^-?\d+(\.\d+)?(e-\d+)?$", str(value))

    if valid_metric and valid_value:
        return True

    # TODO
    # report_invalid_metric(metric) unless valid_metric
    # report_invalid_value(metric, value) unless valid_value
    return False

def is_valid_note(message):
    """Returns True/False if a notice message is valid."""
    return not bool(re.search("[\n\r]", message))

def join(strings, joiner):
    """
    Joins a list of strings together with an interleaved joiner string.
    This is a compatibility function for Python 2/3.
    """
    if sys.version_info[0] < 3:
        return string.join(strings, joiner)
    else:
        return joiner.join(strings)

def add_stderr_logger(level=logging.DEBUG):
    """
    Helper for quickly adding a StreamHandler to the logger. Useful for
    debugging.

    Returns the handler after adding it.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    logger.addHandler(handler)
    logger.setLevel(level)
    logger.debug('Added a stderr logging handler to logger: %s', __name__)
    return handler

class Agent(object):
    """
    Used to connect to Instrumental and send metric data.
    """
    backoff = 2
    connect_timeout = 20
    reply_timeout = 10
    exit_flush_timeout = 5
    hostname = socket.gethostname()
    max_buffer = 5000
    max_reconnect_delay = 15
    exit_timeout = 1
    version = "1.1.0"

    def __init__(self, api_key, collector="collector.instrumentalapp.com:8001", enabled=True, secure=True, verify_cert=True, synchronous=False):
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)
        self.logger.debug("Initializing...")

        self.api_key = api_key
        self.host, self.port = collector.split(":")
        self.port = int(self.port)
        self.secure = secure
        self.verify_cert = verify_cert
        self.enabled = enabled
        self.synchronous = synchronous
        self.worker = False
        self.bare_socket = False
        self.socket = False

        self.pid = None
        self.failures = 0

        if self.enabled:
            self.queue = Queue(Agent.max_buffer)
            self._setup_cleanup_at_exit()

    def gauge(self, metric, value, timestamp=None, count=1):
        """
        Store a gauge for a metric, optionally at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("gauge", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def increment(self, metric, value=1, timestamp=None, count=1):
        """
        Increment a metric, optionally more than one or at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("increment", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def notice(self, note, timestamp=None, duration=0):
        """
        Records a note at a specific time and duration. Useful for things like
        deploys or other significant changes.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid_note(note):
                self._send_command("notice", normalize_time(timestamp), normalize_time(duration), note)
                return note
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time(self, metric, fun, multiplier=1):
        """
        Store the execution duration of a function in a metric. Multiplier can
        be used to scale the duration to desired unit or change the duration
        in some meaningful way. Default is in seconds.
        """
        try:
            start = time.time()
            value = fun()
            finish = time.time()
            duration = finish - start
            self.gauge(metric, duration * multiplier, start)
            return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time_ms(self, metric, fun):
        """
        Store the execution duration of a function in a metric. Execution time
        is measured in milliseconds.
        """
        try:
            return self.time(metric, fun, 1000)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def is_running(self):
        """Returns True/False if the worker is running"""
        return self._same_pid() and self._worker_alive()

    def _same_pid(self):
        return bool(os.getpid() == self.pid)

    def _worker_alive(self):
        return bool(self.worker) and self.worker.is_alive()

    def _setup_cleanup_at_exit(self):
        self.logger.debug("registering exit handler")
        atexit.register(self._cleanup)

    def _cleanup(self):
        try:
            if self.is_running:
                if self.queue.empty():
                    self.logger.debug("At Exit handler, join skipped, queue empty.")
                else:
                    self.logger.debug("At Exit handler, waiting up to %0.3f seconds (count: %i) ", Agent.exit_timeout, self.queue.qsize())
                    started = time.time()
                    while (time.time() - started) < Agent.exit_timeout and not self.queue.empty():
                        time.sleep(0.05)
                    if self.queue.empty():
                        self.logger.debug("All metrics pushed.")
                    else:
                        self.logger.info("Discarding %i metrics.", self.queue.qsize())
            else:
                self.logger.debug("At Exit handler, join skipped, worker not running.")
        except Exception as error:
            self.logger.error("At Exit ERROR: " + str(error))

    def _start_connection_worker(self):
        if self.enabled:
            self.pid = os.getpid()
            self.failures = 0
            self.logger.debug("Starting thread...")

            self.worker = Thread(target=self._worker_loop)
            self.worker.setDaemon(True)  # So exit handler won't wait on this
            self.worker.start()

    def _worker_loop(self):
        self.logger.debug("worker starting...")
        while True:
            try:
                self.bare_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

                if self.secure:
                    # Rely on server to enforce secure protocol/ciphers
                    self.socket = ssl.wrap_socket(self.bare_socket)
                else:
                    self.socket = self.bare_socket

                self.socket.connect((self.host, self.port))
                self.socket.settimeout(self.reply_timeout)

                def socket_send(data):
                    if sys.version_info[0] < 3:
                        self.socket.send(data)
                    else:
                        self.socket.send(bytes(data, "ASCII"))

                socket_send("hello version python/instrumental_agent/%(version)s hostname %(hostname)s\n" % {"version": Agent.version, "hostname": Agent.hostname})
                socket_send("authenticate %s\n" % self.api_key)

                data = b""
                ok_count = 0
                receiving = True
                connected = False

                while receiving:
                    data += self.socket.recv(1024)
                    while b"\n" in data:
                        self.logger.debug("initial data: %s" % repr(data))
                        response, data = data.split(b"\n", 1)
                        self.logger.debug("  response: %s" % repr(response))
                        self.logger.debug("  data: %s" % repr(data))
                        if response == b"ok":
                            ok_count += 1
                            if ok_count >= 2:
                                self.logger.debug("auth a-ok")
                                receiving = False
                                connected = True
                                break
                        else:
                            self.logger.debug("auth failed...")
                            break
                if connected:
                    self.socket.settimeout(None)
                    while True:
                        item = self.queue.get()
                        if self._test_connection():
                            socket_send(item)
                            self.queue.task_done()
                        else:
                            self.queue.put(item, False)
                            raise Exception("socket error")
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception as error:
                self.failures += 1
                delay = min((self.failures - 1) ** Agent.backoff, Agent.max_reconnect_delay)
                time.sleep(delay)
                self._log_exception(error)
                import traceback
                traceback.print_exc(file=sys.stdout)
                sys.exit()

    def _log_exception(self, error):
        self.logger.debug("EXCEPTION %s", str(error))

    def _send_command(self, cmd, *args):
        if self.enabled:
            args = [str(arg) for arg in args]
            string_cmd = "%s %s\n" % (cmd, join(args, " "))
            if not self.is_running():
                self._start_connection_worker()
            try:
                self.queue.put(string_cmd, False)
            except Full:
                self.logger.debug("Queue full(limit %i), discarding metric", Agent.max_buffer)

    def _test_connection(self):
        s = self.bare_socket
        fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK)

        try:
            msg = s.recv(1)
            return msg == ""
        except socket.error as e:
            err = e.args[0]
            if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
                return True
            else:
                self.logger.debug("Socket connection error")
                return False
<commit_msg>Suppress warning when logging to a logger without a handler<commit_after>import os
import sys
import fcntl
import errno
import atexit
import logging
import socket
import ssl
import time
import datetime
import calendar
import re
import string
if sys.version_info[0] < 3:
    from Queue import Queue, Full
else:
    from queue import Queue, Full

from threading import Thread
from logging import NullHandler
import pkg_resources

def normalize_time(time_like):
    """Returns unix timestamp integer for all common time/duration formats."""
    if isinstance(time_like, time.struct_time):
        time_like = time.mktime(time_like)
    if isinstance(time_like, datetime.datetime):
        time_like = calendar.timegm(time_like.utctimetuple())
    if isinstance(time_like, datetime.timedelta):
        time_like = time_like.total_seconds()
    return int(time_like)

def is_valid(metric, value, time, count):
    """Returns True/False if a metric/value/time/count is valid"""
    valid_metric = re.search(r"^([\d\w\-_]+\.)*[\d\w\-_]+$", metric)

    valid_value = re.search(r"^-?\d+(\.\d+)?(e-\d+)?$", str(value))

    if valid_metric and valid_value:
        return True

    # TODO
    # report_invalid_metric(metric) unless valid_metric
    # report_invalid_value(metric, value) unless valid_value
    return False

def is_valid_note(message):
    """Returns True/False if a notice message is valid."""
    return not bool(re.search("[\n\r]", message))

def join(strings, joiner):
    """
    Joins a list of strings together with an interleaved joiner string.
    This is a compatibility function for Python 2/3.
    """
    if sys.version_info[0] < 3:
        return string.join(strings, joiner)
    else:
        return joiner.join(strings)

def add_stderr_logger(level=logging.DEBUG):
    """
    Helper for quickly adding a StreamHandler to the logger. Useful for
    debugging.

    Returns the handler after adding it.
    """
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    logger.addHandler(handler)
    logger.setLevel(level)
    logger.debug('Added a stderr logging handler to logger: %s', __name__)
    return handler

class Agent(object):
    """
    Used to connect to Instrumental and send metric data.
    """
    backoff = 2
    connect_timeout = 20
    reply_timeout = 10
    exit_flush_timeout = 5
    hostname = socket.gethostname()
    max_buffer = 5000
    max_reconnect_delay = 15
    exit_timeout = 1
    version = "1.1.0"

    def __init__(self, api_key, collector="collector.instrumentalapp.com:8001", enabled=True, secure=True, verify_cert=True, synchronous=False):
        self.logger = logging.getLogger(__name__)
        self.logger..addHandler(NullHandler())
        self.logger.setLevel(logging.DEBUG)
        self.logger.debug("Initializing...")

        self.api_key = api_key
        self.host, self.port = collector.split(":")
        self.port = int(self.port)
        self.secure = secure
        self.verify_cert = verify_cert
        self.enabled = enabled
        self.synchronous = synchronous
        self.worker = False
        self.bare_socket = False
        self.socket = False

        self.pid = None
        self.failures = 0

        if self.enabled:
            self.queue = Queue(Agent.max_buffer)
            self._setup_cleanup_at_exit()

    def gauge(self, metric, value, timestamp=None, count=1):
        """
        Store a gauge for a metric, optionally at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("gauge", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def increment(self, metric, value=1, timestamp=None, count=1):
        """
        Increment a metric, optionally more than one or at a specific time.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid(metric, value, timestamp, count):
                self._send_command("increment", metric, value, normalize_time(timestamp), count)
                return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def notice(self, note, timestamp=None, duration=0):
        """
        Records a note at a specific time and duration. Useful for things like
        deploys or other significant changes.
        """
        if timestamp is None:
            timestamp = time.time()
        try:
            if is_valid_note(note):
                self._send_command("notice", normalize_time(timestamp), normalize_time(duration), note)
                return note
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time(self, metric, fun, multiplier=1):
        """
        Store the execution duration of a function in a metric. Multiplier can
        be used to scale the duration to desired unit or change the duration
        in some meaningful way. Default is in seconds.
        """
        try:
            start = time.time()
            value = fun()
            finish = time.time()
            duration = finish - start
            self.gauge(metric, duration * multiplier, start)
            return value
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def time_ms(self, metric, fun):
        """
        Store the execution duration of a function in a metric. Execution time
        is measured in milliseconds.
        """
        try:
            return self.time(metric, fun, 1000)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as error:
            self._log_exception(error)

    def is_running(self):
        """Returns True/False if the worker is running"""
        return self._same_pid() and self._worker_alive()

    def _same_pid(self):
        return bool(os.getpid() == self.pid)

    def _worker_alive(self):
        return bool(self.worker) and self.worker.is_alive()

    def _setup_cleanup_at_exit(self):
        self.logger.debug("registering exit handler")
        atexit.register(self._cleanup)

    def _cleanup(self):
        try:
            if self.is_running:
                if self.queue.empty():
                    self.logger.debug("At Exit handler, join skipped, queue empty.")
                else:
                    self.logger.debug("At Exit handler, waiting up to %0.3f seconds (count: %i) ", Agent.exit_timeout, self.queue.qsize())
                    started = time.time()
                    while (time.time() - started) < Agent.exit_timeout and not self.queue.empty():
                        time.sleep(0.05)
                    if self.queue.empty():
                        self.logger.debug("All metrics pushed.")
                    else:
                        self.logger.info("Discarding %i metrics.", self.queue.qsize())
            else:
                self.logger.debug("At Exit handler, join skipped, worker not running.")
        except Exception as error:
            self.logger.error("At Exit ERROR: " + str(error))

    def _start_connection_worker(self):
        if self.enabled:
            self.pid = os.getpid()
            self.failures = 0
            self.logger.debug("Starting thread...")

            self.worker = Thread(target=self._worker_loop)
            self.worker.setDaemon(True)  # So exit handler won't wait on this
            self.worker.start()

    def _worker_loop(self):
        self.logger.debug("worker starting...")
        while True:
            try:
                self.bare_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

                if self.secure:
                    # Rely on server to enforce secure protocol/ciphers
                    self.socket = ssl.wrap_socket(self.bare_socket)
                else:
                    self.socket = self.bare_socket

                self.socket.connect((self.host, self.port))
                self.socket.settimeout(self.reply_timeout)

                def socket_send(data):
                    if sys.version_info[0] < 3:
                        self.socket.send(data)
                    else:
                        self.socket.send(bytes(data, "ASCII"))

                socket_send("hello version python/instrumental_agent/%(version)s hostname %(hostname)s\n" % {"version": Agent.version, "hostname": Agent.hostname})
                socket_send("authenticate %s\n" % self.api_key)

                data = b""
                ok_count = 0
                receiving = True
                connected = False

                while receiving:
                    data += self.socket.recv(1024)
                    while b"\n" in data:
                        self.logger.debug("initial data: %s" % repr(data))
                        response, data = data.split(b"\n", 1)
                        self.logger.debug("  response: %s" % repr(response))
                        self.logger.debug("  data: %s" % repr(data))
                        if response == b"ok":
                            ok_count += 1
                            if ok_count >= 2:
                                self.logger.debug("auth a-ok")
                                receiving = False
                                connected = True
                                break
                        else:
                            self.logger.debug("auth failed...")
                            break
                if connected:
                    self.socket.settimeout(None)
                    while True:
                        item = self.queue.get()
                        if self._test_connection():
                            socket_send(item)
                            self.queue.task_done()
                        else:
                            self.queue.put(item, False)
                            raise Exception("socket error")
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception as error:
                self.failures += 1
                delay = min((self.failures - 1) ** Agent.backoff, Agent.max_reconnect_delay)
                time.sleep(delay)
                self._log_exception(error)
                import traceback
                traceback.print_exc(file=sys.stdout)
                sys.exit()

    def _log_exception(self, error):
        self.logger.debug("EXCEPTION %s", str(error))

    def _send_command(self, cmd, *args):
        if self.enabled:
            args = [str(arg) for arg in args]
            string_cmd = "%s %s\n" % (cmd, join(args, " "))
            if not self.is_running():
                self._start_connection_worker()
            try:
                self.queue.put(string_cmd, False)
            except Full:
                self.logger.debug("Queue full(limit %i), discarding metric", Agent.max_buffer)

    def _test_connection(self):
        s = self.bare_socket
        fcntl.fcntl(s, fcntl.F_SETFL, os.O_NONBLOCK)

        try:
            msg = s.recv(1)
            return msg == ""
        except socket.error as e:
            err = e.args[0]
            if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
                return True
            else:
                self.logger.debug("Socket connection error")
                return False
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2010-2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the scenario_damage risk calculator.
"""

import numpy

from django import db

from openquake.commonlib.riskmodels import get_damage_states_and_risk_models

from openquake.engine.calculators.risk import (
    base, hazard_getters, writers, validation)
from openquake.engine.performance import EnginePerformanceMonitor
from openquake.engine.utils import tasks
from openquake.engine.db import models

@tasks.oqtask
def scenario_damage(job_id, risk_model, getters, outputdict, params):
    """
    Celery task for the scenario damage risk calculator.

    :param int job_id:
      ID of the currently running job
    :param risk_model:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getters:
      A list of callable hazard getters
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (in this case only `LossMap`)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
   :returns:
      A matrix of fractions and a taxonomy string
    """
    monitor = EnginePerformanceMonitor(
        None, job_id, scenario_damage, tracing=True)
    # in scenario damage calculation the only loss_type is 'damage'
    [getter] = getters
    [ffs] = risk_model.vulnerability_functions

    # and NO containes
    assert len(outputdict) == 0
    with db.transaction.commit_on_success(using='job_init'):

        with monitor.copy('getting hazard'):
            ground_motion_values = getter.get_data(ffs.imt)

        with monitor.copy('computing risk'):
            fractions = risk_model.workflow(ground_motion_values)
            aggfractions = sum(fractions[i] * asset.number_of_units
                               for i, asset in enumerate(getter.assets))

        with monitor.copy('saving damage per assets'):
            writers.damage_distribution(
                getter.assets, fractions, params.damage_state_ids)

        return {risk_model.taxonomy: aggfractions}

class ScenarioDamageRiskCalculator(base.RiskCalculator):
    """
    Scenario Damage Risk Calculator. Computes four kinds of damage
    distributions: per asset, per taxonomy, total and collapse map.

    :attr dict fragility_functions:
        A dictionary of dictionary mapping taxonomy ->
        (limit state -> fragility function) where a fragility function is an
        instance of
        :class:`openquake.risklib.scientific.FragilityFunctionContinuous` or
        :class:`openquake.risklib.scientific.FragilityFunctionDiscrete`.
    """

    #: The core calculation celery task function
    core_calc_task = scenario_damage
    validators = [validation.HazardIMT, validation.EmptyExposure,
                  validation.OrphanTaxonomies,
                  validation.NoRiskModels, validation.RequireScenarioHazard]

    # FIXME. scenario damage calculator does not use output builders
    output_builders = []
    getter_class = hazard_getters.GroundMotionValuesGetter

    def __init__(self, job):
        super(ScenarioDamageRiskCalculator, self).__init__(job)
        # let's define a dictionary taxonomy -> fractions
        # updated in task_completed method when the fractions per taxonomy
        # becomes available, as computed by the workers
        self.acc = {}
        self.damage_state_ids = None

    @EnginePerformanceMonitor.monitor
    def agg_result(self, acc, task_result):
        """
        Update the dictionary acc, i.e. aggregate the damage distribution
        by taxonomy; called every time a block of assets is computed for each
        taxonomy. Fractions and taxonomy are extracted from task_result

        :param task_result:
            A pair (fractions, taxonomy)
        """
        acc = acc.copy()
        for taxonomy, fractions in task_result.iteritems():
            if fractions is not None:
                if taxonomy not in acc:
                    acc[taxonomy] = numpy.zeros(fractions.shape)
                acc[taxonomy] += fractions
        return acc

    def post_process(self):
        """
        Save the damage distributions by taxonomy and total on the db.
        """

        models.Output.objects.create_output(
            self.job, "Damage Distribution per Asset",
            "dmg_dist_per_asset")

        models.Output.objects.create_output(
            self.job, "Collapse Map per Asset",
            "collapse_map")

        if self.acc:
            models.Output.objects.create_output(
                self.job, "Damage Distribution per Taxonomy",
                "dmg_dist_per_taxonomy")

        tot = None
        for taxonomy, fractions in self.acc.iteritems():
            writers.damage_distribution_per_taxonomy(
                fractions, self.damage_state_ids, taxonomy)
            if tot is None:  # only the first time
                tot = numpy.zeros(fractions.shape)
            tot += fractions

        if tot is not None:
            models.Output.objects.create_output(
                self.job, "Damage Distribution Total",
                "dmg_dist_total")
            writers.total_damage_distribution(tot, self.damage_state_ids)

    def get_risk_models(self):
        """
        Load fragility model and store damage states
        """
        damage_states, risk_models = get_damage_states_and_risk_models(
            self.rc.inputs['fragility'])

        for lsi, dstate in enumerate(damage_states):
            models.DmgState.objects.get_or_create(
                risk_calculation=self.rc, dmg_state=dstate, lsi=lsi)

        self.damage_state_ids = [d.id for d in models.DmgState.objects.filter(
            risk_calculation=self.rc).order_by('lsi')]

        self.loss_types.add('damage')  # single loss_type
        return risk_models

    @property
    def calculator_parameters(self):
        """
        Provides calculator specific params coming from
        :class:`openquake.engine.db.RiskCalculation`
        """
        return base.make_calc_params(damage_state_ids=self.damage_state_ids)
<commit_msg>Used get_risk_models from commonlib<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2010-2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the scenario_damage risk calculator.
"""

import numpy

from django import db

from openquake.commonlib.riskmodels import get_risk_models

from openquake.engine.calculators.risk import (
    base, hazard_getters, writers, validation)
from openquake.engine.performance import EnginePerformanceMonitor
from openquake.engine.utils import tasks
from openquake.engine.db import models

@tasks.oqtask
def scenario_damage(job_id, risk_model, getters, outputdict, params):
    """
    Celery task for the scenario damage risk calculator.

    :param int job_id:
      ID of the currently running job
    :param risk_model:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getters:
      A list of callable hazard getters
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (in this case only `LossMap`)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
   :returns:
      A matrix of fractions and a taxonomy string
    """
    monitor = EnginePerformanceMonitor(
        None, job_id, scenario_damage, tracing=True)
    # in scenario damage calculation the only loss_type is 'damage'
    [getter] = getters
    [ffs] = risk_model.vulnerability_functions

    # and NO containes
    assert len(outputdict) == 0
    with db.transaction.commit_on_success(using='job_init'):

        with monitor.copy('getting hazard'):
            ground_motion_values = getter.get_data(ffs.imt)

        with monitor.copy('computing risk'):
            fractions = risk_model.workflow(ground_motion_values)
            aggfractions = sum(fractions[i] * asset.number_of_units
                               for i, asset in enumerate(getter.assets))

        with monitor.copy('saving damage per assets'):
            writers.damage_distribution(
                getter.assets, fractions, params.damage_state_ids)

        return {risk_model.taxonomy: aggfractions}

class ScenarioDamageRiskCalculator(base.RiskCalculator):
    """
    Scenario Damage Risk Calculator. Computes four kinds of damage
    distributions: per asset, per taxonomy, total and collapse map.

    :attr dict fragility_functions:
        A dictionary of dictionary mapping taxonomy ->
        (limit state -> fragility function) where a fragility function is an
        instance of
        :class:`openquake.risklib.scientific.FragilityFunctionContinuous` or
        :class:`openquake.risklib.scientific.FragilityFunctionDiscrete`.
    """

    #: The core calculation celery task function
    core_calc_task = scenario_damage
    validators = [validation.HazardIMT, validation.EmptyExposure,
                  validation.OrphanTaxonomies,
                  validation.NoRiskModels, validation.RequireScenarioHazard]

    # FIXME. scenario damage calculator does not use output builders
    output_builders = []
    getter_class = hazard_getters.GroundMotionValuesGetter

    def __init__(self, job):
        super(ScenarioDamageRiskCalculator, self).__init__(job)
        # let's define a dictionary taxonomy -> fractions
        # updated in task_completed method when the fractions per taxonomy
        # becomes available, as computed by the workers
        self.acc = {}
        self.damage_state_ids = None

    @EnginePerformanceMonitor.monitor
    def agg_result(self, acc, task_result):
        """
        Update the dictionary acc, i.e. aggregate the damage distribution
        by taxonomy; called every time a block of assets is computed for each
        taxonomy. Fractions and taxonomy are extracted from task_result

        :param task_result:
            A pair (fractions, taxonomy)
        """
        acc = acc.copy()
        for taxonomy, fractions in task_result.iteritems():
            if fractions is not None:
                if taxonomy not in acc:
                    acc[taxonomy] = numpy.zeros(fractions.shape)
                acc[taxonomy] += fractions
        return acc

    def post_process(self):
        """
        Save the damage distributions by taxonomy and total on the db.
        """

        models.Output.objects.create_output(
            self.job, "Damage Distribution per Asset",
            "dmg_dist_per_asset")

        models.Output.objects.create_output(
            self.job, "Collapse Map per Asset",
            "collapse_map")

        if self.acc:
            models.Output.objects.create_output(
                self.job, "Damage Distribution per Taxonomy",
                "dmg_dist_per_taxonomy")

        tot = None
        for taxonomy, fractions in self.acc.iteritems():
            writers.damage_distribution_per_taxonomy(
                fractions, self.damage_state_ids, taxonomy)
            if tot is None:  # only the first time
                tot = numpy.zeros(fractions.shape)
            tot += fractions

        if tot is not None:
            models.Output.objects.create_output(
                self.job, "Damage Distribution Total",
                "dmg_dist_total")
            writers.total_damage_distribution(tot, self.damage_state_ids)

    def get_risk_models(self):
        """
        Load fragility model and store damage states
        """
        risk_models = get_risk_models(models.oqparam(self.job.id))

        for lsi, dstate in enumerate(risk_models.damage_states):
            models.DmgState.objects.get_or_create(
                risk_calculation=self.rc, dmg_state=dstate, lsi=lsi)

        self.damage_state_ids = [d.id for d in models.DmgState.objects.filter(
            risk_calculation=self.rc).order_by('lsi')]

        self.loss_types.add('damage')  # single loss_type
        return risk_models

    @property
    def calculator_parameters(self):
        """
        Provides calculator specific params coming from
        :class:`openquake.engine.db.RiskCalculation`
        """
        return base.make_calc_params(damage_state_ids=self.damage_state_ids)
<|endoftext|>


<commit_before>from . import base
from grow.common import utils
from contentful.cda import client
from contentful.cda import resources
from protorpc import messages

class KeyMessage(messages.Message):
    preview = messages.StringField(1)
    production = messages.StringField(2)

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    type = messages.StringField(2)

class ContentfulPreprocessor(base.BasePreprocessor):
    KIND = 'contentful'

    class Config(messages.Message):
        space = messages.StringField(2)
        keys = messages.MessageField(KeyMessage, 3)
        bind = messages.MessageField(BindingMessage, 4, repeated=True)

    def bind_collection(self, entries, collection_pod_path, contentful_type):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path for doc in collection.list_docs()]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            if entry.sys['contentType']['sys']['id'] != contentful_type:
                continue
            body = entry.fields.pop('body', None)
            fields = entry.fields
            for key, value in entry.fields.iteritems():
                if isinstance(value, resources.Asset):
                    entry.fields[key] = value.url
                elif isinstance(value, resources.Entry):
                    entry.fields[key] = value.sys['id']
            if body:
                body = body.encode('utf-8')
                ext = 'md'
            else:
                body = ''
                ext = 'yaml'
            if 'title' in entry.fields:
                title = entry.fields.pop('title')
                entry.fields['$title'] = title
            entry.fields['$order'] = i
            basename = '{}.{}'.format(entry.sys['id'], ext)
            doc = collection.create_doc(basename, fields=fields, body=body)
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))
        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def run(self, *args, **kwargs):
        token = self.config.keys.preview
        token = self.config.keys.production
        cda = client.Client(self.config.space, token)
        entries = cda.fetch(resources.Entry).all()
        for binding in self.config.bind:
            self.bind_collection(entries, binding.collection, binding.type)
<commit_msg>Add doc injection method.<commit_after>import webapp2
from . import base
from grow.common import utils
from contentful.cda import client
from contentful.cda import resources
from protorpc import messages

class KeyMessage(messages.Message):
    preview = messages.StringField(1)
    production = messages.StringField(2)

class BindingMessage(messages.Message):
    collection = messages.StringField(1)
    type = messages.StringField(2)

class ContentfulPreprocessor(base.BasePreprocessor):
    KIND = 'contentful'

    class Config(messages.Message):
        space = messages.StringField(2)
        keys = messages.MessageField(KeyMessage, 3)
        bind = messages.MessageField(BindingMessage, 4, repeated=True)

    def parse_entry(self, entry):
        body = entry.fields.pop('body', None)
        fields = entry.fields
        for key, value in entry.fields.iteritems():
            if isinstance(value, resources.Asset):
                entry.fields[key] = value.url
            elif isinstance(value, resources.Entry):
                entry.fields[key] = value.sys['id']
        if body:
            body = body.encode('utf-8')
            ext = 'md'
        else:
            body = ''
            ext = 'yaml'
        if 'title' in entry.fields:
            title = entry.fields.pop('title')
            entry.fields['$title'] = title
        basename = '{}.{}'.format(entry.sys['id'], ext)
        return fields, body, basename

    def bind_collection(self, entries, collection_pod_path, contentful_type):
        collection = self.pod.get_collection(collection_pod_path)
        existing_pod_paths = [
            doc.pod_path for doc in collection.list_docs()]
        new_pod_paths = []
        for i, entry in enumerate(entries):
            if entry.sys['contentType']['sys']['id'] != contentful_type:
                continue
            fields, body, basename = self.parse_entry(entry)
            doc = collection.create_doc(basename, fields=fields, body=body)
            new_pod_paths.append(doc.pod_path)
            self.pod.logger.info('Saved -> {}'.format(doc.pod_path))
        pod_paths_to_delete = set(existing_pod_paths) - set(new_pod_paths)
        for pod_path in pod_paths_to_delete:
            self.pod.delete_file(pod_path)
            self.pod.logger.info('Deleted -> {}'.format(pod_path))

    def run(self, *args, **kwargs):
        entries = self.cda.fetch(resources.Entry).all()
        for binding in self.config.bind:
            self.bind_collection(entries, binding.collection, binding.type)

    @webapp2.cached_property
    def cda(self):
        token = self.config.keys.preview
        token = self.config.keys.production
        return client.Client(self.config.space, token)

    def inject(self, doc):
        query = {'sys.id': doc.base}
        entry = self.cda.fetch(resources.Entry).where(query).first()
        if not entry:
            return
        fields, body, basename = self.parse_entry(entry)
        doc.fields.update(fields)
        doc.format.body = body
        doc.pod.logger.info('Injected -> {}'.format(doc.pod_path))
        return
<|endoftext|>


<commit_before>from immercv.cvgraph.forms import form_for_node_properties
from immercv.cvgraph.models import editable_params, get_by_id, Person, Note

COMMAND_FUNCTIONS = {
    # Created via application of `command` decorator.
    #
    # (labels, operation, relationship_name): update-function,
}

def apply_command(request, properties):
    labels = properties.pop('_labels')[0]
    node_id = properties.pop('_id', [None])[0]
    operation = properties.pop('_operation')[0]
    relationship_name = properties.pop('_relationship_name', [None])[0]
    key = (labels, operation, relationship_name)
    print(key)
    fn = COMMAND_FUNCTIONS.get(key)
    if callable(fn):
        fn(request, properties, node_id)

def command(labels, operation, relationship=None):
    def command_decorator(fn):
        COMMAND_FUNCTIONS[(labels, operation, relationship)] = fn
        return fn
    return command_decorator

def create_node_from_params(cls, params):
    node = cls()
    set_node_properties_from_params(node, params)
    node.save()
    return node

def set_node_properties_from_params(node, params):
    for k, v in params.items():
        property = getattr(node.__class__, k)
        if not property.required and v == '':
            v = None
        setattr(node, k, v)

@command(':Note', 'delete')
def delete_note(request, params, node_id):
    note = get_by_id(Note, node_id)
    note.delete()

@command(':Note', 'update')
def update_note(request, params, node_id):
    note = get_by_id(Note, node_id)
    params = editable_params(params, ':Note')
    form = form_for_node_properties(note, params.keys(), params)
    if form.is_valid():
        set_node_properties_from_params(note, form.cleaned_data)
        note.save()

@command(':Person', 'create', 'notes')
def create_person_note(request, params, node_id):
    person = get_by_id(Person, node_id)
    params = editable_params(params, ':Note')
    form = form_for_node_properties(Note, params.keys(), params)
    if form.is_valid():
        note = create_node_from_params(Note, params)
        person.notes.connect(note)

@command(':Person', 'update')
def update_person(request, params, node_id):
    params = editable_params(params, ':Person')
    person = get_by_id(Person, node_id)
    set_node_properties_from_params(person, params)
    person.save()
<commit_msg>Remove print() call<commit_after>from immercv.cvgraph.forms import form_for_node_properties
from immercv.cvgraph.models import editable_params, get_by_id, Person, Note

COMMAND_FUNCTIONS = {
    # Created via application of `command` decorator.
    #
    # (labels, operation, relationship_name): update-function,
}

def apply_command(request, properties):
    labels = properties.pop('_labels')[0]
    node_id = properties.pop('_id', [None])[0]
    operation = properties.pop('_operation')[0]
    relationship_name = properties.pop('_relationship_name', [None])[0]
    key = (labels, operation, relationship_name)
    fn = COMMAND_FUNCTIONS.get(key)
    if callable(fn):
        fn(request, properties, node_id)

def command(labels, operation, relationship=None):
    def command_decorator(fn):
        COMMAND_FUNCTIONS[(labels, operation, relationship)] = fn
        return fn
    return command_decorator

def create_node_from_params(cls, params):
    node = cls()
    set_node_properties_from_params(node, params)
    node.save()
    return node

def set_node_properties_from_params(node, params):
    for k, v in params.items():
        property = getattr(node.__class__, k)
        if not property.required and v == '':
            v = None
        setattr(node, k, v)

@command(':Note', 'delete')
def delete_note(request, params, node_id):
    note = get_by_id(Note, node_id)
    note.delete()

@command(':Note', 'update')
def update_note(request, params, node_id):
    note = get_by_id(Note, node_id)
    params = editable_params(params, ':Note')
    form = form_for_node_properties(note, params.keys(), params)
    if form.is_valid():
        set_node_properties_from_params(note, form.cleaned_data)
        note.save()

@command(':Person', 'create', 'notes')
def create_person_note(request, params, node_id):
    person = get_by_id(Person, node_id)
    params = editable_params(params, ':Note')
    form = form_for_node_properties(Note, params.keys(), params)
    if form.is_valid():
        note = create_node_from_params(Note, params)
        person.notes.connect(note)

@command(':Person', 'update')
def update_person(request, params, node_id):
    params = editable_params(params, ':Person')
    person = get_by_id(Person, node_id)
    set_node_properties_from_params(person, params)
    person.save()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Hazard getters for Risk calculators.

An HazardGetter is responsible to get hazard outputs needed by a risk
calculation.
"""

from openquake.engine import logs
from openquake.hazardlib import geo
from openquake.engine.db import models
from django.db import connection

class HazardGetter(object):
    """
    Base abstract class of an Hazard Getter.

    An Hazard Getter is used to query for the closest hazard data for
    each given asset. An Hazard Getter must be pickable such that it
    should be possible to use different strategies (e.g. distributed
    or not, using postgis or not).

    :attr hazard_id:
      the ID of an Hazard output container (e.g.
      :class:`openquake.engine.db.models.HazardCurve`)

    :attr imt: the imt of the hazard considered by the getter

    :attr assets: the assets for which we wants to compute

    :attr max_distance: the maximum distance, in meters, to use
    """
    def __init__(self, hazard_id, imt, assets, max_distance):
        self.hazard_id = hazard_id
        self.imt = imt
        self.assets = assets
        self.max_distance = max_distance

        self._assets_mesh = geo.mesh.Mesh.from_points_list([
            geo.point.Point(asset.site.x, asset.site.y)
            for asset in self.assets])
        self._imt, self._sa_period, self._sa_damping = (
            models.parse_imt(self.imt))
        self.asset_dict = dict((asset.id, asset) for asset in self.assets)
        self._cache = {}

    def get_data(self):
        """
        :returns: an dict mapping ID of
        :class:`openquake.engine.db.models.ExposureData` objects to
        hazard_data (e.g. an array with the poes, or an array with the
        ground motion values). Mind that the returned data could lack
        some assets being filtered out by the ``maximum_distance``
        criteria.

        Subclasses must implement this.
        """
        raise NotImplementedError

    def __call__(self):
        """
        :returns: a tuple with three elements. The first is an array
        of instances of
        :class:`openquake.engine.db.models.ExposureData`, the second
        one is an array with the corresponding hazard data, the third
        one is an array with the IDs of assets that has been filtered
        out as for the ``maximum_distance`` criteria.
        """
        data = self.get_data()

        filtered_asset_ids = data.keys()
        missing_asset_ids = set(self.asset_dict) - set(filtered_asset_ids)

        # FIXME(lp). It might happen that the convex hull contains
        # more assets than the requested ones. At this moment, we just
        # ignore them. We instead log the missing assets, that is the
        # assets that have been filtered out by the maximum distance
        # criteria.
        for missing_asset_id in missing_asset_ids:
            logs.LOG.warn(
                "No hazard has been found for the asset %s",
                self.asset_dict[missing_asset_id].asset_ref)

        return ([self.asset_dict[asset_id] for asset_id in filtered_asset_ids
                 if asset_id in self.asset_dict],
                [data[asset_id] for asset_id in filtered_asset_ids
                 if asset_id in self.asset_dict],
                missing_asset_ids)

class HazardCurveGetterPerAsset(HazardGetter):
    """
    Simple HazardCurve Getter that performs a spatial query for each
    asset.

    :attr imls: the intensity measure levels of the curves we are
    going to get.

    :attr dict _cache: a cache of the computed hazard curve object on
    a per-location basis.
    """

    def __init__(self, hazard_id, imt, assets, max_distance):
        super(HazardCurveGetterPerAsset, self).__init__(
            hazard_id, imt, assets, max_distance)
        self.imls = models.HazardCurve.objects.get(
            pk=self.hazard_id).imls

    def get_data(self):
        """
        Calls ``get_by_site`` for each asset and pack the results as
        requested by the :method:`HazardGetter.get_data` interface.
        """
        return dict(
            [(data[0], data[1][0]) for data in
                [(asset.id, self.get_by_site(asset.site))
                 for asset in self.assets]
                if data[1][1] < self.max_distance])

    def get_by_site(self, site):
        """
        :param site: an instance of
        :class:`django.contrib.gis.geos.point.Point` corresponding to
        the location of an asset.
        """
        if site.wkt in self._cache:
            return self._cache[site.wkt]

        cursor = connection.cursor()

        query = """
        SELECT
            hzrdr.hazard_curve_data.poes,
            min(ST_Distance(location::geography,
                            ST_GeographyFromText(%s), false))
                AS min_distance
        FROM hzrdr.hazard_curve_data
        WHERE hazard_curve_id = %s
        GROUP BY id
        ORDER BY min_distance
        LIMIT 1;"""

        args = (site.wkt, self.hazard_id)

        cursor.execute(query, args)
        poes, distance = cursor.fetchone()

        hazard = zip(self.imls, poes)

        self._cache[site.wkt] = (hazard, distance)

        return hazard, distance

class GroundMotionValuesGetter(HazardGetter):
    """
    Hazard getter for loading ground motion values.
    """

    def get_data(self):
        gmf_collection = models.GmfCollection.objects.get(
            id=self.hazard_id)

        if gmf_collection.output.output_type == "gmf":
            gmf_set_ids = tuple(
                gmf_collection.gmfset_set.values_list('id', flat=True))
        else:
            raise ValueError(
                "Output must be of type `gmf`. "
                "At the moment, we only support computation of loss curves "
                "for a specific logic tree branch.")

        cursor = connection.cursor()

        spectral_filters = ""
        args = (self._imt, gmf_set_ids)

        if self._imt == "SA":
            spectral_filters = "AND sa_period = %s AND sa_damping = %s"
            args += (self._sa_period, self._sa_damping)

        # Query explanation. We need to get for each asset the closest
        # ground motion values for a given logic tree realization and
        # a given imt.

        # We first concatenate ground motion values grouped by
        # location (so for each location we have all the ground motion
        # values found in different gmf_sets, collected in a column
        # called ``allgmvs_arr``).

        # For performance reasons, we help the query by filtering also
        # on a polygon built by dilating the assets extent of the
        # maximum distance

        # Then, we perform a spatial join with the exposure table that
        # is previously filtered by the assets extent, exposure model
        # and taxonomy. We are not filtering with an IN statement on
        # the ids of the assets for perfomance reasons.

        # The ``distinct ON (exposure_data.id)`` combined by the
        # ``ORDER BY ST_Distance`` does the job to select the closest
        # gmvs
        query = """
  SELECT DISTINCT ON (oqmif.exposure_data.id)
  oqmif.exposure_data.id, gmf_table.allgmvs_arr, gmf_table.allrupture_ids
  FROM oqmif.exposure_data JOIN
    (SELECT location,
            array_concat(gmvs ORDER BY gmf_set_id, result_grp_ordinal)
            AS allgmvs_arr,
            array_concat(rupture_ids ORDER BY gmf_set_id, result_grp_ordinal)
            AS allrupture_ids
     FROM hzrdr.gmf
     WHERE imt = %s AND gmf_set_id IN %s {}
     AND location && %s
     GROUP BY location) AS gmf_table
  ON ST_DWithin(oqmif.exposure_data.site, gmf_table.location, %s)
  WHERE oqmif.exposure_data.site && %s
  AND taxonomy = %s AND exposure_model_id = %s
  AND array_length(gmf_table.allgmvs_arr, 1) > 0
  ORDER BY oqmif.exposure_data.id,
           ST_Distance(oqmif.exposure_data.site, gmf_table.location, false)
           """.format(spectral_filters)  # this will fill in the {}

        assets_extent = self._assets_mesh.get_convex_hull()
        args += (assets_extent.dilate(self.max_distance / 1000).wkt,
                 self.max_distance,
                 assets_extent.wkt,
                 self.assets[0].taxonomy,
                 self.assets[0].exposure_model_id)

        cursor.execute(query, args)

        data = cursor.fetchall()

        # The dict is needed by __call__ in order to scan
        # multiple times the data and still get corresponding values.
        # See the return statement of the __call__ method.

        return dict((row[0], [row[1], row[2]]) for row in data)

class GroundMotionScenarioGetterPerAsset(HazardGetter):
    """
    Hazard getter for loading ground motion values from the table
    gmf_scenario. It performs two spatial queries per asset.

    :attr _cache: a cache of the ground motion values on a
    per-location basis
    """

    def get_data(self):
        return dict([(data[0], data[1][0]) for data in
                            [(asset.id, self.get_by_site(asset.site))
                             for asset in self.assets]
                            if data[1][1] < self.max_distance])

    def get_by_site(self, site):
        """
        Return the closest ground motion values to the given location.

        :param site:
            The reference location. The closest ground motion values
            to this location are returned.
        :type site: `django.contrib.gis.geos.point.Point` object
        """

        if site.wkt in self._cache:
            return self._cache[site.wkt]

        cursor = connection.cursor()

        args = ("SRID=4326; %s" % site.wkt, self._imt, self.hazard_id)

        min_dist_query = """-- find the distance of the closest location
        SELECT min(ST_Distance(location, %s, false)) FROM hzrdr.gmf_scenario
        WHERE imt = %s AND output_id = %s"""

        cursor.execute(min_dist_query, args)
        min_dist = cursor.fetchall()[0][0]  # returns only one row
        if min_dist is None:
            raise RuntimeError(
                'Could not find any gmf with IMT=%s '
                'and output_id=%s' % (self._imt, self.hazard_id))

        dilated_dist = min_dist + 0.1  # 0.1 meters = 10 cm

        gmvs_query = """-- return all the gmvs inside the min_dist radius
        SELECT gmvs FROM hzrdr.gmf_scenario
        WHERE %s > ST_Distance(location, %s, false)
        AND imt = %s AND output_id = %s
        """

        cursor.execute(gmvs_query, (dilated_dist,) + args)

        ground_motion_values = sum([row[0] for row in cursor], [])
        self._cache[site.wkt] = ground_motion_values
        return ground_motion_values, min_dist

class GroundMotionScenarioGetter(HazardGetter):
    """
    Hazard getter for loading ground motion values. It uses the same
    approach used in :class:`GroundMotionValuesGetter`.
    """

    def setup(self):
        super(GroundMotionScenarioGetter, self).setup()
        if self._sa_period:
            self._imt = '%s(%s)' % (self._imt, self._sa_period)

    def get_data(self):
        cursor = connection.cursor()

        # See the comment in `GroundMotionValuesGetter.get_data` for
        # an explanation of the query
        query = """
  SELECT DISTINCT ON (oqmif.exposure_data.id) oqmif.exposure_data.id,
         gmf_table.gmvs
  FROM oqmif.exposure_data JOIN (
    SELECT location, gmvs
           FROM hzrdr.gmf_scenario
           WHERE hzrdr.gmf_scenario.imt = %s
           AND hzrdr.gmf_scenario.output_id = %s
           AND hzrdr.gmf_scenario.location && %s) gmf_table
  ON ST_DWithin(oqmif.exposure_data.site, gmf_table.location, %s)
  WHERE oqmif.exposure_data.site && %s
    AND taxonomy = %s AND exposure_model_id = %s
  ORDER BY oqmif.exposure_data.id,
    ST_Distance(oqmif.exposure_data.site, gmf_table.location, false)
           """

        assets_extent = self._assets_mesh.get_convex_hull()
        args = (self._imt, self.hazard_id,
                assets_extent.dilate(self.max_distance / 1000).wkt,
                self.max_distance,
                assets_extent.wkt,
                self.assets[0].taxonomy,
                self.assets[0].exposure_model_id)

        cursor.execute(query, args)

        return dict(cursor.fetchall())
<commit_msg>Revert "removed usage of OrderedDict in hg"<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Hazard getters for Risk calculators.

An HazardGetter is responsible to get hazard outputs needed by a risk
calculation.
"""

from collections import OrderedDict
from openquake.engine import logs
from openquake.hazardlib import geo
from openquake.engine.db import models
from django.db import connection

class HazardGetter(object):
    """
    Base abstract class of an Hazard Getter.

    An Hazard Getter is used to query for the closest hazard data for
    each given asset. An Hazard Getter must be pickable such that it
    should be possible to use different strategies (e.g. distributed
    or not, using postgis or not).

    :attr hazard_id:
      the ID of an Hazard output container (e.g.
      :class:`openquake.engine.db.models.HazardCurve`)

    :attr imt: the imt of the hazard considered by the getter

    :attr assets: the assets for which we wants to compute

    :attr max_distance: the maximum distance, in meters, to use
    """
    def __init__(self, hazard_id, imt, assets, max_distance):
        self.hazard_id = hazard_id
        self.imt = imt
        self.assets = assets
        self.max_distance = max_distance

        self._assets_mesh = geo.mesh.Mesh.from_points_list([
            geo.point.Point(asset.site.x, asset.site.y)
            for asset in self.assets])
        self._imt, self._sa_period, self._sa_damping = (
            models.parse_imt(self.imt))
        self.asset_dict = dict((asset.id, asset) for asset in self.assets)
        self._cache = {}

    def get_data(self):
        """
        :returns: an OrderedDict mapping ID of
        :class:`openquake.engine.db.models.ExposureData` objects to
        hazard_data (e.g. an array with the poes, or an array with the
        ground motion values). Mind that the returned data could lack
        some assets being filtered out by the ``maximum_distance``
        criteria.

        Subclasses must implement this.
        """
        raise NotImplementedError

    def __call__(self):
        """
        :returns: a tuple with three elements. The first is an array
        of instances of
        :class:`openquake.engine.db.models.ExposureData`, the second
        is an array with the corresponding hazard data, the third is
        the array of IDs of assets that has been filtered out by the
        getter by the ``maximum_distance`` criteria.
        """
        data = self.get_data()

        filtered_asset_ids = set(data.keys())
        all_asset_ids = set(self.asset_dict.keys())
        missing_asset_ids = all_asset_ids - filtered_asset_ids
        extra_asset_ids = filtered_asset_ids - all_asset_ids

        # FIXME(lp). It might happens that the convex hull contains
        # more assets than the requested ones. At this moment, we just
        # ignore them.
        if extra_asset_ids:
            logs.LOG.debug("Extra asset have been computed ids: %s" % (
                models.ExposureData.objects.filter(
                    pk__in=extra_asset_ids)))

        for missing_asset_id in missing_asset_ids:
            logs.LOG.warn(
                "No hazard has been found for the asset %s",
                self.asset_dict[missing_asset_id].asset_ref)

        return ([self.asset_dict[asset_id] for asset_id in data
                 if asset_id in self.asset_dict],
                [data[asset_id] for asset_id in data
                 if asset_id in self.asset_dict],
                missing_asset_ids)

class HazardCurveGetterPerAsset(HazardGetter):
    """
    Simple HazardCurve Getter that performs a spatial query for each
    asset.

    :attr imls: the intensity measure levels of the curves we are
    going to get.

    :attr dict _cache: a cache of the computed hazard curve object on
    a per-location basis.
    """

    def __init__(self, hazard_id, imt, assets, max_distance):
        super(HazardCurveGetterPerAsset, self).__init__(
            hazard_id, imt, assets, max_distance)
        self.imls = models.HazardCurve.objects.get(
            pk=self.hazard_id).imls

    def get_data(self):
        """
        Calls ``get_by_site`` for each asset and pack the results as
        requested by the :method:`HazardGetter.get_data` interface.
        """
        return OrderedDict(
            [(data[0], data[1][0]) for data in
                [(asset.id, self.get_by_site(asset.site))
                 for asset in self.assets]
                if data[1][1] < self.max_distance])

    def get_by_site(self, site):
        """
        :param site: an instance of
        :class:`django.contrib.gis.geos.point.Point` corresponding to
        the location of an asset.
        """
        if site.wkt in self._cache:
            return self._cache[site.wkt]

        cursor = connection.cursor()

        query = """
        SELECT
            hzrdr.hazard_curve_data.poes,
            min(ST_Distance(location::geography,
                            ST_GeographyFromText(%s), false))
                AS min_distance
        FROM hzrdr.hazard_curve_data
        WHERE hazard_curve_id = %s
        GROUP BY id
        ORDER BY min_distance
        LIMIT 1;"""

        args = (site.wkt, self.hazard_id)

        cursor.execute(query, args)
        poes, distance = cursor.fetchone()

        hazard = zip(self.imls, poes)

        self._cache[site.wkt] = (hazard, distance)

        return hazard, distance

class GroundMotionValuesGetter(HazardGetter):
    """
    Hazard getter for loading ground motion values.
    """

    def get_data(self):
        gmf_collection = models.GmfCollection.objects.get(
            id=self.hazard_id)

        if gmf_collection.output.output_type == "gmf":
            gmf_set_ids = tuple(
                gmf_collection.gmfset_set.values_list('id', flat=True))
        else:
            raise ValueError(
                "Output must be of type `gmf`. "
                "At the moment, we only support computation of loss curves "
                "for a specific logic tree branch.")

        cursor = connection.cursor()

        spectral_filters = ""
        args = (self._imt, gmf_set_ids)

        if self._imt == "SA":
            spectral_filters = "AND sa_period = %s AND sa_damping = %s"
            args += (self._sa_period, self._sa_damping)

        # Query explanation. We need to get for each asset the closest
        # ground motion values for a given logic tree realization and
        # a given imt.

        # We first concatenate ground motion values grouped by
        # location (so for each location we have all the ground motion
        # values found in different gmf_sets, collected in a column
        # called ``allgmvs_arr``).

        # For performance reasons, we help the query by filtering also
        # on a polygon built by dilating the assets extent of the
        # maximum distance

        # Then, we perform a spatial join with the exposure table that
        # is previously filtered by the assets extent, exposure model
        # and taxonomy. We are not filtering with an IN statement on
        # the ids of the assets for perfomance reasons.

        # The ``distinct ON (exposure_data.id)`` combined by the
        # ``ORDER BY ST_Distance`` does the job to select the closest
        # gmvs
        query = """
  SELECT DISTINCT ON (oqmif.exposure_data.id)
  oqmif.exposure_data.id, gmf_table.allgmvs_arr, gmf_table.allrupture_ids
  FROM oqmif.exposure_data JOIN
    (SELECT location,
            array_concat(gmvs ORDER BY gmf_set_id, result_grp_ordinal)
            AS allgmvs_arr,
            array_concat(rupture_ids ORDER BY gmf_set_id, result_grp_ordinal)
            AS allrupture_ids
     FROM hzrdr.gmf
     WHERE imt = %s AND gmf_set_id IN %s {}
     AND location && %s
     GROUP BY location) AS gmf_table
  ON ST_DWithin(oqmif.exposure_data.site, gmf_table.location, %s)
  WHERE oqmif.exposure_data.site && %s
  AND taxonomy = %s AND exposure_model_id = %s
  AND array_length(gmf_table.allgmvs_arr, 1) > 0
  ORDER BY oqmif.exposure_data.id,
           ST_Distance(oqmif.exposure_data.site, gmf_table.location, false)
           """.format(spectral_filters)  # this will fill in the {}

        assets_extent = self._assets_mesh.get_convex_hull()
        args += (assets_extent.dilate(self.max_distance / 1000).wkt,
                 self.max_distance,
                 assets_extent.wkt,
                 self.assets[0].taxonomy,
                 self.assets[0].exposure_model_id)

        cursor.execute(query, args)

        data = cursor.fetchall()

        # The OrderedDict is needed by __call__ in order to scan
        # multiple times the data and still get corresponding values.
        # See the return statement of the __call__ method.

        return OrderedDict((row[0], [row[1], row[2]]) for row in data)

class GroundMotionScenarioGetterPerAsset(HazardGetter):
    """
    Hazard getter for loading ground motion values from the table
    gmf_scenario. It performs two spatial queries per asset.

    :attr _cache: a cache of the ground motion values on a
    per-location basis
    """

    def get_data(self):
        return OrderedDict([(data[0], data[1][0]) for data in
                            [(asset.id, self.get_by_site(asset.site))
                             for asset in self.assets]
                            if data[1][1] < self.max_distance])

    def get_by_site(self, site):
        """
        Return the closest ground motion values to the given location.

        :param site:
            The reference location. The closest ground motion values
            to this location are returned.
        :type site: `django.contrib.gis.geos.point.Point` object
        """

        if site.wkt in self._cache:
            return self._cache[site.wkt]

        cursor = connection.cursor()

        args = ("SRID=4326; %s" % site.wkt, self._imt, self.hazard_id)

        min_dist_query = """-- find the distance of the closest location
        SELECT min(ST_Distance(location, %s, false)) FROM hzrdr.gmf_scenario
        WHERE imt = %s AND output_id = %s"""

        cursor.execute(min_dist_query, args)
        min_dist = cursor.fetchall()[0][0]  # returns only one row
        if min_dist is None:
            raise RuntimeError(
                'Could not find any gmf with IMT=%s '
                'and output_id=%s' % (self._imt, self.hazard_id))

        dilated_dist = min_dist + 0.1  # 0.1 meters = 10 cm

        gmvs_query = """-- return all the gmvs inside the min_dist radius
        SELECT gmvs FROM hzrdr.gmf_scenario
        WHERE %s > ST_Distance(location, %s, false)
        AND imt = %s AND output_id = %s
        """

        cursor.execute(gmvs_query, (dilated_dist,) + args)

        ground_motion_values = sum([row[0] for row in cursor], [])
        self._cache[site.wkt] = ground_motion_values
        return ground_motion_values, min_dist

class GroundMotionScenarioGetter(HazardGetter):
    """
    Hazard getter for loading ground motion values. It uses the same
    approach used in :class:`GroundMotionValuesGetter`.
    """

    def setup(self):
        super(GroundMotionScenarioGetter, self).setup()
        if self._sa_period:
            self._imt = '%s(%s)' % (self._imt, self._sa_period)

    def get_data(self):
        cursor = connection.cursor()

        # See the comment in `GroundMotionValuesGetter.get_data` for
        # an explanation of the query
        query = """
  SELECT DISTINCT ON (oqmif.exposure_data.id) oqmif.exposure_data.id,
         gmf_table.gmvs
  FROM oqmif.exposure_data JOIN (
    SELECT location, gmvs
           FROM hzrdr.gmf_scenario
           WHERE hzrdr.gmf_scenario.imt = %s
           AND hzrdr.gmf_scenario.output_id = %s
           AND hzrdr.gmf_scenario.location && %s) gmf_table
  ON ST_DWithin(oqmif.exposure_data.site, gmf_table.location, %s)
  WHERE oqmif.exposure_data.site && %s
    AND taxonomy = %s AND exposure_model_id = %s
  ORDER BY oqmif.exposure_data.id,
    ST_Distance(oqmif.exposure_data.site, gmf_table.location, false)
           """

        assets_extent = self._assets_mesh.get_convex_hull()
        args = (self._imt, self.hazard_id,
                assets_extent.dilate(self.max_distance / 1000).wkt,
                self.max_distance,
                assets_extent.wkt,
                self.assets[0].taxonomy,
                self.assets[0].exposure_model_id)

        cursor.execute(query, args)

        return OrderedDict(cursor.fetchall())
<|endoftext|>


<commit_before>"""
interactive_repy.py --- an interactive RepyV2 console

Useful if you are tired of typing "from repyportability..." and
"add_dy_support..." into an interactive Python prompt over and over again.

Relevant Python docs:
* https://docs.python.org/2/library/code.html#code.interact

Thank you for the helpful hints:
* http://stackoverflow.com/questions/13432717/enter-interactive-mode-in-python
* http://stackoverflow.com/questions/11796474/start-interactive-mode-on-a-specific-script-line

"""

# This gives us the interactive prompt
import code

# This links to the Repy API docs
import webbrowser

def repy_api_help():
  webbrowser.open("https://seattle.poly.edu/wiki/RepyV2API")

# These make the Repy API and dy_import_module* functions available
# on the interactive prompt
from repyportability import *
add_dy_support(locals())

# Launch the console!
code.interact(local=locals(), banner="""
Welcome to the Interactive RepyV2 console!

Type ``repy_api_help()'' to open the Repy API documentation at
https://seattle.poly.edu/wiki/RepyV2API in a browser window, and
visit https://github.com/SeattleTestbed for the latest code updates.
""")

# Once they are done, ...
print """Thanks for using the Interactive RepyV2 console. Have a great day!
"""

<commit_msg>Update Interactive Repy Console help URL<commit_after>"""
interactive_repy_console.py --- an interactive RepyV2 console

Useful if you are tired of typing "from repyportability..." and
"add_dy_support..." into an interactive Python prompt over and over again.

Relevant Python docs:
* https://docs.python.org/2/library/code.html#code.interact

Thank you for the helpful hints:
* http://stackoverflow.com/questions/13432717/enter-interactive-mode-in-python
* http://stackoverflow.com/questions/11796474/start-interactive-mode-on-a-specific-script-line

"""

REPY_API_HELP_URL = "https://github.com/SeattleTestbed/docs/blob/master/Programming/RepyV2API.md"

# This gives us the interactive prompt
import code

# This links to the Repy API docs
import webbrowser

def repy_api_help():
  webbrowser.open(REPY_API_HELP_URL)

# These make the Repy API and dy_import_module* functions available
# on the interactive prompt
from repyportability import *
add_dy_support(locals())

# Launch the console!
code.interact(local=locals(), banner="""
Welcome to the Interactive RepyV2 console!

Type ``repy_api_help()'' to open the Repy API documentation at
""" + REPY_API_HELP_URL + """
in a browser window, and visit https://github.com/SeattleTestbed
for the latest code updates.
""")

# Once they are done, ...
print """Thanks for using the Interactive RepyV2 console. Have a great day!
"""

<|endoftext|>


<commit_before>import os.path
import unittest
import numpy
from openquake.hazardlib import geo, imt
from openquake.hazardlib.shakemap import (
    get_shakemap_array, get_sitecol_shakemap, to_gmfs, amplify_ground_shaking,
    spatial_correlation_array, spatial_covariance_array,
    cross_correlation_matrix, cholesky)

aae = numpy.testing.assert_almost_equal
F64 = numpy.float64
imts = [imt.from_string(x) for x in ['PGA', 'SA(0.3)', 'SA(1.0)', 'SA(3.0)']]
imt_dt = numpy.dtype([(str(imt), float) for imt in imts])
shakemap_dt = numpy.dtype([('lon', float), ('lat', float), ('val', imt_dt),
                           ('std', imt_dt), ('vs30', float)])
CDIR = os.path.dirname(__file__)

def mean_gmf(shakemap):
    gmfs = to_gmfs(
        shakemap, crosscorr='cross', site_effects=True, trunclevel=3,
        num_gmfs=10, seed=42)
    return [gmfs[..., i].mean() for i in range(len(imts))]

class ShakemapTestCase(unittest.TestCase):

    def test_gmfs(self):
        f1 = os.path.join(CDIR, 'ghorka_grid.xml')
        f2 = os.path.join(CDIR, 'ghorka_uncertainty.xml')
        array = get_shakemap_array(f1, f2)
        sitecol, shakemap = get_sitecol_shakemap(array)
        n = 4  # number of sites
        self.assertEqual(len(sitecol), n)
        gmf_by_imt = mean_gmf(shakemap)
        aae(gmf_by_imt, [0.0035974, 0.0135442, 0.0296358, 0.0150663])

    def test_amplify(self):
        res = amplify_ground_shaking(T=3.0, vs30=780, gmvs=[0.1, 0.2, 0.3])
        aae(res, [0.09832577, 0.19690711, 0.2958982])

        res = amplify_ground_shaking(T=0.3, vs30=780, gmvs=[0.1, 0.2, 0.3])
        aae(res, [0.09909498, 0.19870543, 0.29922175])

    def test_matrices(self):

        # distance matrix
        lons = numpy.array([84., 84., 84., 85.5, 85.5, 85.5, 87., 87., 87.])
        lats = numpy.array([26., 27.5, 29., 26., 27.5, 29., 26., 27.5, 29.])
        dmatrix = geo.geodetic.distance_matrix(lons, lats)
        aae(dmatrix.sum(), 18539.6131407)

        # spatial correlation
        sca = spatial_correlation_array(dmatrix, imts, 'spatial')
        aae(sca.sum(), 36.000370229)

        # spatial covariance
        std = numpy.array([(0.5, 0.52, 0.64, 0.73)] * 9, imt_dt)  # 9 sites
        scov = spatial_covariance_array([std[n] for n in imt_dt.names], sca)
        aae(scov.sum(), 13.166200147)

        # cross correlation
        ccor = cross_correlation_matrix(imts, 'cross')
        aae(ccor.sum(), 10.49124788)

        # cholesky decomposition
        L = cholesky(scov, ccor)
        self.assertEqual(L.shape, (36, 36))
        aae(L.sum(), 30.5121263)

        # intensity
        val = numpy.array(
            [(5.38409665, 3.9383686, 3.55435415, 4.37692394)] * 9, imt_dt)

        shakemap = numpy.zeros(9, shakemap_dt)  # 9 sites
        shakemap['lon'] = lons
        shakemap['lat'] = lats
        shakemap['vs30'] = numpy.array([301.17] * 9)
        shakemap['val'] = val
        shakemap['std'] = std
        gmfs = to_gmfs(
            shakemap, crosscorr=True, site_effects=False, trunclevel=3,
            num_gmfs=2, seed=42)
        # shape (R, N, E, M)
        aae(gmfs[..., 0].sum(axis=1), [[0.3708301, 0.5671011]])  # PGA

        gmfs = to_gmfs(
            shakemap, crosscorr=True, site_effects=True, trunclevel=3,
            num_gmfs=2, seed=42)
        aae(gmfs[..., 0].sum(axis=1), [[0.2832467, 0.433162]])  # PGA
        aae(gmfs[..., 2].sum(axis=1), [[0.3972117, 0.401503]])  # SA(1.0)
<commit_msg>Fixed test [skip CI]<commit_after>import os.path
import unittest
import numpy
from openquake.hazardlib import geo, imt
from openquake.hazardlib.shakemap import (
    get_shakemap_array, get_sitecol_shakemap, to_gmfs, amplify_ground_shaking,
    spatial_correlation_array, spatial_covariance_array,
    cross_correlation_matrix, cholesky)

aae = numpy.testing.assert_almost_equal
F64 = numpy.float64
imts = [imt.from_string(x) for x in ['PGA', 'SA(0.3)', 'SA(1.0)', 'SA(3.0)']]
imt_dt = numpy.dtype([(str(imt), float) for imt in imts])
shakemap_dt = numpy.dtype([('lon', float), ('lat', float), ('val', imt_dt),
                           ('std', imt_dt), ('vs30', float)])
CDIR = os.path.dirname(__file__)

def mean_gmf(shakemap):
    gmfs = to_gmfs(
        shakemap, crosscorr='cross', site_effects=True, trunclevel=3,
        num_gmfs=10, seed=42)
    return [gmfs[..., i].mean() for i in range(len(imts))]

class ShakemapTestCase(unittest.TestCase):

    def test_gmfs(self):
        f1 = os.path.join(CDIR, 'ghorka_grid.xml')
        f2 = os.path.join(CDIR, 'ghorka_uncertainty.xml')
        array = get_shakemap_array(f1, f2)
        sitecol, shakemap = get_sitecol_shakemap(array)
        n = 4  # number of sites
        self.assertEqual(len(sitecol), n)
        gmf_by_imt = mean_gmf(shakemap)
        aae(gmf_by_imt, [0.0035974, 0.0135442, 0.0296358, 0.0150663])

    def test_amplify(self):
        res = amplify_ground_shaking(T=3.0, vs30=780, gmvs=[0.1, 0.2, 0.3])
        aae(res, [0.09832577, 0.19690711, 0.2958982])

        res = amplify_ground_shaking(T=0.3, vs30=780, gmvs=[0.1, 0.2, 0.3])
        aae(res, [0.09909498, 0.19870543, 0.29922175])

    def test_matrices(self):

        # distance matrix
        lons = numpy.array([84., 84., 84., 85.5, 85.5, 85.5, 87., 87., 87.])
        lats = numpy.array([26., 27.5, 29., 26., 27.5, 29., 26., 27.5, 29.])
        dmatrix = geo.geodetic.distance_matrix(lons, lats)
        aae(dmatrix.sum(), 18539.6131407)

        # spatial correlation
        sca = spatial_correlation_array(dmatrix, imts, 'spatial')
        aae(sca.sum(), 36.000370229)

        # spatial covariance
        std = numpy.array([(0.5, 0.52, 0.64, 0.73)] * 9, imt_dt)  # 9 sites
        scov = spatial_covariance_array([std[n] for n in imt_dt.names], sca)
        aae(scov.sum(), 13.166200147)

        # cross correlation
        ccor = cross_correlation_matrix(imts, 'cross')
        aae(ccor.sum(), 10.49124788)

        # cholesky decomposition
        L = cholesky(scov, ccor)
        self.assertEqual(L.shape, (36, 36))
        aae(L.sum(), 30.5121263)

        # intensity
        val = numpy.array(
            [(5.38409665, 3.9383686, 3.55435415, 4.37692394)] * 9, imt_dt)

        shakemap = numpy.zeros(9, shakemap_dt)  # 9 sites
        shakemap['lon'] = lons
        shakemap['lat'] = lats
        shakemap['vs30'] = numpy.array([301.17] * 9)
        shakemap['val'] = val
        shakemap['std'] = std
        gmfs = to_gmfs(
            shakemap, crosscorr='corr', site_effects=False, trunclevel=3,
            num_gmfs=2, seed=42)
        # shape (R, N, E, M)
        aae(gmfs[..., 0].sum(axis=1), [[0.3708301, 0.5671011]])  # PGA

        gmfs = to_gmfs(
            shakemap, crosscorr='cross', site_effects=True, trunclevel=3,
            num_gmfs=2, seed=42)
        aae(gmfs[..., 0].sum(axis=1), [[0.2832467, 0.433162]])  # PGA
        aae(gmfs[..., 2].sum(axis=1), [[0.3279128, 0.4475009]])  # SA(1.0)
<|endoftext|>


<commit_before>'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
'''
import unittest
from ..data_structures.sarray import SArray
from ..data_structures.sframe import SFrame

class VariantCheckTest(unittest.TestCase):

    def identical(self, reference, b):
    	if type(reference) in [int, long]:
	    self.assertTrue(type(b) in [int, long])
	else:
            self.assertEquals(type(reference), type(b))
        if isinstance(reference, list):
            self.assertEquals(len(reference), len(b))
            for i in range(len(reference)):
                self.identical(reference[i], b[i])
        if isinstance(reference, dict):
            self.assertEqual(sorted(reference.iterkeys()), sorted(b.iterkeys()))
            for i in reference:
                self.identical(reference[i], b[i])
        if isinstance(reference, SArray):
            self.identical(list(reference), list(b))
        if isinstance(reference, SFrame):
            self.identical(list(reference), list(b))

    def variant_turnaround(self, reference, expected_result=None):
        if expected_result is None:
            expected_result = reference
        from ..extensions import _demo_identity
        self.identical(expected_result, _demo_identity(reference))

    def test_variant_check(self):
        sa = SArray([1,2,3,4,5])
        sf = SFrame({'a':sa})
        import array
        self.variant_turnaround(1)
        self.variant_turnaround(1.0)
        self.variant_turnaround(array.array('d', [1.0, 2.0, 3.0]))
         # numeric lists currently converts to array
        self.variant_turnaround([1, 2, 3], array.array('d',[1,.0,2.0,3.0]))
        self.variant_turnaround("abc")
        self.variant_turnaround(["abc", "def"])
        self.variant_turnaround({'a':1,'b':'c'})
        self.variant_turnaround({'a':[1,2,'d'],'b':['a','b','c']})
         # numeric lists currently converts to array
        self.variant_turnaround({'a':[1,2,3],'b':['a','b','c']},
                                {'a':array.array('d',[1,2,3]),'b':['a','b','c']})
        self.variant_turnaround(sa)
        self.variant_turnaround(sf)
        self.variant_turnaround([sa,sf])
        self.variant_turnaround([sa,sa])
        self.variant_turnaround([sf,sf])
        self.variant_turnaround({'a':sa, 'b':sf, 'c':['a','b','c','d']})
        self.variant_turnaround({'a':[sa,sf,{'a':sa,'b':'c'}],
            'b':sf, 'c':['a','b','c','d']})

<commit_msg>tabs --> spaces<commit_after>'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
'''
import unittest
from ..data_structures.sarray import SArray
from ..data_structures.sframe import SFrame

class VariantCheckTest(unittest.TestCase):

    def identical(self, reference, b):
        if type(reference) in [int, long]:
            self.assertTrue(type(b) in [int, long])
        else:
            self.assertEquals(type(reference), type(b))
        if isinstance(reference, list):
            self.assertEquals(len(reference), len(b))
            for i in range(len(reference)):
                self.identical(reference[i], b[i])
        if isinstance(reference, dict):
            self.assertEqual(sorted(reference.iterkeys()), sorted(b.iterkeys()))
            for i in reference:
                self.identical(reference[i], b[i])
        if isinstance(reference, SArray):
            self.identical(list(reference), list(b))
        if isinstance(reference, SFrame):
            self.identical(list(reference), list(b))

    def variant_turnaround(self, reference, expected_result=None):
        if expected_result is None:
            expected_result = reference
        from ..extensions import _demo_identity
        self.identical(expected_result, _demo_identity(reference))

    def test_variant_check(self):
        sa = SArray([1,2,3,4,5])
        sf = SFrame({'a':sa})
        import array
        self.variant_turnaround(1)
        self.variant_turnaround(1.0)
        self.variant_turnaround(array.array('d', [1.0, 2.0, 3.0]))
         # numeric lists currently converts to array
        self.variant_turnaround([1, 2, 3], array.array('d',[1,.0,2.0,3.0]))
        self.variant_turnaround("abc")
        self.variant_turnaround(["abc", "def"])
        self.variant_turnaround({'a':1,'b':'c'})
        self.variant_turnaround({'a':[1,2,'d'],'b':['a','b','c']})
         # numeric lists currently converts to array
        self.variant_turnaround({'a':[1,2,3],'b':['a','b','c']},
                                {'a':array.array('d',[1,2,3]),'b':['a','b','c']})
        self.variant_turnaround(sa)
        self.variant_turnaround(sf)
        self.variant_turnaround([sa,sf])
        self.variant_turnaround([sa,sa])
        self.variant_turnaround([sf,sf])
        self.variant_turnaround({'a':sa, 'b':sf, 'c':['a','b','c','d']})
        self.variant_turnaround({'a':[sa,sf,{'a':sa,'b':'c'}],
            'b':sf, 'c':['a','b','c','d']})

<|endoftext|>


<commit_before>from django.shortcuts import render

# Create your views here.
<commit_msg>Added CRUD views for EntryTag<commit_after>from django.core.urlresolvers import reverse_lazy
from django.shortcuts import render
from django.views.generic import ListView
from django.views.generic.detail import DetailView
from django.views.generic.edit import UpdateView, DeleteView
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseForbidden

from entries.models import EntryTag

class LoginRequiredMixin(object):
    @classmethod
    def as_view(cls, **initkwargs):
        view = super(LoginRequiredMixin, cls).as_view(**initkwargs)
        return login_required(view)

class TagList(LoginRequiredMixin, ListView):
    model = EntryTag

    def get_queryset(self):
        # get all tags
        queryset = EntryTag.objects.all()
        return queryset

class TagRead(LoginRequiredMixin, DetailView):
    model = EntryTag
    template_name_suffix = '_detail'

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        context = self.get_context_data(object=self.object)
        return self.render_to_response(context)

class TagUpdate(LoginRequiredMixin, UpdateView):
    model = EntryTag
    template_name_suffix = '_update_form'
    fields = ['name', 'wiki']

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
	context = self.get_context_data(object=self.object)
        return super(TagUpdate, self).get(request, *args, **kwargs)

class TagDelete(LoginRequiredMixin, DeleteView):
    model = EntryTag
    success_url = reverse_lazy('entries:index')

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        context = self.get_context_data(object=self.object)
        return self.render_to_response(context)
<|endoftext|>


<commit_before>'''Module containing a handful of methods for
aggregating data from markets throughout the world'''

import requests
import json

def dowjonesIndustrialAvg(apikey):
    '''Returns JSON data of the Dow Jones Average.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/BCB/UDJIAD1.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    parsedData = []
    stockData = {}
    for datum in data:
        stockData['name'] = data['name']
        stockData['description'] = data['description']
        stockData['data'] = data['data']
        stockData['code'] = data['code']
    parsedData.append(stockData)
    return parsedData

def snp500IndexPull(apikey):
    '''Returns JSON data of the S&P 500 Index.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/YAHOO/INDEX_GSPC.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    return data

def nasdaqPull(apikey):
    '''Returns JSON data of the Nasdaq Index.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/NASDAQOMX/COMP.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    return data
<commit_msg>Fixing snp500IndexPull method<commit_after>'''Module containing a handful of methods for
aggregating data from markets throughout the world'''

import requests
import json

def dowjonesIndustrialAvg(apikey):
    '''Returns JSON data of the Dow Jones Average.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/BCB/UDJIAD1.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    parsedData = []
    stockData = {}
    for datum in data:
        stockData['name'] = data['name']
        stockData['description'] = data['description']
        stockData['data'] = data['data']
        stockData['code'] = data['code']
    parsedData.append(stockData)
    return parsedData

def snp500IndexPull(apikey):
    '''Returns JSON data of the S&P 500 Index.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/YAHOO/INDEX_GSPC.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    parsedData = []
    stockData = {}
    for datum in data:
        stockData['name'] = data['name']
        stockData['description'] = data['description']
        stockData['data'] = data['data']
        stockData['code'] = data['code']
    parsedData.append(stockData)
    return parsedData

def nasdaqPull(apikey):
    '''Returns JSON data of the Nasdaq Index.'''
    parameters = {'rows' : 1, 'auth_token' : apikey}
    apiurl = 'https://www.quandl.com/api/v1/datasets/NASDAQOMX/COMP.json?'
    req = requests.get(apiurl, params=parameters)
    data = json.loads(req.content)
    return data
<|endoftext|>