<commit_before>import tempfile
import six
import overrides_hack

from .fs_test import FSTestCase

import overrides_hack
import utils
from utils import TestTags, tag_test

from gi.repository import BlockDev, GLib

class NTFSTestCase(FSTestCase):
    def setUp(self):
        if not self.ntfs_avail:
            self.skipTest("skipping NTFS: not available")

        super(NTFSTestCase, self).setUp()

        self.mount_dir = tempfile.mkdtemp(prefix="libblockdev.", suffix="ntfs_test")

class NTFSSetLabel(NTFSTestCase):
    def test_ntfs_set_label(self):
        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "TEST_LABEL")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "TEST_LABEL")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "TEST_LABEL2")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "TEST_LABEL2")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "")

        succ = BlockDev.fs_ntfs_check_label("TEST_LABEL")
        self.assertTrue(succ)

        with six.assertRaisesRegex(self, GLib.GError, "at most 128 characters long."):
            BlockDev.fs_ntfs_check_label(129 * "a")

class NTFSSetUUID(NTFSTestCase):

    test_uuid = "54E1629A44FD724B"

    def test_ntfs_set_uuid(self):
        """Verify that it is possible to set UUID of an ntfs file system"""
        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        succ = BlockDev.fs_ntfs_set_uuid(self.loop_dev, self.test_uuid)
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.uuid, self.test_uuid)

        # no uuid -> random
        succ = BlockDev.fs_ntfs_set_uuid(self.loop_dev, None)
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertNotEqual(fi.uuid, "")
        self.assertNotEqual(fi.uuid, self.test_uuid)

        succ = BlockDev.fs_ntfs_check_uuid(self.test_uuid)
        self.assertTrue(succ)

        with six.assertRaisesRegex(self, GLib.GError, "8 or 16 characters long"):
            BlockDev.fs_ntfs_check_uuid(9 * "a")

        with six.assertRaisesRegex(self, GLib.GError, "must be a hexadecimal number"):
            BlockDev.fs_ntfs_check_uuid(16 * "z")
<commit_msg>tests: Add missing NTFS test cases<commit_after>import tempfile
import six
import overrides_hack

from .fs_test import FSTestCase, mounted

import overrides_hack
import utils
from utils import TestTags, tag_test

from gi.repository import BlockDev, GLib

class NTFSTestCase(FSTestCase):
    def setUp(self):
        if not self.ntfs_avail:
            self.skipTest("skipping NTFS: not available")

        super(NTFSTestCase, self).setUp()

        self.mount_dir = tempfile.mkdtemp(prefix="libblockdev.", suffix="ntfs_test")

class NTFSTestMkfs(NTFSTestCase):
    def test_ntfs_mkfs(self):
        """Verify that it is possible to create a new NTFS file system"""

        with self.assertRaises(GLib.GError):
            BlockDev.fs_ntfs_mkfs("/non/existing/device", None)

        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev)
        self.assertTrue(succ)

        # just try if we can mount the file system
        with mounted(self.loop_dev, self.mount_dir):
            pass

        # check the fstype
        fstype = BlockDev.fs_get_fstype(self.loop_dev)
        self.assertEqual(fstype, "ntfs")

        BlockDev.fs_wipe(self.loop_dev, True)

class NTFSMkfsWithLabel(NTFSTestCase):
    def test_ntfs_mkfs_with_label(self):
        """Verify that it is possible to create an NTFS file system with label"""

        ea = BlockDev.ExtraArg.new("-L", "test_label")
        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, [ea])
        self.assertTrue(succ)

        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "test_label")

class NTFSTestWipe(NTFSTestCase):
    def test_ntfs_wipe(self):
        """Verify that it is possible to wipe an NTFS file system"""

        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        succ = BlockDev.fs_ntfs_wipe(self.loop_dev)
        self.assertTrue(succ)

        # already wiped, should fail this time
        with self.assertRaises(GLib.GError):
            BlockDev.fs_ntfs_wipe(self.loop_dev)

        utils.run("pvcreate -ff -y %s >/dev/null" % self.loop_dev)

        # LVM PV signature, not an ntfs file system
        with self.assertRaises(GLib.GError):
            BlockDev.fs_ntfs_wipe(self.loop_dev)

        BlockDev.fs_wipe(self.loop_dev, True)

        utils.run("mkfs.ext2 -F %s >/dev/null 2>&1" % self.loop_dev)

        # ext2, not an ntfs file system
        with self.assertRaises(GLib.GError):
            BlockDev.fs_ntfs_wipe(self.loop_dev)

        BlockDev.fs_wipe(self.loop_dev, True)

class NTFSGetInfo(NTFSTestCase):
    def test_ntfs_get_info(self):
        """Verify that it is possible to get info about an NTFS file system"""

        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "")
        # should be an non-empty string
        self.assertTrue(fi.uuid)
        self.assertGreater(fi.size, 0)
        self.assertLess(fi.free_space, fi.size)

class NTFSResize(NTFSTestCase):
    def test_ntfs_resize(self):
        """Verify that it is possible to resize an NTFS file system"""

        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        succ = BlockDev.fs_ntfs_repair(self.loop_dev)
        self.assertTrue(succ)

        # shrink
        succ = BlockDev.fs_ntfs_resize(self.loop_dev, 80 * 1024**2)
        self.assertTrue(succ)

        succ = BlockDev.fs_ntfs_repair(self.loop_dev)
        self.assertTrue(succ)

        # resize to maximum size
        succ = BlockDev.fs_ntfs_resize(self.loop_dev, 0)
        self.assertTrue(succ)

class NTFSSetLabel(NTFSTestCase):
    def test_ntfs_set_label(self):
        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "TEST_LABEL")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "TEST_LABEL")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "TEST_LABEL2")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "TEST_LABEL2")

        succ = BlockDev.fs_ntfs_set_label(self.loop_dev, "")
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.label, "")

        succ = BlockDev.fs_ntfs_check_label("TEST_LABEL")
        self.assertTrue(succ)

        with six.assertRaisesRegex(self, GLib.GError, "at most 128 characters long."):
            BlockDev.fs_ntfs_check_label(129 * "a")

class NTFSSetUUID(NTFSTestCase):

    test_uuid = "54E1629A44FD724B"

    def test_ntfs_set_uuid(self):
        """Verify that it is possible to set UUID of an ntfs file system"""
        succ = BlockDev.fs_ntfs_mkfs(self.loop_dev, None)
        self.assertTrue(succ)

        succ = BlockDev.fs_ntfs_set_uuid(self.loop_dev, self.test_uuid)
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertEqual(fi.uuid, self.test_uuid)

        # no uuid -> random
        succ = BlockDev.fs_ntfs_set_uuid(self.loop_dev, None)
        self.assertTrue(succ)
        fi = BlockDev.fs_ntfs_get_info(self.loop_dev)
        self.assertTrue(fi)
        self.assertNotEqual(fi.uuid, "")
        self.assertNotEqual(fi.uuid, self.test_uuid)

        succ = BlockDev.fs_ntfs_check_uuid(self.test_uuid)
        self.assertTrue(succ)

        with six.assertRaisesRegex(self, GLib.GError, "8 or 16 characters long"):
            BlockDev.fs_ntfs_check_uuid(9 * "a")

        with six.assertRaisesRegex(self, GLib.GError, "must be a hexadecimal number"):
            BlockDev.fs_ntfs_check_uuid(16 * "z")
<|endoftext|>


<commit_before># Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the classical PSHA risk calculator.
"""
import collections
import itertools
import numpy

from django import db

from openquake.hazardlib.geo import mesh
from openquake.risklib import scientific, workflows

from openquake.engine.calculators import post_processing
from openquake.engine.calculators.risk import (
    base, hazard_getters, validation, writers)
from openquake.engine.db import models
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor
from openquake.engine.utils import tasks

@tasks.oqtask
def event_based(job_id, risk_model, getters, outputdict, params):
    """
    Celery task for the event based risk calculator.

    :param job_id: the id of the current
        :class:`openquake.engine.db.models.OqJob`
    :param risk_model:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getters:
      A list of callable hazard getters
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (e.g. a LossCurve)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
    :returns:
      A dictionary {loss_type: event_loss_table}
    """
    monitor = EnginePerformanceMonitor(
        None, job_id, event_based, tracing=True)

    # Do the job in other functions, such that they can be unit tested
    # without the celery machinery
    with db.transaction.commit_on_success(using='job_init'):
        return do_event_based(risk_model, getters, outputdict, params, monitor)

def do_event_based(risk_model, getters, outputdict, params, monitor):
    """
    See `event_based` for a description of the params

    :returns: the event loss table generated by risk_model
    """
    outputs_per_loss_type = risk_model.compute_outputs(
        getters, monitor.copy('getting data'))
    stats_per_loss_type = risk_model.compute_stats(
        outputs_per_loss_type, params.quantiles, post_processing)

    # NB: event_loss_table is a dictionary (loss_type, out_id) -> loss,
    # out_id can be None, and it that case it stores the statistics
    event_loss_table = {}
    rupture_ids = getters[0].rupture_ids  # in the right order

    # save outputs and stats and populate the event loss table
    for loss_type, outputs in outputs_per_loss_type.iteritems():
        for out in outputs:
            if params.sites_disagg:
                with monitor.copy('disaggregating results'):
                    ruptures = [models.SESRupture.objects.get(pk=rid)
                                for rid in rupture_ids]
                    disagg_outputs = disaggregate(
                        out.output, [r.rupture for r in ruptures], params)
            else:
                disagg_outputs = None

            with monitor.copy('saving individual risk'):
                save_individual_outputs(
                    outputdict.with_args(hazard_output_id=out.hid,
                                         loss_type=loss_type),
                    out.output, disagg_outputs, params)

            event_loss_table[loss_type, out.hid] = out.output.event_loss_table

        stats = stats_per_loss_type[loss_type]
        if stats is not None:
            with monitor.copy('saving risk statistics'):
                save_statistical_output(
                    outputdict.with_args(
                        hazard_output_id=None, loss_type=loss_type),
                    stats, params)
            event_loss_table[loss_type, None] = stats.event_loss_table

    return event_loss_table

def save_individual_outputs(outputdict, outputs, disagg_outputs, params):
    """
    Save loss curves, loss maps and loss fractions associated with a
    calculation unit

    :param outputdict:
        a :class:`openquake.engine.calculators.risk.writers.OutputDict`
        instance holding the reference to the output container objects
    :param outputs:
        a :class:`openquake.risklib.workflows.ProbabilisticEventBased.Output`
        holding the output data for a calculation unit
    :param disagg_outputs:
        a :class:`.DisaggregationOutputs` holding the disaggreation
        output data for a calculation unit
    :param params:
        a :class:`openquake.engine.calculators.risk.base.CalcParams`
        holding the parameters for this calculation
    """

    outputdict.write(
        outputs.assets,
        (outputs.loss_curves, outputs.average_losses, outputs.stddev_losses),
        output_type="event_loss_curve")

    outputdict.write_all(
        "poe", params.conditional_loss_poes,
        outputs.loss_maps,
        outputs.assets,
        output_type="loss_map")

    if disagg_outputs is not None:
        # FIXME. We should avoid synthetizing the generator
        assets = list(disagg_outputs.assets_disagg)
        outputdict.write(
            assets,
            disagg_outputs.magnitude_distance,
            disagg_outputs.fractions,
            output_type="loss_fraction",
            variable="magnitude_distance")
        outputdict.write(
            assets,
            disagg_outputs.coordinate, disagg_outputs.fractions,
            output_type="loss_fraction",
            variable="coordinate")

    if outputs.insured_curves is not None:
        outputdict.write(
            outputs.assets,
            (outputs.insured_curves, outputs.average_insured_losses,
             outputs.stddev_insured_losses),
            output_type="event_loss_curve", insured=True)

def save_statistical_output(outputdict, stats, params):
    """
    Save statistical outputs (mean and quantile loss curves, mean and
    quantile loss maps) for the calculation.

    :param outputdict:
        a :class:`openquake.engine.calculators.risk.writers.OutputDict`
        instance holding the reference to the output container objects
    :param stats:
        :class:
        `openquake.risklib.workflows.ProbabilisticEventBased.StatisticalOutput`
        holding the statistical output data
    :param params:
        a :class:`openquake.engine.calculators.risk.base.CalcParams`
        holding the parameters for this calculation
    """

    outputdict.write(
        stats.assets, (stats.mean_curves, stats.mean_average_losses),
        output_type="loss_curve", statistics="mean")

    outputdict.write_all(
        "poe", params.conditional_loss_poes, stats.mean_maps,
        stats.assets, output_type="loss_map", statistics="mean")

    # quantile curves and maps
    outputdict.write_all(
        "quantile", params.quantiles,
        [(c, a) for c, a in itertools.izip(stats.quantile_curves,
                                           stats.quantile_average_losses)],
        stats.assets, output_type="loss_curve", statistics="quantile")

    if params.quantiles:
        for quantile, maps in zip(params.quantiles, stats.quantile_maps):
            outputdict.write_all(
                "poe", params.conditional_loss_poes, maps,
                stats.assets, output_type="loss_map",
                statistics="quantile", quantile=quantile)

    # mean and quantile insured curves
    if stats.mean_insured_curves is not None:
        outputdict.write(
            stats.assets, (stats.mean_insured_curves,
                           stats.mean_average_insured_losses),
            output_type="loss_curve", statistics="mean", insured=True)

        outputdict.write_all(
            "quantile", params.quantiles,
            [(c, a) for c, a in itertools.izip(
                stats.quantile_insured_curves,
                stats.quantile_average_insured_losses)],
            stats.assets,
            output_type="loss_curve", statistics="quantile", insured=True)

class DisaggregationOutputs(object):
    def __init__(self, assets_disagg, magnitude_distance,
                 coordinate, fractions):
        self.assets_disagg = assets_disagg
        self.magnitude_distance = magnitude_distance
        self.coordinate = coordinate
        self.fractions = fractions

def disaggregate(outputs, ruptures, params):
    """
    Compute disaggregation outputs given the individual `outputs` and `params`

    :param outputs:
      an instance of
      :class:`openquake.risklib.workflows.ProbabilisticEventBased.Output`
    :param params:
      an instance of :class:`..base.CalcParams`
    :param list rupture_ids:
      a list of :class:`openquake.engine.db.models.SESRupture` IDs
    :returns:
      an instance of :class:`DisaggregationOutputs`
    """
    def disaggregate_site(site, loss_ratios):
        for fraction, rupture in zip(loss_ratios, ruptures):
            s = rupture.surface
            m = mesh.Mesh(numpy.array([site.x]), numpy.array([site.y]), None)

            mag = numpy.floor(rupture.magnitude / params.mag_bin_width)
            dist = numpy.floor(
                s.get_joyner_boore_distance(m))[0] / params.distance_bin_width

            closest_point = iter(s.get_closest_points(m)).next()
            lon = closest_point.longitude / params.coordinate_bin_width
            lat = closest_point.latitude / params.coordinate_bin_width

            yield "%d,%d" % (mag, dist), "%d,%d" % (lon, lat), fraction

    assets_disagg = []
    disagg_matrix = []

    for asset, losses in zip(outputs.assets, outputs.loss_matrix):
        if asset.site in params.sites_disagg:
            disagg_matrix.extend(list(disaggregate_site(asset.site, losses)))

            # FIXME. the functions in
            # openquake.engine.calculators.risk.writers requires an
            # asset per each row in the disaggregation matrix. To this
            # aim, we repeat the assets that will be passed to such
            # functions
            assets_disagg = itertools.chain(
                assets_disagg,
                itertools.repeat(asset, len(ruptures)))

    if assets_disagg:
        magnitudes, coordinates, fractions = zip(*disagg_matrix)
    else:
        magnitudes, coordinates, fractions = [], [], []

    return DisaggregationOutputs(
        assets_disagg, magnitudes, coordinates, fractions)

class EventBasedRiskCalculator(base.RiskCalculator):
    """
    Probabilistic Event Based PSHA risk calculator. Computes loss
    curves, loss maps, aggregate losses and insured losses for a given
    set of assets.
    """

    #: The core calculation celery task function
    core_calc_task = event_based

    # FIXME(lp). Validate sites_disagg to ensure non-empty outputs
    validators = base.RiskCalculator.validators + [
        validation.RequireEventBasedHazard,
        validation.ExposureHasInsuranceBounds]

    output_builders = [writers.EventLossCurveMapBuilder,
                       writers.LossFractionBuilder]
    getter_class = hazard_getters.GroundMotionValuesGetter

    def __init__(self, job):
        super(EventBasedRiskCalculator, self).__init__(job)
        # accumulator for the event loss tables
        self.acc = collections.defaultdict(collections.Counter)

    def agg_result(self, acc, event_loss_table):
        """
        Updates the event loss table
        """
        newdict = acc.copy()
        for (loss_type, out_id), counter in event_loss_table.iteritems():
            try:
                c = newdict[loss_type, out_id]
            except KeyError:
                pass
            else:
                newdict[loss_type, out_id] = c + counter
        return newdict

    def post_process(self):
        """
          Compute aggregate loss curves and event loss tables
        """
        with EnginePerformanceMonitor('post processing', self.job.id):

            time_span, tses = self.hazard_times()
            for (loss_type, out_id), event_loss_table in self.acc.items():
                if out_id:  # values for individual realizations
                    hazard_output = models.Output.objects.get(pk=out_id)

                    event_loss = models.EventLoss.objects.create(
                        output=models.Output.objects.create_output(
                            self.job,
                            "Event Loss Table. type=%s, hazard=%s" % (
                                loss_type, hazard_output.id),
                            "event_loss"),
                        loss_type=loss_type,
                        hazard_output=hazard_output)
                    inserter = writer.CacheInserter(models.EventLossData, 9999)
                    if isinstance(hazard_output.output_container,
                                  models.SESCollection):
                        ses_coll = hazard_output.output_container
                    else:  # extract the SES collection from the Gmf
                        ses_coll = models.SESCollection.objects.get(
                            lt_model=hazard_output.output_container.
                            lt_realization.lt_model)
                    rupture_ids = ses_coll.get_ruptures().values_list(
                        'id', flat=True)
                    for rupture_id in rupture_ids:
                        if rupture_id in event_loss_table:
                            inserter.add(
                                models.EventLossData(
                                    event_loss_id=event_loss.id,
                                    rupture_id=rupture_id,
                                    aggregate_loss=event_loss_table[
                                        rupture_id]))
                    inserter.flush()

                    aggregate_losses = [
                        event_loss_table[rupture_id]
                        for rupture_id in rupture_ids
                        if rupture_id in event_loss_table]

                    if aggregate_losses:
                        aggregate_loss_losses, aggregate_loss_poes = (
                            scientific.event_based(
                                aggregate_losses, tses=tses,
                                time_span=time_span,
                                curve_resolution=self.rc.loss_curve_resolution
                            ))

                        models.AggregateLossCurveData.objects.create(
                            loss_curve=models.LossCurve.objects.create(
                                aggregate=True, insured=False,
                                hazard_output=hazard_output,
                                loss_type=loss_type,
                                output=models.Output.objects.create_output(
                                    self.job,
                                    "aggregate loss curves. "
                                    "loss_type=%s hazard=%s" % (
                                        loss_type, hazard_output),
                                    "agg_loss_curve")),
                            losses=aggregate_loss_losses,
                            poes=aggregate_loss_poes,
                            average_loss=scientific.average_loss(
                                aggregate_loss_losses, aggregate_loss_poes),
                            stddev_loss=numpy.std(aggregate_losses))

    def get_workflow(self, vulnerability_functions):
        time_span, tses = self.hazard_times()
        return workflows.ProbabilisticEventBased(
            vulnerability_functions,
            time_span, tses,
            self.rc.loss_curve_resolution,
            self.rc.conditional_loss_poes,
            self.rc.insured_losses)

    def hazard_times(self):
        """
        Return the hazard investigation time related to the ground
        motion field and the so-called time representative of the
        stochastic event set
        """
        return (self.rc.investigation_time,
                self.hc.ses_per_logic_tree_path * self.hc.investigation_time)

    @property
    def calculator_parameters(self):
        """
        Calculator specific parameters
        """
        return base.make_calc_params(
            conditional_loss_poes=self.rc.conditional_loss_poes or [],
            quantiles=self.rc.quantile_loss_curves or [],
            insured_losses=self.rc.insured_losses,
            sites_disagg=self.rc.sites_disagg or [],
            mag_bin_width=self.rc.mag_bin_width,
            distance_bin_width=self.rc.distance_bin_width,
            coordinate_bin_width=self.rc.coordinate_bin_width)
<commit_msg>keep in memory the loss_matrix only when doing disaggregation<commit_after># Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the classical PSHA risk calculator.
"""
import collections
import itertools
import numpy

from django import db

from openquake.hazardlib.geo import mesh
from openquake.risklib import scientific, workflows

from openquake.engine.calculators import post_processing
from openquake.engine.calculators.risk import (
    base, hazard_getters, validation, writers)
from openquake.engine.db import models
from openquake.engine import writer
from openquake.engine.performance import EnginePerformanceMonitor
from openquake.engine.utils import tasks

@tasks.oqtask
def event_based(job_id, risk_model, getters, outputdict, params):
    """
    Celery task for the event based risk calculator.

    :param job_id: the id of the current
        :class:`openquake.engine.db.models.OqJob`
    :param risk_model:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getters:
      A list of callable hazard getters
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (e.g. a LossCurve)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
    :returns:
      A dictionary {loss_type: event_loss_table}
    """
    monitor = EnginePerformanceMonitor(
        None, job_id, event_based, tracing=True)

    # Do the job in other functions, such that they can be unit tested
    # without the celery machinery
    with db.transaction.commit_on_success(using='job_init'):
        return do_event_based(risk_model, getters, outputdict, params, monitor)

def do_event_based(risk_model, getters, outputdict, params, monitor):
    """
    See `event_based` for a description of the params

    :returns: the event loss table generated by risk_model
    """
    # keep in memory the loss_matrix only when doing disaggregation
    risk_model.workflow.return_loss_matrix = bool(params.sites_disagg)
    outputs_per_loss_type = risk_model.compute_outputs(
        getters, monitor.copy('getting data'))
    stats_per_loss_type = risk_model.compute_stats(
        outputs_per_loss_type, params.quantiles, post_processing)

    # NB: event_loss_table is a dictionary (loss_type, out_id) -> loss,
    # out_id can be None, and it that case it stores the statistics
    event_loss_table = {}
    rupture_ids = getters[0].rupture_ids  # in the right order

    # save outputs and stats and populate the event loss table
    for loss_type, outputs in outputs_per_loss_type.iteritems():
        for out in outputs:
            if params.sites_disagg:
                with monitor.copy('disaggregating results'):
                    ruptures = [models.SESRupture.objects.get(pk=rid)
                                for rid in rupture_ids]
                    disagg_outputs = disaggregate(
                        out.output, [r.rupture for r in ruptures], params)
            else:
                disagg_outputs = None

            with monitor.copy('saving individual risk'):
                save_individual_outputs(
                    outputdict.with_args(hazard_output_id=out.hid,
                                         loss_type=loss_type),
                    out.output, disagg_outputs, params)

            event_loss_table[loss_type, out.hid] = out.output.event_loss_table

        stats = stats_per_loss_type[loss_type]
        if stats is not None:
            with monitor.copy('saving risk statistics'):
                save_statistical_output(
                    outputdict.with_args(
                        hazard_output_id=None, loss_type=loss_type),
                    stats, params)
            event_loss_table[loss_type, None] = stats.event_loss_table

    return event_loss_table

def save_individual_outputs(outputdict, outputs, disagg_outputs, params):
    """
    Save loss curves, loss maps and loss fractions associated with a
    calculation unit

    :param outputdict:
        a :class:`openquake.engine.calculators.risk.writers.OutputDict`
        instance holding the reference to the output container objects
    :param outputs:
        a :class:`openquake.risklib.workflows.ProbabilisticEventBased.Output`
        holding the output data for a calculation unit
    :param disagg_outputs:
        a :class:`.DisaggregationOutputs` holding the disaggreation
        output data for a calculation unit
    :param params:
        a :class:`openquake.engine.calculators.risk.base.CalcParams`
        holding the parameters for this calculation
    """

    outputdict.write(
        outputs.assets,
        (outputs.loss_curves, outputs.average_losses, outputs.stddev_losses),
        output_type="event_loss_curve")

    outputdict.write_all(
        "poe", params.conditional_loss_poes,
        outputs.loss_maps,
        outputs.assets,
        output_type="loss_map")

    if disagg_outputs is not None:
        # FIXME. We should avoid synthetizing the generator
        assets = list(disagg_outputs.assets_disagg)
        outputdict.write(
            assets,
            disagg_outputs.magnitude_distance,
            disagg_outputs.fractions,
            output_type="loss_fraction",
            variable="magnitude_distance")
        outputdict.write(
            assets,
            disagg_outputs.coordinate, disagg_outputs.fractions,
            output_type="loss_fraction",
            variable="coordinate")

    if outputs.insured_curves is not None:
        outputdict.write(
            outputs.assets,
            (outputs.insured_curves, outputs.average_insured_losses,
             outputs.stddev_insured_losses),
            output_type="event_loss_curve", insured=True)

def save_statistical_output(outputdict, stats, params):
    """
    Save statistical outputs (mean and quantile loss curves, mean and
    quantile loss maps) for the calculation.

    :param outputdict:
        a :class:`openquake.engine.calculators.risk.writers.OutputDict`
        instance holding the reference to the output container objects
    :param stats:
        :class:
        `openquake.risklib.workflows.ProbabilisticEventBased.StatisticalOutput`
        holding the statistical output data
    :param params:
        a :class:`openquake.engine.calculators.risk.base.CalcParams`
        holding the parameters for this calculation
    """

    outputdict.write(
        stats.assets, (stats.mean_curves, stats.mean_average_losses),
        output_type="loss_curve", statistics="mean")

    outputdict.write_all(
        "poe", params.conditional_loss_poes, stats.mean_maps,
        stats.assets, output_type="loss_map", statistics="mean")

    # quantile curves and maps
    outputdict.write_all(
        "quantile", params.quantiles,
        [(c, a) for c, a in itertools.izip(stats.quantile_curves,
                                           stats.quantile_average_losses)],
        stats.assets, output_type="loss_curve", statistics="quantile")

    if params.quantiles:
        for quantile, maps in zip(params.quantiles, stats.quantile_maps):
            outputdict.write_all(
                "poe", params.conditional_loss_poes, maps,
                stats.assets, output_type="loss_map",
                statistics="quantile", quantile=quantile)

    # mean and quantile insured curves
    if stats.mean_insured_curves is not None:
        outputdict.write(
            stats.assets, (stats.mean_insured_curves,
                           stats.mean_average_insured_losses),
            output_type="loss_curve", statistics="mean", insured=True)

        outputdict.write_all(
            "quantile", params.quantiles,
            [(c, a) for c, a in itertools.izip(
                stats.quantile_insured_curves,
                stats.quantile_average_insured_losses)],
            stats.assets,
            output_type="loss_curve", statistics="quantile", insured=True)

class DisaggregationOutputs(object):
    def __init__(self, assets_disagg, magnitude_distance,
                 coordinate, fractions):
        self.assets_disagg = assets_disagg
        self.magnitude_distance = magnitude_distance
        self.coordinate = coordinate
        self.fractions = fractions

def disaggregate(outputs, ruptures, params):
    """
    Compute disaggregation outputs given the individual `outputs` and `params`

    :param outputs:
      an instance of
      :class:`openquake.risklib.workflows.ProbabilisticEventBased.Output`
    :param params:
      an instance of :class:`..base.CalcParams`
    :param list rupture_ids:
      a list of :class:`openquake.engine.db.models.SESRupture` IDs
    :returns:
      an instance of :class:`DisaggregationOutputs`
    """
    def disaggregate_site(site, loss_ratios):
        for fraction, rupture in zip(loss_ratios, ruptures):
            s = rupture.surface
            m = mesh.Mesh(numpy.array([site.x]), numpy.array([site.y]), None)

            mag = numpy.floor(rupture.magnitude / params.mag_bin_width)
            dist = numpy.floor(
                s.get_joyner_boore_distance(m))[0] / params.distance_bin_width

            closest_point = iter(s.get_closest_points(m)).next()
            lon = closest_point.longitude / params.coordinate_bin_width
            lat = closest_point.latitude / params.coordinate_bin_width

            yield "%d,%d" % (mag, dist), "%d,%d" % (lon, lat), fraction

    assets_disagg = []
    disagg_matrix = []

    for asset, losses in zip(outputs.assets, outputs.loss_matrix):
        if asset.site in params.sites_disagg:
            disagg_matrix.extend(list(disaggregate_site(asset.site, losses)))

            # FIXME. the functions in
            # openquake.engine.calculators.risk.writers requires an
            # asset per each row in the disaggregation matrix. To this
            # aim, we repeat the assets that will be passed to such
            # functions
            assets_disagg = itertools.chain(
                assets_disagg,
                itertools.repeat(asset, len(ruptures)))

    if assets_disagg:
        magnitudes, coordinates, fractions = zip(*disagg_matrix)
    else:
        magnitudes, coordinates, fractions = [], [], []

    return DisaggregationOutputs(
        assets_disagg, magnitudes, coordinates, fractions)

class EventBasedRiskCalculator(base.RiskCalculator):
    """
    Probabilistic Event Based PSHA risk calculator. Computes loss
    curves, loss maps, aggregate losses and insured losses for a given
    set of assets.
    """

    #: The core calculation celery task function
    core_calc_task = event_based

    # FIXME(lp). Validate sites_disagg to ensure non-empty outputs
    validators = base.RiskCalculator.validators + [
        validation.RequireEventBasedHazard,
        validation.ExposureHasInsuranceBounds]

    output_builders = [writers.EventLossCurveMapBuilder,
                       writers.LossFractionBuilder]
    getter_class = hazard_getters.GroundMotionValuesGetter

    def __init__(self, job):
        super(EventBasedRiskCalculator, self).__init__(job)
        # accumulator for the event loss tables
        self.acc = collections.defaultdict(collections.Counter)

    def agg_result(self, acc, event_loss_table):
        """
        Updates the event loss table
        """
        newdict = acc.copy()
        for (loss_type, out_id), counter in event_loss_table.iteritems():
            try:
                c = newdict[loss_type, out_id]
            except KeyError:
                pass
            else:
                newdict[loss_type, out_id] = c + counter
        return newdict

    def post_process(self):
        """
          Compute aggregate loss curves and event loss tables
        """
        with EnginePerformanceMonitor('post processing', self.job.id):

            time_span, tses = self.hazard_times()
            for (loss_type, out_id), event_loss_table in self.acc.items():
                if out_id:  # values for individual realizations
                    hazard_output = models.Output.objects.get(pk=out_id)

                    event_loss = models.EventLoss.objects.create(
                        output=models.Output.objects.create_output(
                            self.job,
                            "Event Loss Table. type=%s, hazard=%s" % (
                                loss_type, hazard_output.id),
                            "event_loss"),
                        loss_type=loss_type,
                        hazard_output=hazard_output)
                    inserter = writer.CacheInserter(models.EventLossData, 9999)
                    if isinstance(hazard_output.output_container,
                                  models.SESCollection):
                        ses_coll = hazard_output.output_container
                    else:  # extract the SES collection from the Gmf
                        ses_coll = models.SESCollection.objects.get(
                            lt_model=hazard_output.output_container.
                            lt_realization.lt_model)
                    rupture_ids = ses_coll.get_ruptures().values_list(
                        'id', flat=True)
                    for rupture_id in rupture_ids:
                        if rupture_id in event_loss_table:
                            inserter.add(
                                models.EventLossData(
                                    event_loss_id=event_loss.id,
                                    rupture_id=rupture_id,
                                    aggregate_loss=event_loss_table[
                                        rupture_id]))
                    inserter.flush()

                    aggregate_losses = [
                        event_loss_table[rupture_id]
                        for rupture_id in rupture_ids
                        if rupture_id in event_loss_table]

                    if aggregate_losses:
                        aggregate_loss_losses, aggregate_loss_poes = (
                            scientific.event_based(
                                aggregate_losses, tses=tses,
                                time_span=time_span,
                                curve_resolution=self.rc.loss_curve_resolution
                            ))

                        models.AggregateLossCurveData.objects.create(
                            loss_curve=models.LossCurve.objects.create(
                                aggregate=True, insured=False,
                                hazard_output=hazard_output,
                                loss_type=loss_type,
                                output=models.Output.objects.create_output(
                                    self.job,
                                    "aggregate loss curves. "
                                    "loss_type=%s hazard=%s" % (
                                        loss_type, hazard_output),
                                    "agg_loss_curve")),
                            losses=aggregate_loss_losses,
                            poes=aggregate_loss_poes,
                            average_loss=scientific.average_loss(
                                aggregate_loss_losses, aggregate_loss_poes),
                            stddev_loss=numpy.std(aggregate_losses))

    def get_workflow(self, vulnerability_functions):
        time_span, tses = self.hazard_times()
        return workflows.ProbabilisticEventBased(
            vulnerability_functions,
            time_span, tses,
            self.rc.loss_curve_resolution,
            self.rc.conditional_loss_poes,
            self.rc.insured_losses)

    def hazard_times(self):
        """
        Return the hazard investigation time related to the ground
        motion field and the so-called time representative of the
        stochastic event set
        """
        return (self.rc.investigation_time,
                self.hc.ses_per_logic_tree_path * self.hc.investigation_time)

    @property
    def calculator_parameters(self):
        """
        Calculator specific parameters
        """
        return base.make_calc_params(
            conditional_loss_poes=self.rc.conditional_loss_poes or [],
            quantiles=self.rc.quantile_loss_curves or [],
            insured_losses=self.rc.insured_losses,
            sites_disagg=self.rc.sites_disagg or [],
            mag_bin_width=self.rc.mag_bin_width,
            distance_bin_width=self.rc.distance_bin_width,
            coordinate_bin_width=self.rc.coordinate_bin_width)
<|endoftext|>


<commit_before>#    This file is part of mib.
#
#    mib is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    mib is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with mib.  If not, see <http://www.gnu.org/licenses/>.
#

import select
import socket
import sys

class IrcSocket:
    """ Class for talking with an IRC server.
    """
    def __init__(self, host, port, nick, username, realname):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.host = host
        self.port = port
        self.nick = nick
        self.username = username
        self.realname = realname
        self.connected = False
        self.sendqueue = []
        self.channels = set()
        self.onChannels = set()
        self.readline_cbs = set()

    def join(self, channel):
        """ Adds channel to the list of channels that should be joined
        """
        self.channels.append(channel)

    def quit(self, reason=''):
        """ Quits the session. Reason will be sent as the quit message.
        """
        if reason != '':
            reply = 'QUIT :' + msg.params + ' (' + msg.nick + ')'
        else:
            reply = 'QUIT :(' + msg.nick + ')'
        self.send(reply)

    def send(self, msg):
        """ Adds msg to the queue of messages that will be sent to server.
            msg is _not_ a PRIVMSG, but an IRC protocol message.
        """
        self.sendqueue.append(msg)

    def register_readline_cb(self, function):
        """ Registers a callback for function to call with every read line.
            Function must take one string parameter.
            Every registered function will be called once.
        """
        self.readline_cbs.add(function)

    def __readline(self):
        """ Reads line from the socket and strips the newline char(s).
            Calls registered readline callbacks with the stripped line.
            For internal use.
        """
        s = self.buffer.readline()
        if not s:
            self.running = False
        if s[-2:] == '\r\n':
            s = s[:-2]
        elif s[-1:] in '\r\n':
            s = s[:-1]
        for function in self.readline_cbs:
            function(s)
        return s

    def __connect(self):
        """ Connects the socket to the server and registers the connection.
            For internal use.
        """
        self.sock.connect((self.host, self.port))
        self.buffer = self.sock.makefile('r')
        self.send('NICK ' + self.nick)
        self.send('USER ' + self.username + ' ' +
            socket.gethostname() + ' ' + self.host + ' :' + self.realname)

    def __send(self):
        """ Sends a message from the message queue to the server.
            For internal use.
        """
        if len(self.sendqueue) != 0:
            msg = self.sendqueue.pop(0)
            print msg
            self.sock.send(msg + '\r\n')

    def __join(self):
        """ Joins the channels in the channel list.
            For internal use.
        """
        for channel in self.channels.difference(self.onChannels):
            self.send('JOIN ' + channel)
            self.onChannels.add(channel)

    def run(self):
        """ The main loop.
        """
        self.running = True
        self.__connect()
        try:
            while self.running:
                self.__send()
                inputready, _, _ = select.select([sys.stdin, self.buffer],
                                                 [], [], 0.1)
                if sys.stdin in inputready:
                    line = sys.stdin.readline()
                    if line != '':
                        self.send(line)
                if self.buffer in inputready:
                    line = self.__readline()
                    if line != '':
                        print line
                    else:
                        continue
                if line.startswith('PING'):
                    self.__handleping(line)
                if line.startswith('251'):
                    self.__handlelusermsg(line)
                if (self.connected and
                    not self.channels.issubset(self.onChannels)):
                    self.__join()
        except Exception, e:
            print 'ERROR: ', e
        except: # exceptions not of type Exception
            pass
        finally:
            print 'Quiting!'
            self.sock.close()

    def ___handleping(self, line):
        """ Handles PING message from the server
            and answers with PONG message.
            For internal use.
        """
        pong = line.split()[1]
        reply = 'PONG ' + pong
        self.send(reply)

    def ___handlelusermsg(self, line):
        """ Handles LUSERCLIENT (251) message.
            This is here because some irc servers don't react to
            messages sent before the connection is properly initialized.
            For internal use.
        """
        self.connected = True

<commit_msg>Fix IrcSocket's calls to internal server message handling functions<commit_after>#    This file is part of mib.
#
#    mib is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    mib is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with mib.  If not, see <http://www.gnu.org/licenses/>.
#

import select
import socket
import sys

class IrcSocket:
    """ Class for talking with an IRC server.
    """
    def __init__(self, host, port, nick, username, realname):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.host = host
        self.port = port
        self.nick = nick
        self.username = username
        self.realname = realname
        self.connected = False
        self.sendqueue = []
        self.channels = set()
        self.onChannels = set()
        self.readline_cbs = set()

    def join(self, channel):
        """ Adds channel to the list of channels that should be joined
        """
        self.channels.append(channel)

    def quit(self, reason=''):
        """ Quits the session. Reason will be sent as the quit message.
        """
        if reason != '':
            reply = 'QUIT :' + msg.params + ' (' + msg.nick + ')'
        else:
            reply = 'QUIT :(' + msg.nick + ')'
        self.send(reply)

    def send(self, msg):
        """ Adds msg to the queue of messages that will be sent to server.
            msg is _not_ a PRIVMSG, but an IRC protocol message.
        """
        self.sendqueue.append(msg)

    def register_readline_cb(self, function):
        """ Registers a callback for function to call with every read line.
            Function must take one string parameter.
            Every registered function will be called once.
        """
        self.readline_cbs.add(function)

    def __readline(self):
        """ Reads line from the socket and strips the newline char(s).
            Calls registered readline callbacks with the stripped line.
            For internal use.
        """
        s = self.buffer.readline()
        if not s:
            self.running = False
        if s[-2:] == '\r\n':
            s = s[:-2]
        elif s[-1:] in '\r\n':
            s = s[:-1]
        for function in self.readline_cbs:
            function(s)
        return s

    def __connect(self):
        """ Connects the socket to the server and registers the connection.
            For internal use.
        """
        self.sock.connect((self.host, self.port))
        self.buffer = self.sock.makefile('r')
        self.send('NICK ' + self.nick)
        self.send('USER ' + self.username + ' ' +
            socket.gethostname() + ' ' + self.host + ' :' + self.realname)

    def __send(self):
        """ Sends a message from the message queue to the server.
            For internal use.
        """
        if len(self.sendqueue) != 0:
            msg = self.sendqueue.pop(0)
            print msg
            self.sock.send(msg + '\r\n')

    def __join(self):
        """ Joins the channels in the channel list.
            For internal use.
        """
        for channel in self.channels.difference(self.onChannels):
            self.send('JOIN ' + channel)
            self.onChannels.add(channel)

    def run(self):
        """ The main loop.
        """
        self.running = True
        self.__connect()
        try:
            while self.running:
                self.__send()
                inputready, _, _ = select.select([sys.stdin, self.buffer],
                                                 [], [], 0.1)
                if sys.stdin in inputready:
                    line = sys.stdin.readline()
                    if line != '':
                        self.send(line)
                if self.buffer in inputready:
                    line = self.__readline()
                    if line != '':
                        print line
                    else:
                        continue
                if line.startswith('PING'):
                    self.__handleping(line)
                if line.startswith(':%s 251' %(self.host)):
                    self.__handlelusermsg(line)
                if (self.connected and
                    not self.channels.issubset(self.onChannels)):
                    self.__join()
        except Exception, e:
            print 'ERROR: ', e
        except: # exceptions not of type Exception
            pass
        finally:
            print 'Quiting!'
            self.sock.close()

    def __handleping(self, line):
        """ Handles PING message from the server
            and answers with PONG message.
            For internal use.
        """
        pong = line.split()[1]
        reply = 'PONG ' + pong
        self.send(reply)

    def __handlelusermsg(self, line):
        """ Handles LUSERCLIENT (251) message.
            This is here because some irc servers don't react to
            messages sent before the connection is properly initialized.
            For internal use.
        """
        self.connected = True

<|endoftext|>


<commit_before>from os.path import expanduser
from os.path import basename

import os
import sys
import requests
import ipfshttpclient as ipfsapi

import re
# Datetime conversion to rfc1123
import locale
import datetime
import logging
import platform

from six.moves.urllib.request import urlopen
import json
from .__init__ import __version__ as ipwbVersion

from pkg_resources import parse_version

# from requests.exceptions import ConnectionError
from ipfshttpclient.exceptions import ConnectionError
from ipfshttpclient.exceptions import AddressError
from multiaddr.exceptions import StringParseError
from pkg_resources import parse_version

logger = logging.getLogger(__name__)

IPFSAPI_MUTLIADDRESS = '/dns/localhost/tcp/5001/http'
# or '/dns/{host}/tcp/{port}/http'
# or '/ip4/{ipaddress}/tcp/{port}/http'
# or '/ip6/{ipaddress}/tcp/{port}/http

IPWBREPLAY_ADDRESS = 'localhost:5000'

(IPWBREPLAY_HOST, IPWBREPLAY_PORT) = IPWBREPLAY_ADDRESS.split(':')
IPWBREPLAY_PORT = int(IPWBREPLAY_PORT)

INDEX_FILE = 'samples/indexes/salam-home.cdxj'

log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)

dtPattern = re.compile(r"^(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?$")

def createIPFSClient(daemonMultiaddr=IPFSAPI_MUTLIADDRESS):
    try:
        return ipfshttpclient.Client(daemonMultiaddr)
    except (StringParseError, AddressError):
        return None  # Malformed multiaddress for the daemon

def isDaemonAlive(daemonMultiaddr=IPFSAPI_MUTLIADDRESS):
    """Ensure that the IPFS daemon is running via HTTP before proceeding"""
    client = createIPFSClient()

    if client is None:
        raise Exception("Error initializing IPFS API client.")

    try:
        # ConnectionError/AttributeError if IPFS daemon not running
        client.id()
        return True

    except ConnectionError as err:
        raise Exception(f'Daemon is not running at: {daemonMultiaddr}') from err

    except OSError as err:
        raise Exception(
            'IPFS is likely not installed. See https://ipfs.io/docs/install/'
        ) from err

    except Exception as e:
        raise Exception(
            'Unknown error in retrieving IPFS daemon status.',
        ) from e

def isValidCDXJ(stringIn):  # TODO: Check specific strict syntax
    # Also, be sure to mind the meta headers starting with @/#, etc.
    return True

def isValidCDXJLine(cdxjLine):
    try:
        (surtURI, datetime, jsonData) = cdxjLine.split(' ', 2)

        json.loads(jsonData)
        validDatetime = len(datetime) == 14

        validSURT = True  # TODO: check valid SURT URI

        return validSURT and validDatetime
    except ValueError:  # Not valid JSON
        return False
    except NameError:
        return isCDXJMetadataRecord(cdxjLine)
    except Exception as e:
        return False

# Compare versions of software, <0 if a<b, 0 if ==, >1 if b>a
def compareVersions(versionA, versionB):
    return parse_version(versionA) < parse_version(versionB)

def isCDXJMetadataRecord(cdxjLine):
    return cdxjLine.strip()[:1] == '!'

def isLocalHosty(uri):
    # TODO: check for these SW conditions
    # (*, localhost, *); (*, 127/8, *); (*, ::1/128, *)
    localhosts = ['localhost', '127.0.0.1']
    for lh in localhosts:
        if lh in uri:
            return True
    return False

def setLocale():
    currentOS = platform.system()

    if currentOS == 'Darwin':
        newLocale = 'en_US'
    elif currentOS == 'Windows':
        newLocale = 'english'
    else:  # Assume Linux
        newLocale = 'en_US.utf8'

    try:
        locale.setlocale(locale.LC_TIME, newLocale)
    except locale.Error:
        locale.setlocale(locale.LC_TIME, '')

def digits14ToRFC1123(digits14):
    setLocale()
    d = datetime.datetime.strptime(digits14, '%Y%m%d%H%M%S')
    return d.strftime('%a, %d %b %Y %H:%M:%S GMT')

def rfc1123ToDigits14(rfc1123DateString):
    setLocale()
    d = datetime.datetime.strptime(rfc1123DateString,
                                   '%a, %d %b %Y %H:%M:%S %Z')

    # TODO: Account for conversion if TZ other than GMT not specified

    return d.strftime('%Y%m%d%H%M%S')

def iso8601ToDigits14(iso8601DateString):
    setLocale()
    d = datetime.datetime.strptime(iso8601DateString,
                                   "%Y-%m-%dT%H:%M:%SZ")

    # TODO: Account for conversion if TZ other than GMT not specified

    return d.strftime('%Y%m%d%H%M%S')

def isRFC1123Compliant(dtstr):
    try:
        datetime.datetime.strptime(dtstr, '%a, %d %b %Y %H:%M:%S GMT')
        return True
    except ValueError as err:
        return False

def getRFC1123OfNow():
    setLocale()
    d = datetime.datetime.now()
    return d.strftime('%a, %d %b %Y %H:%M:%S GMT')

def padDigits14(dtstr, validate=False):
    '''Pad datetime to make a 14-digit string and optionally validate it'''
    match = dtPattern.match(dtstr)
    if match:
        Y = match.group(1)
        m = match.group(2) or '01'
        d = match.group(3) or '01'
        H = match.group(4) or '00'
        M = match.group(5) or '00'
        S = match.group(6) or '00'
        dtstr = f'{Y}{m}{d}{H}{M}{S}'
    if validate:
        datetime.datetime.strptime(dtstr, '%Y%m%d%H%M%S')
    return dtstr

def fetch_remote_file(path):
    try:
        r = requests.get(path)
        return r.text

    except ConnectionError:
        raise Exception(f'File at {path} is unavailable.')

    except Exception as err:
        raise Exception(
            'An unknown error occurred trying to fetch {}'.format(path)
        ) from err

# IPFS Config manipulation from here on out.
def readIPFSConfig():
    ipfsConfigPath = expanduser("~") + '/.ipfs/config'
    if 'IPFS_PATH' in os.environ:
        ipfsConfigPath = os.environ.get('IPFS_PATH') + '/config'

    try:
        with open(ipfsConfigPath, 'r') as f:
            return json.load(f)

    except IOError as err:
        raise Exception(
            'IPFS config not found. Have you installed ipfs and run ipfs init?'
        ) from err

def writeIPFSConfig(jsonToWrite):
    ipfsConfigPath = expanduser("~") + '/.ipfs/config'
    if 'IPFS_PATH' in os.environ:
        ipfsConfigPath = os.environ.get('IPFS_PATH') + '/config'

    with open(ipfsConfigPath, 'w') as f:
        f.write(json.dumps(jsonToWrite, indent=4, sort_keys=True))

def getIPFSAPIHostAndPort(ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()

    (scheme, host, protocol, port) = (
        ipfsJSON['Addresses']['API'][1:].split('/')
    )
    return host + ':' + port

def getIPWBReplayConfig(ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()
    port = None
    if ('Ipwb' in ipfsJSON and 'Replay' in ipfsJSON['Ipwb'] and
       'Port' in ipfsJSON['Ipwb']['Replay']):
        host = ipfsJSON['Ipwb']['Replay']['Host']
        port = ipfsJSON['Ipwb']['Replay']['Port']
        return (host, port)
    else:
        return None

def setIPWBReplayConfig(Host, Port, ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()
    ipfsJSON['Ipwb'] = {}
    ipfsJSON['Ipwb']['Replay'] = {
      u'Host': Host,
      u'Port': Port
    }
    writeIPFSConfig(ipfsJSON)

def setIPWBReplayIndexPath(cdxj):
    if cdxj is None:
        cdxj = INDEX_FILE
    ipfsJSON = readIPFSConfig()
    ipfsJSON['Ipwb']['Replay']['Index'] = cdxj
    writeIPFSConfig(ipfsJSON)
    return

def getIPWBReplayIndexPath():
    ipfsJSON = readIPFSConfig()
    if 'Ipwb' not in ipfsJSON:
        setIPWBReplayConfig(IPWBREPLAY_HOST, IPWBREPLAY_PORT)
        ipfsJSON = readIPFSConfig()

    if 'Index' in ipfsJSON['Ipwb']['Replay']:
        return ipfsJSON['Ipwb']['Replay']['Index']
    else:
        return ''

# From pywb 2.0.4
def unsurt(surt):
    try:
        index = surt.index(')/')
        parts = surt[0:index].split(',')
        parts.reverse()
        host = '.'.join(parts)
        host += surt[index+1:]
        return host

    except ValueError:
        # May not be a valid surt
        return surt

def compareCurrentAndLatestIPWBVersions():
    try:
        resp = urlopen('https://pypi.python.org/pypi/ipwb/json')
        jResp = json.loads(resp.read())
        latestVersion = jResp['info']['version']
        currentVersion = re.sub(r'\.0+', '.', ipwbVersion)
        return (currentVersion, latestVersion)
    except Exception as e:
        return (None, None)

def checkForUpdate(_):
    (current, latest) = compareCurrentAndLatestIPWBVersions()

    if current != latest and current is not None:
        print('This version of ipwb is outdated.'
              ' Please run pip install --upgrade ipwb.')
    print(f'* Latest version: {latest}')
    print(f'* Installed version: {current}')
<commit_msg>Fixed issues after merging conflicts<commit_after>from os.path import expanduser

import os

import ipfshttpclient
import requests

import re
# Datetime conversion to rfc1123
import locale
import datetime
import logging
import platform

from six.moves.urllib.request import urlopen
import json
from .__init__ import __version__ as ipwbVersion

from pkg_resources import parse_version

# from requests.exceptions import ConnectionError
from ipfshttpclient.exceptions import ConnectionError
from ipfshttpclient.exceptions import AddressError
from multiaddr.exceptions import StringParseError
from pkg_resources import parse_version

logger = logging.getLogger(__name__)

IPFSAPI_MUTLIADDRESS = '/dns/localhost/tcp/5001/http'
# or '/dns/{host}/tcp/{port}/http'
# or '/ip4/{ipaddress}/tcp/{port}/http'
# or '/ip6/{ipaddress}/tcp/{port}/http

IPWBREPLAY_ADDRESS = 'localhost:5000'

(IPWBREPLAY_HOST, IPWBREPLAY_PORT) = IPWBREPLAY_ADDRESS.split(':')
IPWBREPLAY_PORT = int(IPWBREPLAY_PORT)

INDEX_FILE = 'samples/indexes/salam-home.cdxj'

log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)

dtPattern = re.compile(r"^(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?$")

def createIPFSClient(daemonMultiaddr=IPFSAPI_MUTLIADDRESS):
    try:
        return ipfshttpclient.Client(daemonMultiaddr)
    except (StringParseError, AddressError):
        return None  # Malformed multiaddress for the daemon

def isDaemonAlive(daemonMultiaddr=IPFSAPI_MUTLIADDRESS):
    """Ensure that the IPFS daemon is running via HTTP before proceeding"""
    client = createIPFSClient()

    if client is None:
        raise Exception("Error initializing IPFS API client.")

    try:
        # ConnectionError/AttributeError if IPFS daemon not running
        client.id()
        return True

    except ConnectionError as err:
        raise Exception(f'Daemon is not running at: {daemonMultiaddr}') from err

    except OSError as err:
        raise Exception(
            'IPFS is likely not installed. See https://ipfs.io/docs/install/'
        ) from err

    except Exception as e:
        raise Exception(
            'Unknown error in retrieving IPFS daemon status.',
        ) from e

def isValidCDXJ(stringIn):  # TODO: Check specific strict syntax
    # Also, be sure to mind the meta headers starting with @/#, etc.
    return True

def isValidCDXJLine(cdxjLine):
    try:
        (surtURI, datetime, jsonData) = cdxjLine.split(' ', 2)

        json.loads(jsonData)
        validDatetime = len(datetime) == 14

        validSURT = True  # TODO: check valid SURT URI

        return validSURT and validDatetime
    except ValueError:  # Not valid JSON
        return False
    except NameError:
        return isCDXJMetadataRecord(cdxjLine)
    except Exception as e:
        return False

# Compare versions of software, <0 if a<b, 0 if ==, >1 if b>a
def compareVersions(versionA, versionB):
    return parse_version(versionA) < parse_version(versionB)

def isCDXJMetadataRecord(cdxjLine):
    return cdxjLine.strip()[:1] == '!'

def isLocalHosty(uri):
    # TODO: check for these SW conditions
    # (*, localhost, *); (*, 127/8, *); (*, ::1/128, *)
    localhosts = ['localhost', '127.0.0.1']
    for lh in localhosts:
        if lh in uri:
            return True
    return False

def setLocale():
    currentOS = platform.system()

    if currentOS == 'Darwin':
        newLocale = 'en_US'
    elif currentOS == 'Windows':
        newLocale = 'english'
    else:  # Assume Linux
        newLocale = 'en_US.utf8'

    try:
        locale.setlocale(locale.LC_TIME, newLocale)
    except locale.Error:
        locale.setlocale(locale.LC_TIME, '')

def digits14ToRFC1123(digits14):
    setLocale()
    d = datetime.datetime.strptime(digits14, '%Y%m%d%H%M%S')
    return d.strftime('%a, %d %b %Y %H:%M:%S GMT')

def rfc1123ToDigits14(rfc1123DateString):
    setLocale()
    d = datetime.datetime.strptime(rfc1123DateString,
                                   '%a, %d %b %Y %H:%M:%S %Z')

    # TODO: Account for conversion if TZ other than GMT not specified

    return d.strftime('%Y%m%d%H%M%S')

def iso8601ToDigits14(iso8601DateString):
    setLocale()
    d = datetime.datetime.strptime(iso8601DateString,
                                   "%Y-%m-%dT%H:%M:%SZ")

    # TODO: Account for conversion if TZ other than GMT not specified

    return d.strftime('%Y%m%d%H%M%S')

def isRFC1123Compliant(dtstr):
    try:
        datetime.datetime.strptime(dtstr, '%a, %d %b %Y %H:%M:%S GMT')
        return True
    except ValueError as err:
        return False

def getRFC1123OfNow():
    setLocale()
    d = datetime.datetime.now()
    return d.strftime('%a, %d %b %Y %H:%M:%S GMT')

def padDigits14(dtstr, validate=False):
    '''Pad datetime to make a 14-digit string and optionally validate it'''
    match = dtPattern.match(dtstr)
    if match:
        Y = match.group(1)
        m = match.group(2) or '01'
        d = match.group(3) or '01'
        H = match.group(4) or '00'
        M = match.group(5) or '00'
        S = match.group(6) or '00'
        dtstr = f'{Y}{m}{d}{H}{M}{S}'
    if validate:
        datetime.datetime.strptime(dtstr, '%Y%m%d%H%M%S')
    return dtstr

def fetch_remote_file(path):
    try:
        r = requests.get(path)
        return r.text

    except ConnectionError:
        raise Exception(f'File at {path} is unavailable.')

    except Exception as err:
        raise Exception(
            'An unknown error occurred trying to fetch {}'.format(path)
        ) from err

# IPFS Config manipulation from here on out.
def readIPFSConfig():
    ipfsConfigPath = expanduser("~") + '/.ipfs/config'
    if 'IPFS_PATH' in os.environ:
        ipfsConfigPath = os.environ.get('IPFS_PATH') + '/config'

    try:
        with open(ipfsConfigPath, 'r') as f:
            return json.load(f)

    except IOError as err:
        raise Exception(
            'IPFS config not found. Have you installed ipfs and run ipfs init?'
        ) from err

def writeIPFSConfig(jsonToWrite):
    ipfsConfigPath = expanduser("~") + '/.ipfs/config'
    if 'IPFS_PATH' in os.environ:
        ipfsConfigPath = os.environ.get('IPFS_PATH') + '/config'

    with open(ipfsConfigPath, 'w') as f:
        f.write(json.dumps(jsonToWrite, indent=4, sort_keys=True))

def getIPFSAPIHostAndPort(ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()

    (scheme, host, protocol, port) = (
        ipfsJSON['Addresses']['API'][1:].split('/')
    )
    return host + ':' + port

def getIPWBReplayConfig(ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()
    port = None
    if ('Ipwb' in ipfsJSON and 'Replay' in ipfsJSON['Ipwb'] and
       'Port' in ipfsJSON['Ipwb']['Replay']):
        host = ipfsJSON['Ipwb']['Replay']['Host']
        port = ipfsJSON['Ipwb']['Replay']['Port']
        return (host, port)
    else:
        return None

def setIPWBReplayConfig(Host, Port, ipfsJSON=None):
    if not ipfsJSON:
        ipfsJSON = readIPFSConfig()
    ipfsJSON['Ipwb'] = {}
    ipfsJSON['Ipwb']['Replay'] = {
      u'Host': Host,
      u'Port': Port
    }
    writeIPFSConfig(ipfsJSON)

def setIPWBReplayIndexPath(cdxj):
    if cdxj is None:
        cdxj = INDEX_FILE
    ipfsJSON = readIPFSConfig()
    ipfsJSON['Ipwb']['Replay']['Index'] = cdxj
    writeIPFSConfig(ipfsJSON)
    return

def getIPWBReplayIndexPath():
    ipfsJSON = readIPFSConfig()
    if 'Ipwb' not in ipfsJSON:
        setIPWBReplayConfig(IPWBREPLAY_HOST, IPWBREPLAY_PORT)
        ipfsJSON = readIPFSConfig()

    if 'Index' in ipfsJSON['Ipwb']['Replay']:
        return ipfsJSON['Ipwb']['Replay']['Index']
    else:
        return ''

# From pywb 2.0.4
def unsurt(surt):
    try:
        index = surt.index(')/')
        parts = surt[0:index].split(',')
        parts.reverse()
        host = '.'.join(parts)
        host += surt[index+1:]
        return host

    except ValueError:
        # May not be a valid surt
        return surt

def compareCurrentAndLatestIPWBVersions():
    try:
        resp = urlopen('https://pypi.python.org/pypi/ipwb/json')
        jResp = json.loads(resp.read())
        latestVersion = jResp['info']['version']
        currentVersion = re.sub(r'\.0+', '.', ipwbVersion)
        return (currentVersion, latestVersion)
    except Exception as e:
        return (None, None)

def checkForUpdate(_):
    (current, latest) = compareCurrentAndLatestIPWBVersions()

    if current != latest and current is not None:
        print('This version of ipwb is outdated.'
              ' Please run pip install --upgrade ipwb.')
    print(f'* Latest version: {latest}')
    print(f'* Installed version: {current}')
<|endoftext|>


<commit_before>import webapp2
import jinja2
import os, urllib2, re, base64, urllib, datetime
from time import gmtime, strftime

from google.appengine.ext import db
from google.appengine.api import images, files, memcache, users

# cache bustin
#
# feel free to clear memcached from the appengine dashboard if you forget to
# bump this
version = '4'

development = False
cachetime = 3600 * 24 * 5
if os.environ.get('SERVER_SOFTWARE').startswith('Development'):
    development = True
    cachetime = 1

jinja_environment = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

#
# ((-)_(-))!!!! Thats a sloth by the way!
#

class SlothDrawing(db.Model):
    date = db.DateTimeProperty(auto_now_add=True)
    image = db.BlobProperty()

class Draw(webapp2.RequestHandler):
    def post(self):
        try:
            key = re.sub('[\W_]+', '', self.request.get('key_name'))

            if SlothDrawing.get_by_key_name(key) != None:
                self.response.set_status(409)
                return

            sloth_drawing = SlothDrawing(key_name=key)

            data = str(self.request.get('image'))
            data_to_64 = re.search(r'base64,(.*)', data).group(1)
            decoded = data_to_64.decode('base64')

            sloth_drawing.image = db.Blob(decoded)
            sloth_drawing.put()

            self.response.headers['Content-Type'] = 'application/json'
            self.response.out.write('{ "path": "%s" }' % key)

        except Exception, e:
            u_suck = e
            print u_suck

class FetchSloth(webapp2.RequestHandler):
    def get(self, sloth_path):
        memcached_key = '%s-sloth:%s' % (version, sloth_path)
        sloth_image = memcache.get(memcached_key)

        if sloth_image is not None:
            self.response.headers['Content-Type'] = 'image/png'
            self.response.out.write(sloth_image)
        else:

            sloth = SlothDrawing.get_by_key_name(sloth_path)

            if sloth == None:
                self.response.set_status(404)
                self.response.headers['Content-Type'] = 'text/plain'
                self.response.out.write('booourns no sloths here check your url!')
                return

            sloth_image = sloth.image

            if not memcache.add(memcached_key, sloth_image, cachetime):
                logging.error('Memcache set failed.')

            self.response.headers['Content-Type'] = 'image/png'
            self.response.out.write(sloth_image)

class Fetch(webapp2.RequestHandler):
    def get(self, sloth_path):
        memcached_key = '%s-fetch:%s' % (version, sloth_path)

        fetched = memcache.get(memcached_key)

        if fetched is not None:
            self.response.out.write(fetched)
        else:
            sloth = SlothDrawing.get_by_key_name(sloth_path)

            if sloth == None:
                self.response.set_status(404)
                self.response.headers['Content-Type'] = 'text/plain'
                self.response.out.write('booourns no sloths here check your url!')
                return

            # smrt ppl wouldn't hard code this
            # i'm not claiming to be one of them
            share_url_enc = urllib.quote_plus('http://sloths.arerad.com/' +
                sloth_path)

            template = jinja_environment.get_template('fetch.html')
            data = {
                'sloth_path': sloth_path,
                'title': sloth_path,
                'twitter_share_url':
                    'https://twitter.com/share',

                'facebook_share_url':
                    'https://www.facebook.com/sharer/sharer.php?u=' +
                    share_url_enc,

                'email_share_url': share_url_enc,
                'version': version,
                'render_date':
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()),

                'development': development
            }

            fetched = template.render(data)

            if not memcache.add(memcached_key, fetched, cachetime):
                logging.error('Memcache set failed.')

            self.response.out.write(fetched)

class MainPage(webapp2.RequestHandler):
    def get(self):
        memcached_key = '%s-homepage' % version
        homepage = memcache.get(memcached_key)
        if homepage is not None:
            self.response.out.write(homepage)
        else:
            template = jinja_environment.get_template('index.html')
            homepage = template.render({
                    'version': version,
                    'render_date': strftime("%Y-%m-%d %H:%M:%S", gmtime()),
                    'development': development
                })

            if not memcache.add(memcached_key, homepage, cachetime):
                logging.error('Memcache set failed.')

            self.response.out.write(homepage)

class About(webapp2.RequestHandler):
    def get(self):
        template = jinja_environment.get_template('about.html')
        self.response.out.write(template.render({
                'title': 'about',
                'version': version,
                'bodyclass': 'about-page'}))

class Gallery(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if user:

            if users.is_current_user_admin():
                sloths = db.GqlQuery("SELECT * FROM SlothDrawing ORDER BY date DESC LIMIT 100")

                gallery = "
"
                for sloth in sloths:
                    gallery += """

                            %s

                                %s

                            """ % (sloth.key().name(), sloth.key().name(), sloth.date)
                gallery += "
"
            else:
                gallery = 'email peter to get access to this'

            greeting = ("%s
sup %s, (sign out)" %
                        (gallery, user.nickname(), users.create_logout_url("/")))
        else:
            greeting = ("Sign in" %
                        users.create_login_url("/gallery"))

        self.response.out.write("%s" % greeting)

app = webapp2.WSGIApplication([(r'/', MainPage),
                               (r'/draw', Draw),
                               (r'/about', About),
                               (r'/gallery', Gallery),
                               (r'/(\w+)\.png', FetchSloth),
                               (r'/(\w+)', Fetch)],
                              debug=True)
<commit_msg>sup v bump<commit_after>import webapp2
import jinja2
import os, urllib2, re, base64, urllib, datetime
from time import gmtime, strftime

from google.appengine.ext import db
from google.appengine.api import images, files, memcache, users

# cache bustin
#
# feel free to clear memcached from the appengine dashboard if you forget to
# bump this
version = '5'

development = False
cachetime = 3600 * 24 * 5
if os.environ.get('SERVER_SOFTWARE').startswith('Development'):
    development = True
    cachetime = 1

jinja_environment = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

#
# ((-)_(-))!!!! Thats a sloth by the way!
#

class SlothDrawing(db.Model):
    date = db.DateTimeProperty(auto_now_add=True)
    image = db.BlobProperty()

class Draw(webapp2.RequestHandler):
    def post(self):
        try:
            key = re.sub('[\W_]+', '', self.request.get('key_name'))

            if SlothDrawing.get_by_key_name(key) != None:
                self.response.set_status(409)
                return

            sloth_drawing = SlothDrawing(key_name=key)

            data = str(self.request.get('image'))
            data_to_64 = re.search(r'base64,(.*)', data).group(1)
            decoded = data_to_64.decode('base64')

            sloth_drawing.image = db.Blob(decoded)
            sloth_drawing.put()

            self.response.headers['Content-Type'] = 'application/json'
            self.response.out.write('{ "path": "%s" }' % key)

        except Exception, e:
            u_suck = e
            print u_suck

class FetchSloth(webapp2.RequestHandler):
    def get(self, sloth_path):
        memcached_key = '%s-sloth:%s' % (version, sloth_path)
        sloth_image = memcache.get(memcached_key)

        if sloth_image is not None:
            self.response.headers['Content-Type'] = 'image/png'
            self.response.out.write(sloth_image)
        else:

            sloth = SlothDrawing.get_by_key_name(sloth_path)

            if sloth == None:
                self.response.set_status(404)
                self.response.headers['Content-Type'] = 'text/plain'
                self.response.out.write('booourns no sloths here check your url!')
                return

            sloth_image = sloth.image

            if not memcache.add(memcached_key, sloth_image, cachetime):
                logging.error('Memcache set failed.')

            self.response.headers['Content-Type'] = 'image/png'
            self.response.out.write(sloth_image)

class Fetch(webapp2.RequestHandler):
    def get(self, sloth_path):
        memcached_key = '%s-fetch:%s' % (version, sloth_path)

        fetched = memcache.get(memcached_key)

        if fetched is not None:
            self.response.out.write(fetched)
        else:
            sloth = SlothDrawing.get_by_key_name(sloth_path)

            if sloth == None:
                self.response.set_status(404)
                self.response.headers['Content-Type'] = 'text/plain'
                self.response.out.write('booourns no sloths here check your url!')
                return

            # smrt ppl wouldn't hard code this
            # i'm not claiming to be one of them
            share_url_enc = urllib.quote_plus('http://sloths.arerad.com/' +
                sloth_path)

            template = jinja_environment.get_template('fetch.html')
            data = {
                'sloth_path': sloth_path,
                'title': sloth_path,
                'twitter_share_url':
                    'https://twitter.com/share',

                'facebook_share_url':
                    'https://www.facebook.com/sharer/sharer.php?u=' +
                    share_url_enc,

                'email_share_url': share_url_enc,
                'version': version,
                'render_date':
                    strftime("%Y-%m-%d %H:%M:%S", gmtime()),

                'development': development
            }

            fetched = template.render(data)

            if not memcache.add(memcached_key, fetched, cachetime):
                logging.error('Memcache set failed.')

            self.response.out.write(fetched)

class MainPage(webapp2.RequestHandler):
    def get(self):
        memcached_key = '%s-homepage' % version
        homepage = memcache.get(memcached_key)
        if homepage is not None:
            self.response.out.write(homepage)
        else:
            template = jinja_environment.get_template('index.html')
            homepage = template.render({
                    'version': version,
                    'render_date': strftime("%Y-%m-%d %H:%M:%S", gmtime()),
                    'development': development
                })

            if not memcache.add(memcached_key, homepage, cachetime):
                logging.error('Memcache set failed.')

            self.response.out.write(homepage)

class About(webapp2.RequestHandler):
    def get(self):
        template = jinja_environment.get_template('about.html')
        self.response.out.write(template.render({
                'title': 'about',
                'version': version,
                'bodyclass': 'about-page'}))

class Gallery(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if user:

            if users.is_current_user_admin():
                sloths = db.GqlQuery("SELECT * FROM SlothDrawing ORDER BY date DESC LIMIT 100")

                gallery = "
"
                for sloth in sloths:
                    gallery += """

                            %s

                                %s

                            """ % (sloth.key().name(), sloth.key().name(), sloth.date)
                gallery += "
"
            else:
                gallery = 'email peter to get access to this'

            greeting = ("%s
sup %s, (sign out)" %
                        (gallery, user.nickname(), users.create_logout_url("/")))
        else:
            greeting = ("Sign in" %
                        users.create_login_url("/gallery"))

        self.response.out.write("%s" % greeting)

app = webapp2.WSGIApplication([(r'/', MainPage),
                               (r'/draw', Draw),
                               (r'/about', About),
                               (r'/gallery', Gallery),
                               (r'/(\w+)\.png', FetchSloth),
                               (r'/(\w+)', Fetch)],
                              debug=True)
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import pyaf.ForecastEngine as autof
import pyaf.Bench.TS_datasets as tsds

def test_transformation(itransformation):

    b1 = tsds.load_ozone()
    df = b1.mPastData

    lEngine = autof.cForecastEngine()
    lEngine

    H = b1.mHorizon;
    # lEngine.mOptions.enable_slow_mode();
    lEngine.mOptions.mDebugPerformance = True;

    lEngine.mOptions.disable_all_transformations();
    lEngine.mOptions.mActiveTransformation[itransformation] = True;
    lEngine.mOptions.mBoxCoxOrders = lEngine.mOptions.mExtensiveBoxCoxOrders;

    lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
    lEngine.getModelInfo();
    print(lEngine.mSignalDecomposition.mTrPerfDetails.head());

    lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution

    lEngine.standrdPlots("outputs/my_ozone_" + itransformation);

    dfapp_in = df.copy();
    dfapp_in.tail()

    dfapp_out = lEngine.forecast(dfapp_in, H);
    dfapp_out.to_csv("outputs/ozone_apply_out_" + itransformation + ".csv")
    dfapp_out.tail(H)
    print("Forecast Columns " , dfapp_out.columns);
    Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
    print(Forecast_DF.info())
    print("Forecasts\n" , Forecast_DF.tail(H));

    print("\n\n<ModelInfo>")
    print(lEngine.to_json());
    print("</ModelInfo>\n\n")
    print("\n\n<Forecast>")
    print(Forecast_DF.head(H).to_json(date_format='iso'))
    print("</Forecast>\n\n")
<commit_msg>Add the possibility to disable or force a model #17<commit_after>import pandas as pd
import numpy as np
import pyaf.ForecastEngine as autof
import pyaf.Bench.TS_datasets as tsds

def test_transformation(itransformation):

    b1 = tsds.load_ozone()
    df = b1.mPastData

    lEngine = autof.cForecastEngine()
    lEngine

    H = b1.mHorizon;
    # lEngine.mOptions.enable_slow_mode();
    lEngine.mOptions.mDebugPerformance = True;

    lEngine.mOptions.disable_all_transformations();
    lEngine.mOptions.mActiveTransformations[itransformation] = True;
    lEngine.mOptions.mBoxCoxOrders = lEngine.mOptions.mExtensiveBoxCoxOrders;

    lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
    lEngine.getModelInfo();
    print(lEngine.mSignalDecomposition.mTrPerfDetails.head());

    lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution

    lEngine.standrdPlots("outputs/my_ozone_" + itransformation);

    dfapp_in = df.copy();
    dfapp_in.tail()

    dfapp_out = lEngine.forecast(dfapp_in, H);
    dfapp_out.to_csv("outputs/ozone_apply_out_" + itransformation + ".csv")
    dfapp_out.tail(H)
    print("Forecast Columns " , dfapp_out.columns);
    Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
    print(Forecast_DF.info())
    print("Forecasts\n" , Forecast_DF.tail(H));

    print("\n\n<ModelInfo>")
    print(lEngine.to_json());
    print("</ModelInfo>\n\n")
    print("\n\n<Forecast>")
    print(Forecast_DF.head(H).to_json(date_format='iso'))
    print("</Forecast>\n\n")
<|endoftext|>


<commit_before>import psutil
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.dates import strpdate2num
import pandas as p
import sys
from pySPM.tools.fpanel import Fpanel

def plotLog(filename, watch=False, **kargs):
    fig, ax = plt.subplots(1,1)
    fig.subplots_adjust(hspace=0)
    plt.show(block=False)
    while True:
        with open(filename, 'r') as f:
            names = f.readline().rstrip().split('\t')
        df = pd.read_csv(filename, skiprows=1, delimiter='\t', parse_dates=[0], na_values="<undefined>", names=names)
        #df = df.dropna()
        ax2 = df.plot("Time", subplots=True, ax=ax, sharex=True)
        dt = df.iloc[-1,0]-df.iloc[0,0]
        for a in ax2:
            if dt.seconds < 15*60:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=1))
            elif dt.seconds < 3*60*60:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=5))
            else:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=15))
            a.xaxis.set_major_formatter(mpl.dates.DateFormatter('%H:%M'))
            a.grid()
        plt.minorticks_off()

        mypause(3)
        if not watch:
            break

def mypause(interval):
    backend = plt.rcParams['backend']
    if backend in mpl.rcsetup.interactive_bk:
        figManager = mpl._pylab_helpers.Gcf.get_active()
        if figManager is not None:
            canvas = figManager.canvas
            if canvas.figure.stale:
                canvas.draw()
            canvas.start_event_loop(interval)
            return

def main():
    if len(sys.argv)>1:
        filename = sys.argv[1]
        plotLog(filename, watch=False)
    else:
        F = Fpanel()
        logfile = F.getLogFile()
        plotLog(logfile, watch=True)

if __name__ == '__main__':
    main()<commit_msg>typo in import<commit_after>import psutil
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.dates import strpdate2num
import pandas as pd
import sys
from pySPM.tools.fpanel import Fpanel

def plotLog(filename, watch=False, **kargs):
    fig, ax = plt.subplots(1,1)
    fig.subplots_adjust(hspace=0)
    plt.show(block=False)
    while True:
        with open(filename, 'r') as f:
            names = f.readline().rstrip().split('\t')
        df = pd.read_csv(filename, skiprows=1, delimiter='\t', parse_dates=[0], na_values="<undefined>", names=names)
        #df = df.dropna()
        ax2 = df.plot("Time", subplots=True, ax=ax, sharex=True)
        dt = df.iloc[-1,0]-df.iloc[0,0]
        for a in ax2:
            if dt.seconds < 15*60:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=1))
            elif dt.seconds < 3*60*60:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=5))
            else:
                a.xaxis.set_major_locator(mpl.dates.MinuteLocator(interval=15))
            a.xaxis.set_major_formatter(mpl.dates.DateFormatter('%H:%M'))
            a.grid()
        plt.minorticks_off()

        mypause(3)
        if not watch:
            break

def mypause(interval):
    backend = plt.rcParams['backend']
    if backend in mpl.rcsetup.interactive_bk:
        figManager = mpl._pylab_helpers.Gcf.get_active()
        if figManager is not None:
            canvas = figManager.canvas
            if canvas.figure.stale:
                canvas.draw()
            canvas.start_event_loop(interval)
            return

def main():
    if len(sys.argv)>1:
        filename = sys.argv[1]
        plotLog(filename, watch=False)
    else:
        F = Fpanel()
        logfile = F.getLogFile()
        plotLog(logfile, watch=True)

if __name__ == '__main__':
    main()<|endoftext|>