<commit_before>from __future__ import division
from __future__ import absolute_import
from past.utils import old_div
from builtins import object
from nose.tools import (assert_equal, assert_not_equal, raises,
                        assert_almost_equal)
from nose.plugins.skip import SkipTest
from .test_helpers import assert_items_almost_equal, assert_items_equal
import pytest
import logging
logging.getLogger('openpathsampling.initialization').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.ensemble').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.storage').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.netcdfplus').setLevel(logging.CRITICAL)

import collections

from openpathsampling.numerics import (Histogram, SparseHistogram,
                                       HistogramPlotter2D)

class MockAxes(object):
    def __init__(self, xticks, yticks):
        self.xticks = xticks
        self.yticks = yticks

    def get_xticks(self): return self.xticks
    def get_yticks(self): return self.yticks

class TestHistogram(object):
    def setup(self):
        self.data = [1.0, 1.1, 1.2, 1.3, 2.0, 1.4, 2.3, 2.5, 3.1, 3.5]
        self.nbins = 5
        hist_counts = [5, 0, 2, 1, 1, 1]
        self.bins = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5]
        self.left_bin_edges = (1.0,)
        self.bin_widths = (0.5,)
        self.hist = collections.Counter({(0,): 5, (2,): 2, (3,): 1,
                                         (4,): 1, (5,): 1})

        self.default_hist = Histogram()
        self.hist_nbins = Histogram(n_bins=5)
        self.hist_nbins_binwidth = Histogram(n_bins=5, bin_width=1.0)
        self.hist_nbins_range = Histogram(n_bins=5, bin_range=(1.0, 3.5))
        self.hist_binwidth_range = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))

    def test_initialization(self):
        assert_equal(self.default_hist.bins, 20)

        assert_equal(self.hist_nbins.bins, 5)
        assert_equal(self.hist_nbins.bin_width, None)

        assert_equal(self.hist_nbins_binwidth.bins, 5)
        assert_equal(self.hist_nbins_binwidth.bin_width, None)

        assert_items_equal(self.hist_nbins_range.bins, self.bins)
        assert_items_equal(self.hist_binwidth_range.bins, self.bins)

    def test_build_from_data(self):
        hist = self.hist_nbins.histogram(self.data)
        assert_equal(self.hist_nbins.count, 10)
        assert_items_equal(hist, self.hist)

        hist2 = self.hist_binwidth_range.histogram(self.data)
        assert_equal(self.hist_binwidth_range.count, 10)
        assert_items_equal(hist2, self.hist)

    @raises(RuntimeError)
    def test_build_from_data_fail(self):
        histo = Histogram(n_bins=5)
        histo.histogram()

    def test_add_data_to_histogram(self):
        histogram = Histogram(n_bins=5, bin_range=(1.0, 3.5))
        hist = histogram.add_data_to_histogram(self.data)
        assert_equal(histogram.count, 10)
        assert_items_equal(hist, self.hist)

        hist2 = histogram.add_data_to_histogram(self.data)
        assert_items_equal(hist2, hist+hist)
        assert_equal(histogram.count, 20)

    def test_compare_parameters(self):
        assert_equal(self.hist_nbins.compare_parameters(None), False)
        assert_equal(
            self.hist_nbins_range.compare_parameters(self.hist_binwidth_range),
            True
        )
        assert_equal(
            self.hist_binwidth_range.compare_parameters(self.hist_nbins_range),
            True
        )
        histo = Histogram(n_bins=5)
        assert_equal(self.hist_nbins_range.compare_parameters(histo), False)
        histo.histogram(self.data)
        assert_equal(self.hist_nbins_range.compare_parameters(histo), False)
        assert_equal(
            self.hist_nbins_range.compare_parameters(self.hist_nbins),
            False
        )
        assert_equal(histo.compare_parameters(self.hist_nbins), False)
        assert_equal(self.hist_nbins.compare_parameters(histo), False)

    def test_xvals(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data) # need this to set the bins
        assert_equal(histo.left_bin_edges, self.left_bin_edges)
        assert_equal(histo.bin_widths, self.bin_widths)
        assert_items_equal(histo.xvals("l"), [1.0, 1.5, 2.0, 2.5, 3.0, 3.5])
        assert_items_equal(histo.xvals("r"), [1.5, 2.0, 2.5, 3.0, 3.5, 4.0])
        assert_items_equal(histo.xvals("m"),
                           [1.25, 1.75, 2.25, 2.75, 3.25, 3.75])

    def test_normalization(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        assert_equal(histo._normalization(), 5.0)

    def test_normalized(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        assert_items_equal(list(histo.normalized().values()),
                           [1.0, 0.0, 0.4, 0.2, 0.2, 0.2])
        assert_items_equal(list(histo.normalized(raw_probability=True).values()),
                           [0.5, 0.0, 0.2, 0.1, 0.1, 0.1])

    def test_cumulative(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        cumulative = list(histo.cumulative(None).values())
        assert_items_almost_equal(cumulative, [5.0, 5.0, 7.0, 8.0, 9.0, 10.0])
        assert_items_almost_equal(histo.cumulative(maximum=1.0),
                                  [0.5, 0.5, 0.7, 0.8, 0.9, 1.0])

    def test_cumulative_all_zero_warn(self):
        histo = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))
        histo._histogram = collections.Counter({(0,): 0, (1,): 0})
        with pytest.warns(UserWarning, match=r"No non-zero"):
            histo.cumulative()

    def test_reverse_cumulative(self):
        histo = Histogram(n_bins=5)
        histo.histogram(self.data)
        rev_cumulative = histo.reverse_cumulative(maximum=None)
        assert_items_almost_equal(list(rev_cumulative.values()),
                                  [10, 5, 5, 3, 2, 1])
        rev_cumulative = histo.reverse_cumulative(maximum=1.0)
        assert_items_almost_equal(list(rev_cumulative.values()),
                                  [1.0, 0.5, 0.5, 0.3, 0.2, 0.1])

    def test_reverse_cumulative_all_zero_warn(self):
        histo = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))
        histo._histogram = collections.Counter({(0,): 0, (1,): 0})
        with pytest.warns(UserWarning, match=r"No non-zero"):
            histo.reverse_cumulative()

    def test_left_bin_error(self):
        histo = Histogram(bin_width=0.5, bin_range=(-1.0, 3.5))
        histo.histogram([3.5])
        assert histo.reverse_cumulative() != 0

class TestSparseHistogram(object):
    def setup(self):
        data = [(0.0, 0.1), (0.2, 0.7), (0.3, 0.6), (0.6, 0.9)]
        self.histo = SparseHistogram(bin_widths=(0.5, 0.3),
                                     left_bin_edges=(0.0, -0.1))
        self.histo.histogram(data)

    def test_correct(self):
        correct_results = collections.Counter({
            (0, 0) : 1,
            (0, 2) : 2,
            (1, 3) : 1
        })
        assert_equal(self.histo._histogram, correct_results)

    def test_call(self):
        histo_fcn = self.histo()
        # voxels we have filled
        assert_equal(histo_fcn((0.25, 0.65)), 2)
        assert_equal(histo_fcn((0.01, 0.09)), 1)
        assert_equal(histo_fcn((0.61, 0.89)), 1)
        # empty voxel gives 0
        assert_equal(histo_fcn((2.00, 2.00)), 0)

    def test_normalized(self):
        raw_prob_normed = self.histo.normalized(raw_probability=True)
        assert_almost_equal(raw_prob_normed((0.25, 0.65)), 0.5)
        assert_almost_equal(raw_prob_normed((0.01, 0.09)), 0.25)
        assert_almost_equal(raw_prob_normed((0.61, 0.89)), 0.25)
        normed_fcn = self.histo.normalized()
        assert_almost_equal(normed_fcn((0.25, 0.65)), old_div(0.5,0.15))
        assert_almost_equal(normed_fcn((0.01, 0.09)), old_div(0.25,0.15))
        assert_almost_equal(normed_fcn((0.61, 0.89)), old_div(0.25,0.15))

class TestHistogramPlotter2D(object):
    def setup(self):
        data = [(0.0, 0.1), (0.2, 0.7), (0.3, 0.6), (0.6, 0.9)]
        histo = SparseHistogram(bin_widths=(0.5, 0.3),
                                left_bin_edges=(0.0, -0.1))
        histo.histogram(data)
        self.plotter = HistogramPlotter2D(histo)

    def test_to_bins(self):
        vals = [-0.1, 0.5, 0.8]
        xbins = self.plotter.to_bins(vals, 0)
        assert_items_almost_equal(xbins, [-0.2, 1.0, 1.6])
        ybins = self.plotter.to_bins(vals, 1)
        assert_items_almost_equal(ybins, [0.0, 2.0, 3.0])
        assert_equal(self.plotter.to_bins(None, 0), None)

    def test_axis_input(self):
        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=None,
                                             lims=None,
                                             dof=0)
        assert_equal(xt, None)
        assert_items_equal(xr, (-1.0, 2.0))
        assert_items_equal(xl, (0, 3))

        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=[-1.0, 0.0, 1.0],
                                             lims=None,
                                             dof=0)
        assert_items_equal(xt, [-2.0, 0.0, 2.0])
        assert_items_equal(xr, (-2.0, 2.0))
        assert_items_equal(xl, (0, 4.0))

        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=[-1.0, 0.0, 1.0],
                                             lims=(-2.5, 0.0),
                                             dof=0)
        assert_items_equal(xt, [-2.0, 0.0, 2.0])
        assert_items_equal(xr, (-5.0, 2.0))
        assert_items_equal(xl, (0.0, 5.0))

    def test_ticks_and_labels(self):
        # mock axes, make sure they work as expected
        fake_ax = MockAxes([-1.0, 0.0, 1.0], [-6.0, 0.0, 6.0])
        assert_items_equal(fake_ax.get_xticks(), [-1.0, 0.0, 1.0])
        assert_items_equal(fake_ax.get_yticks(), [-6.0, 0.0, 6.0])

        old_format = self.plotter.label_format
        self.plotter.label_format = "{:4.2f}"

        xticks, xlabels = self.plotter.ticks_and_labels(
            ticks=[-2.0, -1.0, 0.0, 1.0, 2.0], ax=fake_ax, dof=0
        )
        assert_items_almost_equal(xticks, [-2.0, -1.0, 0.0, 1.0, 2.0])
        assert_items_equal(xlabels, ["-1.00", "-0.50", "0.00", "0.50", "1.00"])

        yticks, ylabels = self.plotter.ticks_and_labels(
            ticks=None, ax=fake_ax, dof=1
        )
        assert_items_almost_equal(yticks, [-6.0, 0.0, 6.0])
        assert_items_equal(ylabels, ["-1.90", "-0.10", "1.70"])

        self.plotter.label_format = old_format

<commit_msg>actually test the returned lookup function<commit_after>from __future__ import division
from __future__ import absolute_import
from past.utils import old_div
from builtins import object
from nose.tools import (assert_equal, assert_not_equal, raises,
                        assert_almost_equal)
from nose.plugins.skip import SkipTest
from .test_helpers import assert_items_almost_equal, assert_items_equal
import pytest
import logging
logging.getLogger('openpathsampling.initialization').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.ensemble').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.storage').setLevel(logging.CRITICAL)
logging.getLogger('openpathsampling.netcdfplus').setLevel(logging.CRITICAL)

import collections

from openpathsampling.numerics import (Histogram, SparseHistogram,
                                       HistogramPlotter2D)

class MockAxes(object):
    def __init__(self, xticks, yticks):
        self.xticks = xticks
        self.yticks = yticks

    def get_xticks(self): return self.xticks
    def get_yticks(self): return self.yticks

class TestHistogram(object):
    def setup(self):
        self.data = [1.0, 1.1, 1.2, 1.3, 2.0, 1.4, 2.3, 2.5, 3.1, 3.5]
        self.nbins = 5
        hist_counts = [5, 0, 2, 1, 1, 1]
        self.bins = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5]
        self.left_bin_edges = (1.0,)
        self.bin_widths = (0.5,)
        self.hist = collections.Counter({(0,): 5, (2,): 2, (3,): 1,
                                         (4,): 1, (5,): 1})

        self.default_hist = Histogram()
        self.hist_nbins = Histogram(n_bins=5)
        self.hist_nbins_binwidth = Histogram(n_bins=5, bin_width=1.0)
        self.hist_nbins_range = Histogram(n_bins=5, bin_range=(1.0, 3.5))
        self.hist_binwidth_range = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))

    def test_initialization(self):
        assert_equal(self.default_hist.bins, 20)

        assert_equal(self.hist_nbins.bins, 5)
        assert_equal(self.hist_nbins.bin_width, None)

        assert_equal(self.hist_nbins_binwidth.bins, 5)
        assert_equal(self.hist_nbins_binwidth.bin_width, None)

        assert_items_equal(self.hist_nbins_range.bins, self.bins)
        assert_items_equal(self.hist_binwidth_range.bins, self.bins)

    def test_build_from_data(self):
        hist = self.hist_nbins.histogram(self.data)
        assert_equal(self.hist_nbins.count, 10)
        assert_items_equal(hist, self.hist)

        hist2 = self.hist_binwidth_range.histogram(self.data)
        assert_equal(self.hist_binwidth_range.count, 10)
        assert_items_equal(hist2, self.hist)

    @raises(RuntimeError)
    def test_build_from_data_fail(self):
        histo = Histogram(n_bins=5)
        histo.histogram()

    def test_add_data_to_histogram(self):
        histogram = Histogram(n_bins=5, bin_range=(1.0, 3.5))
        hist = histogram.add_data_to_histogram(self.data)
        assert_equal(histogram.count, 10)
        assert_items_equal(hist, self.hist)

        hist2 = histogram.add_data_to_histogram(self.data)
        assert_items_equal(hist2, hist+hist)
        assert_equal(histogram.count, 20)

    def test_compare_parameters(self):
        assert_equal(self.hist_nbins.compare_parameters(None), False)
        assert_equal(
            self.hist_nbins_range.compare_parameters(self.hist_binwidth_range),
            True
        )
        assert_equal(
            self.hist_binwidth_range.compare_parameters(self.hist_nbins_range),
            True
        )
        histo = Histogram(n_bins=5)
        assert_equal(self.hist_nbins_range.compare_parameters(histo), False)
        histo.histogram(self.data)
        assert_equal(self.hist_nbins_range.compare_parameters(histo), False)
        assert_equal(
            self.hist_nbins_range.compare_parameters(self.hist_nbins),
            False
        )
        assert_equal(histo.compare_parameters(self.hist_nbins), False)
        assert_equal(self.hist_nbins.compare_parameters(histo), False)

    def test_xvals(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data) # need this to set the bins
        assert_equal(histo.left_bin_edges, self.left_bin_edges)
        assert_equal(histo.bin_widths, self.bin_widths)
        assert_items_equal(histo.xvals("l"), [1.0, 1.5, 2.0, 2.5, 3.0, 3.5])
        assert_items_equal(histo.xvals("r"), [1.5, 2.0, 2.5, 3.0, 3.5, 4.0])
        assert_items_equal(histo.xvals("m"),
                           [1.25, 1.75, 2.25, 2.75, 3.25, 3.75])

    def test_normalization(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        assert_equal(histo._normalization(), 5.0)

    def test_normalized(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        assert_items_equal(list(histo.normalized().values()),
                           [1.0, 0.0, 0.4, 0.2, 0.2, 0.2])
        assert_items_equal(list(histo.normalized(raw_probability=True).values()),
                           [0.5, 0.0, 0.2, 0.1, 0.1, 0.1])

    def test_cumulative(self):
        histo = Histogram(n_bins=5)
        hist = histo.histogram(self.data)
        cumulative = list(histo.cumulative(None).values())
        assert_items_almost_equal(cumulative, [5.0, 5.0, 7.0, 8.0, 9.0, 10.0])
        assert_items_almost_equal(histo.cumulative(maximum=1.0),
                                  [0.5, 0.5, 0.7, 0.8, 0.9, 1.0])

    def test_cumulative_all_zero_warn(self):
        histo = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))
        histo._histogram = collections.Counter({(0,): 0, (1,): 0})
        with pytest.warns(UserWarning, match=r"No non-zero"):
            cumul = histo.cumulative()

        assert cumul(2.13) == 0
        for val in cumul.values():
            assert val == 0

    def test_reverse_cumulative(self):
        histo = Histogram(n_bins=5)
        histo.histogram(self.data)
        rev_cumulative = histo.reverse_cumulative(maximum=None)
        assert_items_almost_equal(list(rev_cumulative.values()),
                                  [10, 5, 5, 3, 2, 1])
        rev_cumulative = histo.reverse_cumulative(maximum=1.0)
        assert_items_almost_equal(list(rev_cumulative.values()),
                                  [1.0, 0.5, 0.5, 0.3, 0.2, 0.1])

    def test_reverse_cumulative_all_zero_warn(self):
        histo = Histogram(bin_width=0.5, bin_range=(1.0, 3.5))
        histo._histogram = collections.Counter({(0,): 0, (1,): 0})
        with pytest.warns(UserWarning, match=r"No non-zero"):
            rcumul = histo.reverse_cumulative()
        assert rcumul(3.12) == 0
        for val in rcumul.values():
            assert val == 0

    def test_left_bin_error(self):
        histo = Histogram(bin_width=0.5, bin_range=(-1.0, 3.5))
        histo.histogram([3.5])
        assert histo.reverse_cumulative() != 0

class TestSparseHistogram(object):
    def setup(self):
        data = [(0.0, 0.1), (0.2, 0.7), (0.3, 0.6), (0.6, 0.9)]
        self.histo = SparseHistogram(bin_widths=(0.5, 0.3),
                                     left_bin_edges=(0.0, -0.1))
        self.histo.histogram(data)

    def test_correct(self):
        correct_results = collections.Counter({
            (0, 0) : 1,
            (0, 2) : 2,
            (1, 3) : 1
        })
        assert_equal(self.histo._histogram, correct_results)

    def test_call(self):
        histo_fcn = self.histo()
        # voxels we have filled
        assert_equal(histo_fcn((0.25, 0.65)), 2)
        assert_equal(histo_fcn((0.01, 0.09)), 1)
        assert_equal(histo_fcn((0.61, 0.89)), 1)
        # empty voxel gives 0
        assert_equal(histo_fcn((2.00, 2.00)), 0)

    def test_normalized(self):
        raw_prob_normed = self.histo.normalized(raw_probability=True)
        assert_almost_equal(raw_prob_normed((0.25, 0.65)), 0.5)
        assert_almost_equal(raw_prob_normed((0.01, 0.09)), 0.25)
        assert_almost_equal(raw_prob_normed((0.61, 0.89)), 0.25)
        normed_fcn = self.histo.normalized()
        assert_almost_equal(normed_fcn((0.25, 0.65)), old_div(0.5,0.15))
        assert_almost_equal(normed_fcn((0.01, 0.09)), old_div(0.25,0.15))
        assert_almost_equal(normed_fcn((0.61, 0.89)), old_div(0.25,0.15))

class TestHistogramPlotter2D(object):
    def setup(self):
        data = [(0.0, 0.1), (0.2, 0.7), (0.3, 0.6), (0.6, 0.9)]
        histo = SparseHistogram(bin_widths=(0.5, 0.3),
                                left_bin_edges=(0.0, -0.1))
        histo.histogram(data)
        self.plotter = HistogramPlotter2D(histo)

    def test_to_bins(self):
        vals = [-0.1, 0.5, 0.8]
        xbins = self.plotter.to_bins(vals, 0)
        assert_items_almost_equal(xbins, [-0.2, 1.0, 1.6])
        ybins = self.plotter.to_bins(vals, 1)
        assert_items_almost_equal(ybins, [0.0, 2.0, 3.0])
        assert_equal(self.plotter.to_bins(None, 0), None)

    def test_axis_input(self):
        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=None,
                                             lims=None,
                                             dof=0)
        assert_equal(xt, None)
        assert_items_equal(xr, (-1.0, 2.0))
        assert_items_equal(xl, (0, 3))

        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=[-1.0, 0.0, 1.0],
                                             lims=None,
                                             dof=0)
        assert_items_equal(xt, [-2.0, 0.0, 2.0])
        assert_items_equal(xr, (-2.0, 2.0))
        assert_items_equal(xl, (0, 4.0))

        xt, xr, xl = self.plotter.axis_input(hist=[-1.0, 1.0, 2.0],
                                             ticklabels=[-1.0, 0.0, 1.0],
                                             lims=(-2.5, 0.0),
                                             dof=0)
        assert_items_equal(xt, [-2.0, 0.0, 2.0])
        assert_items_equal(xr, (-5.0, 2.0))
        assert_items_equal(xl, (0.0, 5.0))

    def test_ticks_and_labels(self):
        # mock axes, make sure they work as expected
        fake_ax = MockAxes([-1.0, 0.0, 1.0], [-6.0, 0.0, 6.0])
        assert_items_equal(fake_ax.get_xticks(), [-1.0, 0.0, 1.0])
        assert_items_equal(fake_ax.get_yticks(), [-6.0, 0.0, 6.0])

        old_format = self.plotter.label_format
        self.plotter.label_format = "{:4.2f}"

        xticks, xlabels = self.plotter.ticks_and_labels(
            ticks=[-2.0, -1.0, 0.0, 1.0, 2.0], ax=fake_ax, dof=0
        )
        assert_items_almost_equal(xticks, [-2.0, -1.0, 0.0, 1.0, 2.0])
        assert_items_equal(xlabels, ["-1.00", "-0.50", "0.00", "0.50", "1.00"])

        yticks, ylabels = self.plotter.ticks_and_labels(
            ticks=None, ax=fake_ax, dof=1
        )
        assert_items_almost_equal(yticks, [-6.0, 0.0, 6.0])
        assert_items_equal(ylabels, ["-1.90", "-0.10", "1.70"])

        self.plotter.label_format = old_format

<|endoftext|>


<commit_before># Copyright 2015 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Adaptive

Usage:
  adaptive client [--java=<dir>] [--python=<dir>] [--ruby=<dir>] <file> ...
  adaptive server [--java=<dir>] [--python=<dir>] [--ruby=<dir>] <file> ...
  adaptive -h | --help
  adaptive --version

Options:
  -h --help       Show this screen.
  --version       Show version.
  --java=<dir>    Emit java code to specified directory.
  --python=<dir>  Emit python code to specified directory.
  --ruby=<dir>    Emit ruby code to specified directory.
"""

import sys

from docopt import docopt

import _metadata, generate
from quark.compiler import Compiler, ParseError, CompileError
from quark.backend import Java

def main(args):
    if args["--version"]:
        sys.stderr.write("Adaptive %s\n" % _metadata.__version__)
        return
    filenames = args["<file>"]

    java = args["--java"]
    python = args["--python"]
    ruby = args["--ruby"]
    if python: return "python is not yet supported"
    if ruby: return "ruby is not yet supported"

    compiler = Compiler()
    compiler.annotator("value", generate.value)

    if args["client"]:
        compiler.annotator("service", generate.service_client)
    elif args["server"]:
        compiler.annotator("service", generate.service_server)
    else:
        assert False

    try:
        for name in filenames:
            with open(name, "rb") as fd:
                compiler.parse(name, fd.read())
    except IOError, e:
        return e
    except ParseError, e:
        return e
    finally:
        if java:
            compiler.write(java)

    try:
        compiler.compile()
    except CompileError, e:
        return e

    if java:
        j = Java()
        compiler.emit(j)
        j.write(java)

    if not java and not python and not ruby:
        return "no output languages specified"

def call_main():
    exit(main(docopt(__doc__)))

if __name__ == "__main__":
    call_main()
<commit_msg>added python backend<commit_after># Copyright 2015 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Adaptive

Usage:
  adaptive client [--java=<dir>] [--python=<dir>] <file> ...
  adaptive server [--java=<dir>] [--python=<dir>] <file> ...
  adaptive -h | --help
  adaptive --version

Options:
  -h --help       Show this screen.
  --version       Show version.
  --java=<dir>    Emit java code to specified directory.
  --python=<dir>  Emit python code to specified directory.
"""

import sys

from docopt import docopt

import _metadata, generate
from quark.compiler import Compiler, ParseError, CompileError
from quark.backend import Java, Python

def main(args):
    if args["--version"]:
        sys.stderr.write("Adaptive %s\n" % _metadata.__version__)
        return

    compiler = Compiler()
    compiler.annotator("value", generate.value)

    if args["client"]:
        compiler.annotator("service", generate.service_client)
    elif args["server"]:
        compiler.annotator("service", generate.service_server)
    else:
        assert False

    java = args["--java"]
    python = args["--python"]
    if java: compiler.emitter(Java, java)
    if python: compiler.emitter(Python, python)

    try:
        for name in args["<file>"]:
            with open(name, "rb") as fd:
                compiler.parse(name, fd.read())
        compiler.compile()
    except IOError, e:
        return e
    except ParseError, e:
        return e
    except CompileError, e:
        return e

    if not java and not python:
        return "no output languages specified"

def call_main():
    exit(main(docopt(__doc__)))

if __name__ == "__main__":
    call_main()
<|endoftext|>


<commit_before>import scipy as _sp
from openpnm.utils import logging as _logging
_logger = _logging.getLogger(__name__)

def ctc(target, pore_diameter='pore.diameter'):
    r"""
    Calculate throat length assuming point-like pores, i.e. center-to-center
    distance between pores.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    C1 = network['pore.coords'][cn[:, 0]]
    C2 = network['pore.coords'][cn[:, 1]]
    E = _sp.sqrt(_sp.sum((C1-C2)**2, axis=1))
    return E

def straight(target, pore_diameter='pore.diameter', L_negative=1e-9):
    r"""
    Calculate throat length

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    L_negative : float
        The default throat length to use when negative lengths are found.  The
        default is 1 nm.  To accept negative throat lengths, set this value to
        ``None``.
    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    E = ctc(target, pore_diameter=pore_diameter)
    D1 = network[pore_diameter][cn[:, 0]]
    D2 = network[pore_diameter][cn[:, 1]]
    value = E-(D1+D2)/2.
    if _sp.any(value < 0) and L_negative is not None:
        _logger.warn('Negative throat lengths are calculated. Arbitrary ' +
                     'positive length assigned: ' + str(L_negative))
        Ts = _sp.where(value < 0)[0]
        value[Ts] = L_negative
    return value

def spherical_pores(target, pore_diameter='pore.diameter',
                    throat_diameter='throat.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are spheres.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    d1 = target[pore_diameter][cn[:, 0]]
    d2 = target[pore_diameter][cn[:, 1]]
    dt = target[throat_diameter][throats]
    L = ctc(target, pore_diameter=pore_diameter)
    L1 = _sp.sqrt(d1**2 - dt**2) / 2            # Effective length of pore 1
    L2 = _sp.sqrt(d2**2 - dt**2) / 2            # Effective length of pore 2
    Lt = L - (L1+L2)                            # Effective length of throat
    # Handling throats w/ overlapping pores
    L1temp = (4*L**2+d1**2-d2**2) / (8*L)
    L2temp = (4*L**2+d2**2-d1**2) / (8*L)
    htemp = (2*_sp.sqrt(d1**2/4 - L1temp**2)).real
    mask_overlap = ((L - (d1+d2)/2) < 0) & (dt < htemp)
    L1[mask_overlap] = L1temp[mask_overlap]
    L2[mask_overlap] = L2temp[mask_overlap]
    return {'pore1': L1, 'throat': Lt, 'pore2': L2}

def truncated_pyramid(target, pore_diameter='pore.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are spheres.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    L1 = 0.5*target[pore_diameter][cn[:, 0]]        # Effective length of pore1
    L2 = 0.5*target[pore_diameter][cn[:, 1]]        # Effective length of pore2
    L = ctc(target, pore_diameter=pore_diameter)
    Lt = L - (L1+L2)                                # Effective length of throat
    return {'throat.pore1': L1, 'throat.throat': Lt, 'throat.pore2': L2}

def circular_pores(target, pore_diameter='pore.diameter',
                    throat_diameter='throat.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are circles.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    return spherical_pores(target, pore_diameter=pore_diameter,
                           throat_diameter=throat_diameter)
<commit_msg>Set throat length for overlapping pores to 1e-12<commit_after>import scipy as _sp
from openpnm.utils import logging as _logging
_logger = _logging.getLogger(__name__)

def ctc(target, pore_diameter='pore.diameter'):
    r"""
    Calculate throat length assuming point-like pores, i.e. center-to-center
    distance between pores.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    C1 = network['pore.coords'][cn[:, 0]]
    C2 = network['pore.coords'][cn[:, 1]]
    E = _sp.sqrt(_sp.sum((C1-C2)**2, axis=1))
    return E

def straight(target, pore_diameter='pore.diameter', L_negative=1e-9):
    r"""
    Calculate throat length

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    L_negative : float
        The default throat length to use when negative lengths are found.  The
        default is 1 nm.  To accept negative throat lengths, set this value to
        ``None``.
    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    E = ctc(target, pore_diameter=pore_diameter)
    D1 = network[pore_diameter][cn[:, 0]]
    D2 = network[pore_diameter][cn[:, 1]]
    value = E-(D1+D2)/2.
    if _sp.any(value < 0) and L_negative is not None:
        _logger.warn('Negative throat lengths are calculated. Arbitrary ' +
                     'positive length assigned: ' + str(L_negative))
        Ts = _sp.where(value < 0)[0]
        value[Ts] = L_negative
    return value

def spherical_pores(target, pore_diameter='pore.diameter',
                    throat_diameter='throat.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are spheres.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    d1 = target[pore_diameter][cn[:, 0]]
    d2 = target[pore_diameter][cn[:, 1]]
    dt = target[throat_diameter][throats]
    L = ctc(target, pore_diameter=pore_diameter)
    L1 = _sp.sqrt(d1**2 - dt**2) / 2            # Effective length of pore 1
    L2 = _sp.sqrt(d2**2 - dt**2) / 2            # Effective length of pore 2
    Lt = L - (L1+L2)                            # Effective length of throat
    # Handling throats w/ overlapping pores
    L1temp = (4*L**2+d1**2-d2**2) / (8*L)
    L2temp = (4*L**2+d2**2-d1**2) / (8*L)
    htemp = (2*_sp.sqrt(d1**2/4 - L1temp**2)).real
    mask_overlap = ((L - (d1+d2)/2) < 0) & (dt < htemp)
    L1[mask_overlap] = L1temp[mask_overlap]
    L2[mask_overlap] = L2temp[mask_overlap]
    Lt[mask_overlap] = 1e-12
    return {'pore1': L1, 'throat': Lt, 'pore2': L2}

def truncated_pyramid(target, pore_diameter='pore.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are spheres.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    """
    network = target.project.network
    throats = network.map_throats(target['throat._id'])
    cn = network['throat.conns'][throats]
    L1 = 0.5*target[pore_diameter][cn[:, 0]]        # Effective length of pore1
    L2 = 0.5*target[pore_diameter][cn[:, 1]]        # Effective length of pore2
    L = ctc(target, pore_diameter=pore_diameter)
    Lt = L - (L1+L2)                                # Effective length of throat
    return {'throat.pore1': L1, 'throat.throat': Lt, 'throat.pore2': L2}

def circular_pores(target, pore_diameter='pore.diameter',
                    throat_diameter='throat.diameter'):
    r"""
    Calculate conduit lengths, i.e. pore 1 length, throat length,
    and pore 2 length, assuming that pores are circles.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    return spherical_pores(target, pore_diameter=pore_diameter,
                           throat_diameter=throat_diameter)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('game', '0100_merge'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='workspace',
            name='workspace',
        ),
        migrations.AddField(
            model_name='workspace',
            name='contents',
            field=models.TextField(default=b''),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='level',
            name='origin',
            field=models.CharField(default=b'[]', max_length=10),
        ),
        migrations.AlterField(
            model_name='workspace',
            name='owner',
            field=models.ForeignKey(blank=True, to='portal.UserProfile', null=True),
        ),
    ]
<commit_msg>Ok, this autogenerated. Hopefully will like it now.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('game', '0100_merge'),
    ]

    operations = [
        migrations.RenameField(
            model_name='workspace',
            old_name='workspace',
            new_name='contents',
        ),
        migrations.AlterField(
            model_name='level',
            name='origin',
            field=models.CharField(default=b'[]', max_length=10),
        ),
        migrations.AlterField(
            model_name='workspace',
            name='owner',
            field=models.ForeignKey(blank=True, to='portal.UserProfile', null=True),
        ),
    ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import io
import os
import atexit

from blox.utils import read_i64, write_i64, read_json, write_json
from blox.blosc import write_blosc

class File(object):
    def __init__(self, filename, mode=None):
        filename = os.path.abspath(os.path.expanduser(filename))
        if mode is None:
            mode = 'r' if os.path.exists(filename) else 'w'
        elif mode not in 'rw':
            raise ValueError('invalid mode: {!r}; expected r/w'.format(mode))
        self._mode = mode
        self._filename = filename
        self._handle = io.open(filename, mode + 'b')
        self._index = {}
        self._dirty = True
        atexit.register(self.close)
        if not self.writable:
            try:
                self._handle.seek(-8, os.SEEK_END)
                self._handle.seek(read_i64(self._handle))
                self._index = read_json(self._handle)
            except Exception as e:
                raise IOError('cannot read file ({}: {})'.format(type(e).__name__, str(e)))
        self._seek = 0

    @property
    def filename(self):
        return self._filename

    @property
    def mode(self):
        return self._mode

    @property
    def writable(self):
        return self._mode == 'w'

    @property
    def filesize(self):
        return os.stat(self._filename).st_size

    def _write_index(self):
        self._handle.truncate()
        write_json(self._handle, self._index)
        write_i64(self._handle, self._seek)
        self._dirty = False

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        if self._handle is not None:
            if self.writable:
                if self._dirty:
                    self._write_index()
                self._handle.flush()
            self._handle.close()
        self._handle = None

    def create_dataset(self, name, data, compression='lz4', level=5, shuffle=True):
        if not self.writable:
            raise ValueError('file is not writable')
        if name in self._index:
            raise ValueError('dataset {!r} already exists'.format(name))
        self._index[name] = self._seek
        self._handle.seek(self._seek)
        self._dirty = True
        self._seek = write_blosc(data, self._handle, compression, level, shuffle)
        self._write_index()
<commit_msg>Check if the handle has been closed already<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import io
import os
import atexit

from blox.utils import read_i64, write_i64, read_json, write_json
from blox.blosc import write_blosc

class File(object):
    def __init__(self, filename, mode=None):
        filename = os.path.abspath(os.path.expanduser(filename))
        if mode is None:
            mode = 'r' if os.path.exists(filename) else 'w'
        elif mode not in 'rw':
            raise ValueError('invalid mode: {!r}; expected r/w'.format(mode))
        self._mode = mode
        self._filename = filename
        self._handle = io.open(filename, mode + 'b')
        self._index = {}
        self._dirty = True
        atexit.register(self.close)
        if not self.writable:
            try:
                self._handle.seek(-8, os.SEEK_END)
                self._handle.seek(read_i64(self._handle))
                self._index = read_json(self._handle)
            except Exception as e:
                raise IOError('cannot read file ({}: {})'.format(type(e).__name__, str(e)))
        self._seek = 0

    @property
    def filename(self):
        return self._filename

    @property
    def mode(self):
        return self._mode

    @property
    def writable(self):
        return self._mode == 'w'

    @property
    def filesize(self):
        return os.stat(self._filename).st_size

    def _write_index(self):
        self._handle.truncate()
        write_json(self._handle, self._index)
        write_i64(self._handle, self._seek)
        self._dirty = False

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def close(self):
        if self._handle is not None:
            if self.writable:
                if self._dirty:
                    self._write_index()
                self._handle.flush()
            self._handle.close()
        self._handle = None

    def _check_handle(self, write=False):
        if self._handle is None:
            raise IOError('the file handle has been closed')
        if write and not self.writable:
            raise IOError('the file is not writable')

    def create_dataset(self, name, data, compression='lz4', level=5, shuffle=True):
        self._check_handle(write=True)
        if name in self._index:
            raise ValueError('dataset {!r} already exists'.format(name))
        self._index[name] = self._seek
        self._handle.seek(self._seek)
        self._dirty = True
        self._seek = write_blosc(data, self._handle, compression, level, shuffle)
        self._write_index()
<|endoftext|>


<commit_before>import os

from datetime import datetime
from decimal import Decimal

from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ValidationError
from django.test import TestCase

import plata
from plata import plata_settings
from plata.contact.models import Contact
from plata.product.models import TaxClass, Product, Discount
from plata.shop.models import Order, OrderStatus, OrderPayment

class ModelTest(TestCase):
    def assertRaisesWithCode(self, exception, fn, code):
        try:
            fn()
        except exception, e:
            if e.code == code:
                return True
            raise
        raise Exception, '%s did not raise %s' % (fn, exception)

    def setUp(self):
        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def create_contact(self):
        return Contact.objects.create(
            billing_company=u'BigCorp',
            billing_first_name=u'Hans',
            billing_last_name=u'Muster',
            billing_address=u'Musterstrasse 42',
            billing_zip_code=u'8042',
            billing_city=u'Beispielstadt',
            billing_country=u'CH',
            shipping_same_as_billing=True,
            currency='CHF',
            )

    def create_order(self, contact=None):
        contact = contact or self.create_contact()

        return Order.objects.create(
            contact=contact,
            currency='CHF',
            )

    def create_tax_classes(self):
        tax_class, created = TaxClass.objects.get_or_create(
            name='Standard Swiss Tax Rate',
            rate=Decimal('7.60'),
            )

        tax_class_germany, created = TaxClass.objects.get_or_create(
            name='Umsatzsteuer (Germany)',
            rate=Decimal('19.00'),
            )

        tax_class_something, created = TaxClass.objects.get_or_create(
            name='Some tax rate',
            rate=Decimal('12.50'),
            )

        return tax_class, tax_class_germany, tax_class_something

    def create_product(self):
        tax_class, tax_class_germany, tax_class_something = self.create_tax_classes()

        product = Product.objects.create(
            name='Test Product 1',
            )

        # An old price in CHF which should not influence the rest of the tests
        product.prices.create(
            currency='CHF',
            tax_class=tax_class,
            _unit_price=Decimal('99.90'),
            tax_included=True
            )

        product.prices.create(
            currency='CHF',
            tax_class=tax_class,
            _unit_price=Decimal('79.90'),
            tax_included=True,
            )

        product.prices.create(
            currency='EUR',
            tax_class=tax_class_germany,
            _unit_price=Decimal('49.90'),
            tax_included=True,
            )

        product.prices.create(
            currency='ASDF',
            tax_class=tax_class_something,
            _unit_price=Decimal('65.00'),
            tax_included=False,
            )

        return product

    def test_01_basic_order(self):
        product = self.create_product()
        order = self.create_order()

        self.assertEqual(product.get_price(currency=order.currency).currency, order.currency)

        order.modify_item(product, 5)
        order.modify_item(product, -4)
        item = order.modify_item(product, 1)

        self.assertEqual(order.items.count(), 1)

        self.assertEqual(item.quantity, 2)

        item_price = Decimal('79.90')
        line_item_price = item_price * item.quantity
        order_total = Decimal('159.80')
        tax_factor = Decimal('1.076')

        self.assertAlmostEqual(order.items_subtotal, order_total / tax_factor)
        self.assertAlmostEqual(order.items_subtotal + order.items_tax, order_total)
        self.assertAlmostEqual(order.total, order_total)

        self.assertAlmostEqual(item._unit_price, item_price / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal_incl_tax, line_item_price)
        self.assertAlmostEqual(item.discounted_subtotal_excl_tax, line_item_price / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal_incl_tax, line_item_price)

        self.assertAlmostEqual(item.unit_price, item_price)
        self.assertAlmostEqual(item.line_item_discount, 0)
        self.assertAlmostEqual(item.discounted_subtotal, item.discounted_subtotal_incl_tax)

        # Switch around tax handling and re-test
        plata_settings.PLATA_PRICE_INCLUDES_TAX = False

        self.assertAlmostEqual(item.unit_price, item_price / tax_factor)
        self.assertAlmostEqual(item.line_item_discount, 0 / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal, item.discounted_subtotal_excl_tax)

        # Switch tax handling back
        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_02_eur_order(self):
        product = self.create_product()
        order = self.create_order()

        order.currency = 'EUR'
        order.save()

        item = order.modify_item(product, 2)

        self.assertEqual(item.unit_price, Decimal('49.90'))
        self.assertEqual(item.currency, order.currency)

    def test_03_mixed_currencies(self):
        """
        Create an invalid order
        """

        p1 = self.create_product()
        p2 = self.create_product()
        order = self.create_order()

        order.currency = 'CHF'
        i1 = order.modify_item(p1, 3)

        order.currency = 'EUR'
        self.assertRaisesWithCode(ValidationError, lambda: order.modify_item(p2, 2),
            code='multiple_currency')

        # Validation should still fail
        self.assertRaisesWithCode(ValidationError, lambda: order.validate(),
            code='multiple_currency')

        order.currency = 'CHF'
        # Order should validate now
        order.validate()

    def test_04_order_modify_item(self):
        p1 = self.create_product()
        p2 = self.create_product()
        order = self.create_order()

        order.modify_item(p1, 42)
        order.modify_item(p2, 42)
        self.assertEqual(order.items.count(), 2)

        order.modify_item(p1, -42)
        self.assertEqual(order.items.count(), 1)

    def test_05_order_status(self):
        order = self.create_order()

        self.assertRaisesWithCode(ValidationError, lambda: order.update_status(
            Order.CHECKOUT,
            'Checkout process has started',
            ), code='order_empty')

        product = self.create_product()
        order.modify_item(product, 1)

        # Should be possible to update order status now
        order.update_status(
            Order.CHECKOUT,
            'Checkout process has started',
            )

        # Should not be possible to modify order once checkout process has started
        self.assertRaisesWithCode(ValidationError, lambda: order.modify_item(product, 2),
            code='order_sealed')

        self.assertEqual(order.status, Order.CHECKOUT)

    def test_06_order_percentage_discount(self):
        order = self.create_order()
        p1 = self.create_product()
        p2 = self.create_product()

        order.modify_item(p1, 3)
        order.modify_item(p2, 5)

        discount = Discount.objects.create(
            type=Discount.PERCENTAGE,
            key='asdf',
            name='Percentage discount',
            value=30)
        order.add_discount(discount)
        order.recalculate_total()

        tax_factor = Decimal('1.076')
        item_price_incl_tax = Decimal('79.90')
        item_price_excl_tax = item_price_incl_tax / tax_factor

        order.recalculate_total()
        item = order.modify_item(p1, 0)
        item2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(item.unit_price, item_price_incl_tax)
        self.assertAlmostEqual(item.line_item_discount, item_price_incl_tax * 3 * Decimal('0.30'))
        self.assertAlmostEqual(order.total,
            item.discounted_subtotal + item2.discounted_subtotal)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = False
        order.recalculate_total()
        item = order.modify_item(p1, 0)
        item2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(item.unit_price, item_price_excl_tax)
        self.assertAlmostEqual(item.line_item_discount, item_price_excl_tax * 3 * Decimal('0.30'))
        self.assertAlmostEqual(order.total,
            item.discounted_subtotal + item2.discounted_subtotal + order.items_tax)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_07_order_amount_discount(self):
        order = self.create_order()
        p1 = self.create_product()
        p2 = self.create_product()

        normal1 = order.modify_item(p1, 3)
        normal2 = order.modify_item(p2, 5)

        order.recalculate_total()
        self.assertAlmostEqual(order.total, Decimal('639.20'))

        discount = Discount.objects.create(
            type=Discount.AMOUNT_INCL_TAX,
            key='asdf',
            name='Amount discount',
            value=Decimal('50.00'))
        order.add_discount(discount)
        order.recalculate_total()

        discounted1 = order.modify_item(p1, 0)
        discounted2 = order.modify_item(p2, 0)

        tax_factor = Decimal('1.076')
        item_price_incl_tax = Decimal('79.90')
        item_price_excl_tax = item_price_incl_tax / tax_factor

        self.assertAlmostEqual(order.total, Decimal('639.20') - Decimal('50.00'))

        self.assertAlmostEqual(normal1.unit_price, discounted1.unit_price)
        self.assertAlmostEqual(normal2.unit_price, discounted2.unit_price)
        self.assertAlmostEqual(normal1.unit_price, item_price_incl_tax)

        self.assertEqual(normal1.line_item_discount, 0)
        self.assertEqual(normal2.line_item_discount, 0)

        self.assertAlmostEqual(discounted1.line_item_discount, Decimal('50.00') / 8 * 3)
        self.assertAlmostEqual(discounted2.line_item_discount, Decimal('50.00') / 8 * 5)

        self.assertAlmostEqual(discounted1.discounted_subtotal, order.total / 8 * 3)
        self.assertAlmostEqual(discounted2.discounted_subtotal, order.total / 8 * 5)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = False
        order.recalculate_total()
        discounted1 = order.modify_item(p1, 0)
        discounted2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(order.total, Decimal('639.20') - Decimal('50.00'))

        self.assertAlmostEqual(discounted1.unit_price, item_price_excl_tax)
        self.assertAlmostEqual(discounted1.line_item_discount, discount.value / tax_factor / 8 * 3)
        self.assertAlmostEqual(order.total,
            discounted1.discounted_subtotal + discounted2.discounted_subtotal + order.items_tax)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_08_order_payment(self):
        order = self.create_order()
        product = self.create_product()

        order.modify_item(product, 10)
        order.recalculate_total()

        payment = order.payments.create(
            currency='CHF',
            amount=Decimal('49.90'),
            payment_method='Mafia style',
            )

        order = Order.objects.get(pk=order.pk)
        self.assertAlmostEqual(order.paid, 0)

        payment.authorized = datetime.now()
        payment.save()

        order = Order.objects.get(pk=order.pk)
        self.assertAlmostEqual(order.balance_remaining, order.total - payment.amount)

class ShopTest(TestCase):
    def setUp(self):
        self.shop = plata.shop_instance()

        class Empty(object):
            pass

        self.request = Empty()
        self.request.session = {}
        self.request.user = AnonymousUser()

    def test_01_creation(self):
        contact = self.shop.contact_from_request(self.request)
        self.assertEqual(contact, None)

        contact = self.shop.contact_from_request(self.request, create=True)
        self.assertNotEqual(contact, None)

        contact = self.shop.contact_from_request(self.request, create=True)
        self.assertEqual(Contact.objects.count(), 1)

        order = self.shop.order_from_request(self.request)
        self.assertEqual(order, None)

        order = self.shop.order_from_request(self.request, create=True)
        self.assertEqual(Order.objects.count(), 1)
        self.assertEqual(order.contact, contact)

class ViewTest(TestCase):
    def setUp(self):
        self.ORIG_TEMPLATE_DIRS = settings.TEMPLATE_DIRS
        settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)

    def tearDown(self):
        settings.TEMPLATE_DIRS = self.ORIG_TEMPLATE_DIRS

    def test_01_cart_empty(self):
        self.assertContains(self.client.get('/plata/cart/'), 'Cart is empty')
        self.assertRedirects(self.client.get('/plata/checkout/'), '/plata/cart/?empty=1')
        self.assertRedirects(self.client.get('/plata/confirmation/'), '/plata/cart/?empty=1')
<commit_msg>Fetch shop instance later<commit_after>import os

from datetime import datetime
from decimal import Decimal

from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ValidationError
from django.test import TestCase

import plata
from plata import plata_settings
from plata.contact.models import Contact
from plata.product.models import TaxClass, Product, Discount
from plata.shop.models import Order, OrderStatus, OrderPayment

class ModelTest(TestCase):
    def assertRaisesWithCode(self, exception, fn, code):
        try:
            fn()
        except exception, e:
            if e.code == code:
                return True
            raise
        raise Exception, '%s did not raise %s' % (fn, exception)

    def setUp(self):
        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def create_contact(self):
        return Contact.objects.create(
            billing_company=u'BigCorp',
            billing_first_name=u'Hans',
            billing_last_name=u'Muster',
            billing_address=u'Musterstrasse 42',
            billing_zip_code=u'8042',
            billing_city=u'Beispielstadt',
            billing_country=u'CH',
            shipping_same_as_billing=True,
            currency='CHF',
            )

    def create_order(self, contact=None):
        contact = contact or self.create_contact()

        return Order.objects.create(
            contact=contact,
            currency='CHF',
            )

    def create_tax_classes(self):
        tax_class, created = TaxClass.objects.get_or_create(
            name='Standard Swiss Tax Rate',
            rate=Decimal('7.60'),
            )

        tax_class_germany, created = TaxClass.objects.get_or_create(
            name='Umsatzsteuer (Germany)',
            rate=Decimal('19.00'),
            )

        tax_class_something, created = TaxClass.objects.get_or_create(
            name='Some tax rate',
            rate=Decimal('12.50'),
            )

        return tax_class, tax_class_germany, tax_class_something

    def create_product(self):
        tax_class, tax_class_germany, tax_class_something = self.create_tax_classes()

        product = Product.objects.create(
            name='Test Product 1',
            )

        # An old price in CHF which should not influence the rest of the tests
        product.prices.create(
            currency='CHF',
            tax_class=tax_class,
            _unit_price=Decimal('99.90'),
            tax_included=True
            )

        product.prices.create(
            currency='CHF',
            tax_class=tax_class,
            _unit_price=Decimal('79.90'),
            tax_included=True,
            )

        product.prices.create(
            currency='EUR',
            tax_class=tax_class_germany,
            _unit_price=Decimal('49.90'),
            tax_included=True,
            )

        product.prices.create(
            currency='ASDF',
            tax_class=tax_class_something,
            _unit_price=Decimal('65.00'),
            tax_included=False,
            )

        return product

    def test_01_basic_order(self):
        product = self.create_product()
        order = self.create_order()

        self.assertEqual(product.get_price(currency=order.currency).currency, order.currency)

        order.modify_item(product, 5)
        order.modify_item(product, -4)
        item = order.modify_item(product, 1)

        self.assertEqual(order.items.count(), 1)

        self.assertEqual(item.quantity, 2)

        item_price = Decimal('79.90')
        line_item_price = item_price * item.quantity
        order_total = Decimal('159.80')
        tax_factor = Decimal('1.076')

        self.assertAlmostEqual(order.items_subtotal, order_total / tax_factor)
        self.assertAlmostEqual(order.items_subtotal + order.items_tax, order_total)
        self.assertAlmostEqual(order.total, order_total)

        self.assertAlmostEqual(item._unit_price, item_price / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal_incl_tax, line_item_price)
        self.assertAlmostEqual(item.discounted_subtotal_excl_tax, line_item_price / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal_incl_tax, line_item_price)

        self.assertAlmostEqual(item.unit_price, item_price)
        self.assertAlmostEqual(item.line_item_discount, 0)
        self.assertAlmostEqual(item.discounted_subtotal, item.discounted_subtotal_incl_tax)

        # Switch around tax handling and re-test
        plata_settings.PLATA_PRICE_INCLUDES_TAX = False

        self.assertAlmostEqual(item.unit_price, item_price / tax_factor)
        self.assertAlmostEqual(item.line_item_discount, 0 / tax_factor)
        self.assertAlmostEqual(item.discounted_subtotal, item.discounted_subtotal_excl_tax)

        # Switch tax handling back
        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_02_eur_order(self):
        product = self.create_product()
        order = self.create_order()

        order.currency = 'EUR'
        order.save()

        item = order.modify_item(product, 2)

        self.assertEqual(item.unit_price, Decimal('49.90'))
        self.assertEqual(item.currency, order.currency)

    def test_03_mixed_currencies(self):
        """
        Create an invalid order
        """

        p1 = self.create_product()
        p2 = self.create_product()
        order = self.create_order()

        order.currency = 'CHF'
        i1 = order.modify_item(p1, 3)

        order.currency = 'EUR'
        self.assertRaisesWithCode(ValidationError, lambda: order.modify_item(p2, 2),
            code='multiple_currency')

        # Validation should still fail
        self.assertRaisesWithCode(ValidationError, lambda: order.validate(),
            code='multiple_currency')

        order.currency = 'CHF'
        # Order should validate now
        order.validate()

    def test_04_order_modify_item(self):
        p1 = self.create_product()
        p2 = self.create_product()
        order = self.create_order()

        order.modify_item(p1, 42)
        order.modify_item(p2, 42)
        self.assertEqual(order.items.count(), 2)

        order.modify_item(p1, -42)
        self.assertEqual(order.items.count(), 1)

    def test_05_order_status(self):
        order = self.create_order()

        self.assertRaisesWithCode(ValidationError, lambda: order.update_status(
            Order.CHECKOUT,
            'Checkout process has started',
            ), code='order_empty')

        product = self.create_product()
        order.modify_item(product, 1)

        # Should be possible to update order status now
        order.update_status(
            Order.CHECKOUT,
            'Checkout process has started',
            )

        # Should not be possible to modify order once checkout process has started
        self.assertRaisesWithCode(ValidationError, lambda: order.modify_item(product, 2),
            code='order_sealed')

        self.assertEqual(order.status, Order.CHECKOUT)

    def test_06_order_percentage_discount(self):
        order = self.create_order()
        p1 = self.create_product()
        p2 = self.create_product()

        order.modify_item(p1, 3)
        order.modify_item(p2, 5)

        discount = Discount.objects.create(
            type=Discount.PERCENTAGE,
            key='asdf',
            name='Percentage discount',
            value=30)
        order.add_discount(discount)
        order.recalculate_total()

        tax_factor = Decimal('1.076')
        item_price_incl_tax = Decimal('79.90')
        item_price_excl_tax = item_price_incl_tax / tax_factor

        order.recalculate_total()
        item = order.modify_item(p1, 0)
        item2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(item.unit_price, item_price_incl_tax)
        self.assertAlmostEqual(item.line_item_discount, item_price_incl_tax * 3 * Decimal('0.30'))
        self.assertAlmostEqual(order.total,
            item.discounted_subtotal + item2.discounted_subtotal)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = False
        order.recalculate_total()
        item = order.modify_item(p1, 0)
        item2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(item.unit_price, item_price_excl_tax)
        self.assertAlmostEqual(item.line_item_discount, item_price_excl_tax * 3 * Decimal('0.30'))
        self.assertAlmostEqual(order.total,
            item.discounted_subtotal + item2.discounted_subtotal + order.items_tax)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_07_order_amount_discount(self):
        order = self.create_order()
        p1 = self.create_product()
        p2 = self.create_product()

        normal1 = order.modify_item(p1, 3)
        normal2 = order.modify_item(p2, 5)

        order.recalculate_total()
        self.assertAlmostEqual(order.total, Decimal('639.20'))

        discount = Discount.objects.create(
            type=Discount.AMOUNT_INCL_TAX,
            key='asdf',
            name='Amount discount',
            value=Decimal('50.00'))
        order.add_discount(discount)
        order.recalculate_total()

        discounted1 = order.modify_item(p1, 0)
        discounted2 = order.modify_item(p2, 0)

        tax_factor = Decimal('1.076')
        item_price_incl_tax = Decimal('79.90')
        item_price_excl_tax = item_price_incl_tax / tax_factor

        self.assertAlmostEqual(order.total, Decimal('639.20') - Decimal('50.00'))

        self.assertAlmostEqual(normal1.unit_price, discounted1.unit_price)
        self.assertAlmostEqual(normal2.unit_price, discounted2.unit_price)
        self.assertAlmostEqual(normal1.unit_price, item_price_incl_tax)

        self.assertEqual(normal1.line_item_discount, 0)
        self.assertEqual(normal2.line_item_discount, 0)

        self.assertAlmostEqual(discounted1.line_item_discount, Decimal('50.00') / 8 * 3)
        self.assertAlmostEqual(discounted2.line_item_discount, Decimal('50.00') / 8 * 5)

        self.assertAlmostEqual(discounted1.discounted_subtotal, order.total / 8 * 3)
        self.assertAlmostEqual(discounted2.discounted_subtotal, order.total / 8 * 5)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = False
        order.recalculate_total()
        discounted1 = order.modify_item(p1, 0)
        discounted2 = order.modify_item(p2, 0)

        self.assertAlmostEqual(order.total, Decimal('639.20') - Decimal('50.00'))

        self.assertAlmostEqual(discounted1.unit_price, item_price_excl_tax)
        self.assertAlmostEqual(discounted1.line_item_discount, discount.value / tax_factor / 8 * 3)
        self.assertAlmostEqual(order.total,
            discounted1.discounted_subtotal + discounted2.discounted_subtotal + order.items_tax)

        plata_settings.PLATA_PRICE_INCLUDES_TAX = True

    def test_08_order_payment(self):
        order = self.create_order()
        product = self.create_product()

        order.modify_item(product, 10)
        order.recalculate_total()

        payment = order.payments.create(
            currency='CHF',
            amount=Decimal('49.90'),
            payment_method='Mafia style',
            )

        order = Order.objects.get(pk=order.pk)
        self.assertAlmostEqual(order.paid, 0)

        payment.authorized = datetime.now()
        payment.save()

        order = Order.objects.get(pk=order.pk)
        self.assertAlmostEqual(order.balance_remaining, order.total - payment.amount)

class ShopTest(TestCase):
    def setUp(self):
        class Empty(object):
            pass

        self.request = Empty()
        self.request.session = {}
        self.request.user = AnonymousUser()

    def test_01_creation(self):
        shop = plata.shop_instance()

        contact = shop.contact_from_request(self.request)
        self.assertEqual(contact, None)

        contact = shop.contact_from_request(self.request, create=True)
        self.assertNotEqual(contact, None)

        contact = shop.contact_from_request(self.request, create=True)
        self.assertEqual(Contact.objects.count(), 1)

        order = shop.order_from_request(self.request)
        self.assertEqual(order, None)

        order = shop.order_from_request(self.request, create=True)
        self.assertEqual(Order.objects.count(), 1)
        self.assertEqual(order.contact, contact)

class ViewTest(TestCase):
    def setUp(self):
        self.ORIG_TEMPLATE_DIRS = settings.TEMPLATE_DIRS
        settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)

    def tearDown(self):
        settings.TEMPLATE_DIRS = self.ORIG_TEMPLATE_DIRS

    def test_01_cart_empty(self):
        self.assertContains(self.client.get('/plata/cart/'), 'Cart is empty')
        self.assertRedirects(self.client.get('/plata/checkout/'), '/plata/cart/?empty=1')
        self.assertRedirects(self.client.get('/plata/confirmation/'), '/plata/cart/?empty=1')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import time
import logging
import functools
from datetime import datetime
import numpy

from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, AccumDict
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import pmap_from_grp, poe_map
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib import valid, nrml
from openquake.commonlib import source, readinput, config
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base, classical
from openquake.calculators.ucerf_event_based import (
    UCERFControl, DEFAULT_TRT, UcerfSource)
# FIXME: the counting of effective ruptures has to be revised completely

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFControl(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])
SourceConverter.convert_UCERFSource = convert_UCERFSource

def _hazard_curves_per_rupture_subset(
        rupset_idx, ucerf_source, sites, imtls, cmaker,
        truncation_level=None, monitor=Monitor()):
    """
    Calculates the probabilities of exceedence from a set of rupture indices
    """
    imtls = DictArray(imtls)
    ctx_mon = monitor('making contexts', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    pmap = ProbabilityMap(len(imtls.array), len(cmaker.gsims))
    pmap.calc_times = []
    pmap.grp_id = ucerf_source.src_group_id
    nsites = len(sites)
    ucerf_source.rupset_idx = rupset_idx
    ucerf_source.num_ruptures = len(rupset_idx)
    pmap.eff_ruptures = {pmap.grp_id: ucerf_source.num_ruptures}
    # NB: the effective ruptures can be less, some may have zero probability
    t0 = time.time()
    upmap = poe_map(ucerf_source, sites, imtls, cmaker,
                    truncation_level, ctx_mon, pne_mon)
    pmap |= upmap
    pmap.calc_times.append(
        (ucerf_source.source_id, nsites, time.time() - t0))
    return pmap

def ucerf_classical_hazard_by_rupture_set(
        rupset_idx, ucerf_source, src_filter, gsims, monitor):
    """
    :param rupset_idx:
        indices of the rupture sets
    :param ucerf_source:
        an object taking the place of a source for UCERF
    :param src_filter:
        a source filter returning the sites affected by the source
    :param gsims:
        a list of GSIMs
    :param monitor:
        a monitor instance
    :returns:
        a ProbabilityMap
    """
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    max_dist = src_filter.integration_distance[DEFAULT_TRT]
    rupset_idx, s_sites = \
        ucerf_source.filter_sites_by_distance_from_rupture_set(
            rupset_idx, src_filter.sitecol, max_dist)
    if len(s_sites):
        ucerf_source.src_filter = src_filter  # so that .iter_ruptures() work
        cmaker = ContextMaker(gsims, max_dist)
        pm = _hazard_curves_per_rupture_subset(
            rupset_idx, ucerf_source, s_sites, imtls, cmaker,
            truncation_level, monitor=monitor)
    else:  # return an empty probability map
        pm = ProbabilityMap(len(imtls.array), len(gsims))
        pm.calc_times = []  # TODO: fix .calc_times
        pm.eff_ruptures = {ucerf_source.src_group_id: 0}
        pm.grp_id = ucerf_source.src_group_id
    return pm
ucerf_classical_hazard_by_rupture_set.shared_dir_on = config.SHARED_DIR_ON

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(classical.PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = ucerf_classical_hazard_by_rupture_set
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        self.sitecol = readinput.get_site_collection(self.oqparam)
        self.gsim_lt = readinput.get_gsim_lt(self.oqparam, [DEFAULT_TRT])
        self.smlt = readinput.get_source_model_lt(self.oqparam)
        parser = nrml.SourceModelParser(
            SourceConverter(self.oqparam.investigation_time,
                            self.oqparam.rupture_mesh_spacing))
        [self.src_group] = parser.parse_src_groups(
            self.oqparam.inputs["source_model"])
        source_models = []
        for sm in self.smlt.gen_source_models(self.gsim_lt):
            sg = copy.copy(self.src_group)
            sg.id = sm.ordinal
            sm.src_groups = [sg]
            # sm.path[0]=ltbrTrueMean, sm.name=FM0_0/MEANFS/MEANMSR/MeanRates
            sg.sources = [UcerfSource(sg[0], sm.ordinal, sm.path[0], sm.name)]
            source_models.append(sm)
        self.csm = source.CompositeSourceModel(
            self.gsim_lt, self.smlt, source_models, set_weight=True)
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
        self.rup_data = {}
        self.num_tiles = 1

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor.new(self.core_task.__name__)
        monitor.oqparam = oq = self.oqparam
        self.src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.rlzs_assoc.gsims_by_grp_id.items()})
        acc.calc_times = []
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility

        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.rlzs_assoc.gsims_by_grp_id[grp_id]
            [[ucerf_source]] = sm.src_groups
            ucerf_source.nsites = len(self.sitecol)
            self.csm.infos[grp_id, ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            logging.info('Getting the background point sources')
            with self.monitor('getting background sources', autoflush=True):
                background_sids = ucerf_source.get_background_sids(
                    self.src_filter)
                bckgnd_sources = ucerf_source.get_background_sources(
                    background_sids)

            # since there are two kinds of tasks (background and rupture_set)
            # we divide the concurrent_tasks parameter by 2;
            # notice the "or 1" below, to avoid issues when
            # self.oqparam.concurrent_tasks is 0 or 1
            ct2 = (self.oqparam.concurrent_tasks // 2) or 1

            # parallelize on the background sources, small tasks
            args = (bckgnd_sources, self.src_filter, oq.imtls,
                    gsims, self.oqparam.truncation_level, (), monitor)
            bg_res = parallel.Starmap.apply(
                pmap_from_grp, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=ct2).submit_all()

            # parallelize by rupture subsets
            rup_sets = numpy.arange(ucerf_source.num_ruptures)
            taskname = 'ucerf_classical_hazard_by_rupture_set_%d' % grp_id
            rup_res = parallel.Starmap.apply(
                ucerf_classical_hazard_by_rupture_set,
                (rup_sets, ucerf_source, self.src_filter, gsims, monitor),
                concurrent_tasks=ct2, name=taskname
            ).submit_all()

            # compose probabilities from background sources
            for pmap in bg_res:
                acc[grp_id] |= pmap
            self.save_data_transfer(bg_res)

            acc = functools.reduce(self.agg_dicts, rup_res, acc)
            with self.monitor('store source_info', autoflush=True):
                self.store_source_info(self.csm.infos)
                self.save_data_transfer(rup_res)
        self.datastore['csm_info'] = self.csm.info
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
            functools.partial(self.count_eff_ruptures, acc))
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(classical.ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<commit_msg>Fixed ucerf tests<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import time
import logging
import functools
from datetime import datetime
import numpy

from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, AccumDict
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import pmap_from_grp, poe_map
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib import valid, nrml
from openquake.commonlib import source, readinput, config
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base, classical
from openquake.calculators.ucerf_event_based import (
    UCERFControl, DEFAULT_TRT, UcerfSource)
# FIXME: the counting of effective ruptures has to be revised completely

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFControl(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])
SourceConverter.convert_UCERFSource = convert_UCERFSource

def _hazard_curves_per_rupture_subset(
        rupset_idx, ucerf_source, sites, imtls, cmaker,
        truncation_level=None, monitor=Monitor()):
    """
    Calculates the probabilities of exceedence from a set of rupture indices
    """
    imtls = DictArray(imtls)
    ctx_mon = monitor('making contexts', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    pmap = ProbabilityMap(len(imtls.array), len(cmaker.gsims))
    pmap.calc_times = []
    pmap.grp_id = ucerf_source.src_group_id
    nsites = len(sites)
    ucerf_source.rupset_idx = rupset_idx
    ucerf_source.num_ruptures = len(rupset_idx)
    pmap.eff_ruptures = {pmap.grp_id: ucerf_source.num_ruptures}
    # NB: the effective ruptures can be less, some may have zero probability
    t0 = time.time()
    upmap = poe_map(ucerf_source, sites, imtls, cmaker,
                    truncation_level, ctx_mon, pne_mon)
    pmap |= upmap
    pmap.calc_times.append(
        (ucerf_source.source_id, nsites, time.time() - t0))
    return pmap

def ucerf_classical_hazard_by_rupture_set(
        rupset_idx, ucerf_source, src_filter, gsims, monitor):
    """
    :param rupset_idx:
        indices of the rupture sets
    :param ucerf_source:
        an object taking the place of a source for UCERF
    :param src_filter:
        a source filter returning the sites affected by the source
    :param gsims:
        a list of GSIMs
    :param monitor:
        a monitor instance
    :returns:
        a ProbabilityMap
    """
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    max_dist = src_filter.integration_distance
    rupset_idx, s_sites = \
        ucerf_source.filter_sites_by_distance_from_rupture_set(
            rupset_idx, src_filter.sitecol, max_dist[DEFAULT_TRT])
    if len(s_sites):
        ucerf_source.src_filter = src_filter  # so that .iter_ruptures() work
        cmaker = ContextMaker(gsims, max_dist)
        pm = _hazard_curves_per_rupture_subset(
            rupset_idx, ucerf_source, s_sites, imtls, cmaker,
            truncation_level, monitor=monitor)
    else:  # return an empty probability map
        pm = ProbabilityMap(len(imtls.array), len(gsims))
        pm.calc_times = []  # TODO: fix .calc_times
        pm.eff_ruptures = {ucerf_source.src_group_id: 0}
        pm.grp_id = ucerf_source.src_group_id
    return pm
ucerf_classical_hazard_by_rupture_set.shared_dir_on = config.SHARED_DIR_ON

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(classical.PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = ucerf_classical_hazard_by_rupture_set
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        self.sitecol = readinput.get_site_collection(self.oqparam)
        self.gsim_lt = readinput.get_gsim_lt(self.oqparam, [DEFAULT_TRT])
        self.smlt = readinput.get_source_model_lt(self.oqparam)
        parser = nrml.SourceModelParser(
            SourceConverter(self.oqparam.investigation_time,
                            self.oqparam.rupture_mesh_spacing))
        [self.src_group] = parser.parse_src_groups(
            self.oqparam.inputs["source_model"])
        source_models = []
        for sm in self.smlt.gen_source_models(self.gsim_lt):
            sg = copy.copy(self.src_group)
            sg.id = sm.ordinal
            sm.src_groups = [sg]
            # sm.path[0]=ltbrTrueMean, sm.name=FM0_0/MEANFS/MEANMSR/MeanRates
            sg.sources = [UcerfSource(sg[0], sm.ordinal, sm.path[0], sm.name)]
            source_models.append(sm)
        self.csm = source.CompositeSourceModel(
            self.gsim_lt, self.smlt, source_models, set_weight=True)
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
        self.rup_data = {}
        self.num_tiles = 1

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor.new(self.core_task.__name__)
        monitor.oqparam = oq = self.oqparam
        self.src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.rlzs_assoc.gsims_by_grp_id.items()})
        acc.calc_times = []
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility

        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.rlzs_assoc.gsims_by_grp_id[grp_id]
            [[ucerf_source]] = sm.src_groups
            ucerf_source.nsites = len(self.sitecol)
            self.csm.infos[grp_id, ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            logging.info('Getting the background point sources')
            with self.monitor('getting background sources', autoflush=True):
                background_sids = ucerf_source.get_background_sids(
                    self.src_filter)
                bckgnd_sources = ucerf_source.get_background_sources(
                    background_sids)

            # since there are two kinds of tasks (background and rupture_set)
            # we divide the concurrent_tasks parameter by 2;
            # notice the "or 1" below, to avoid issues when
            # self.oqparam.concurrent_tasks is 0 or 1
            ct2 = (self.oqparam.concurrent_tasks // 2) or 1

            # parallelize on the background sources, small tasks
            args = (bckgnd_sources, self.src_filter, oq.imtls,
                    gsims, self.oqparam.truncation_level, (), monitor)
            bg_res = parallel.Starmap.apply(
                pmap_from_grp, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=ct2).submit_all()

            # parallelize by rupture subsets
            rup_sets = numpy.arange(ucerf_source.num_ruptures)
            taskname = 'ucerf_classical_hazard_by_rupture_set_%d' % grp_id
            rup_res = parallel.Starmap.apply(
                ucerf_classical_hazard_by_rupture_set,
                (rup_sets, ucerf_source, self.src_filter, gsims, monitor),
                concurrent_tasks=ct2, name=taskname
            ).submit_all()

            # compose probabilities from background sources
            for pmap in bg_res:
                acc[grp_id] |= pmap
            self.save_data_transfer(bg_res)

            acc = functools.reduce(self.agg_dicts, rup_res, acc)
            with self.monitor('store source_info', autoflush=True):
                self.store_source_info(self.csm.infos)
                self.save_data_transfer(rup_res)
        self.datastore['csm_info'] = self.csm.info
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
            functools.partial(self.count_eff_ruptures, acc))
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(classical.ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<|endoftext|>


<commit_before>import utilities
import subprocess
import datetime
import numpy as np
import rasterio

def forest_age_category(tile_id):

    upload_dir = 's3://gfw2-data/climate/carbon_model/forest_age_category/20180912/'

    print "Processing:", tile_id

    ymax, xmin, ymin, xmax = utilities.coords(tile_id)
    print "  ymax: ", ymax

    tropics = 0

    if ymax > -30 & ymax <= 30:

        tropics = 1

    # start time
    start = datetime.datetime.now()

    # Names of the loss, gain and tree cover density tiles
    loss = '{}.tif'.format(tile_id)
    gain = 'Hansen_GFC2015_gain_{}.tif'.format(tile_id)
    tcd = 'Hansen_GFC2014_treecover2000_{}.tif'.format(tile_id)
    ifl = '{}_res_ifl_2000tif'.format(tile_id)
    biomass = '{}_biomass.tif'.format(tile_id)
    cont_eco = 'fao_ecozones_{0}.tif'.format(tile_id)

    print "  Reading input files and evaluating condtions"

    # open loss and grab metadata about the tif,
    # like its location / projection / cellsize
    with rasterio.open(loss) as loss_src:
        kwargs = loss_src.meta

        # also grab the windows (stripes) so we can iterate
        # over the entire tif without running out of memory
        windows = loss_src.block_windows(1)

        # open gain
        with rasterio.open(gain) as gain_src:
            # open extent
            with rasterio.open(tcd) as extent_src:
                # update those kwargs for the dataset we'll write out
                kwargs.update(
                    driver='GTiff',
                    count=1,
                    compress='lzw',
                    nodata=0
                )

                with rasterio.open('forest_age_category_{}.tif'.format(tile_id), 'w', **kwargs) as dst:
                    for idx, window in windows:
                        loss = loss_src.read(1, window=window)
                        gain = gain_src.read(1, window=window)
                        tcd = extent_src.read(1, window=window)

                        # create an empty array
                        dst_data = np.zeros((window.height, window.width), dtype='uint8')

                        # where loss & gain, set output to 100, otherwise keep dst_data value
                        dst_data[np.where((tcd > 0) & (loss >= 0) & (gain >= 0))] = 100

                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 0))] = 1
                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 1) & (ifl == 0))] = 2
                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 1) & (ifl == 1))] = 3

                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss > 0) & (ifl == 1))] = 6

                        # # where loss & gain, set output to 100, otherwise keep dst_data value
                        # dst_data[np.where((loss_data > 0) & (gain_data > 0))] = 100
                        #
                        # # etc
                        # dst_data[np.where((loss_data > 10) & (gain_data > 0) & (extent_data > 50))] = 99
                        # dst_data[np.where((loss_data < 10) & (gain_data == 0) & (extent_data < 50))] = 1
                        # dst_data[np.where((loss_data > 10) & (gain_data > 0) & (extent_data >= 90))] = 2

                        dst.write_band(1, dst_data, window=window)

    pattern = 'forest_age_category'

    utilities.upload_final(pattern, upload_dir, tile_id)

    end = datetime.datetime.now()
    elapsed_time = end-start

    print "Processing time for tile", tile_id, ":", elapsed_time

<commit_msg>Working on script to create forest age category raster using rasterio.<commit_after>import utilities
import subprocess
import datetime
import numpy as np
import rasterio

def forest_age_category(tile_id):

    upload_dir = 's3://gfw2-data/climate/carbon_model/forest_age_category/20180912/'

    print "Processing:", tile_id

    ymax, xmin, ymin, xmax = utilities.coords(tile_id)
    print "  ymax: ", ymax

    tropics = 0

    if ymax > -30 & ymax <= 30:

        tropics = 1

    print tropics

    # start time
    start = datetime.datetime.now()

    # Names of the loss, gain and tree cover density tiles
    loss = '{}.tif'.format(tile_id)
    gain = 'Hansen_GFC2015_gain_{}.tif'.format(tile_id)
    tcd = 'Hansen_GFC2014_treecover2000_{}.tif'.format(tile_id)
    ifl = '{}_res_ifl_2000tif'.format(tile_id)
    biomass = '{}_biomass.tif'.format(tile_id)
    cont_eco = 'fao_ecozones_{0}.tif'.format(tile_id)

    print "  Reading input files and evaluating condtions"

    # open loss and grab metadata about the tif,
    # like its location / projection / cellsize
    with rasterio.open(loss) as loss_src:
        kwargs = loss_src.meta

        # also grab the windows (stripes) so we can iterate
        # over the entire tif without running out of memory
        windows = loss_src.block_windows(1)

        # open gain
        with rasterio.open(gain) as gain_src:
            # open extent
            with rasterio.open(tcd) as extent_src:
                # update those kwargs for the dataset we'll write out
                kwargs.update(
                    driver='GTiff',
                    count=1,
                    compress='lzw',
                    nodata=0
                )

                with rasterio.open('forest_age_category_{}.tif'.format(tile_id), 'w', **kwargs) as dst:
                    for idx, window in windows:
                        loss = loss_src.read(1, window=window)
                        gain = gain_src.read(1, window=window)
                        tcd = extent_src.read(1, window=window)

                        # create an empty array
                        dst_data = np.zeros((window.height, window.width), dtype='uint8')

                        # where loss & gain, set output to 100, otherwise keep dst_data value
                        dst_data[np.where((tcd > 0) & (loss >= 0) & (gain >= 0))] = 100

                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 0))] = 1
                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 1) & (ifl == 0))] = 2
                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (tropics == 1) & (ifl == 1))] = 3

                        dst_data[np.where((tcd > 0) & (gain == 0) & (loss > 0) & (ifl == 1))] = 6

                        # # where loss & gain, set output to 100, otherwise keep dst_data value
                        # dst_data[np.where((loss_data > 0) & (gain_data > 0))] = 100
                        #
                        # # etc
                        # dst_data[np.where((loss_data > 10) & (gain_data > 0) & (extent_data > 50))] = 99
                        # dst_data[np.where((loss_data < 10) & (gain_data == 0) & (extent_data < 50))] = 1
                        # dst_data[np.where((loss_data > 10) & (gain_data > 0) & (extent_data >= 90))] = 2

                        dst.write_band(1, dst_data, window=window)

    pattern = 'forest_age_category'

    utilities.upload_final(pattern, upload_dir, tile_id)

    end = datetime.datetime.now()
    elapsed_time = end-start

    print "Processing time for tile", tile_id, ":", elapsed_time

<|endoftext|>