<commit_before># -*- coding: utf-8 -*-
import sys
import os
import alabaster

sys.path.insert(0, os.path.abspath(os.path.join(os.path.split(__file__)[0])))
local_template = os.path.join(os.path.abspath(
    os.path.dirname(__file__)), "phdoc_templates")

from pyquickhelper.helpgen.default_conf import set_sphinx_variables, get_default_stylesheet
set_sphinx_variables(__file__, "cpyquickhelper", "Xavier Dupré", 2021,
                     "alabaster", alabaster.get_path(),
                     locals(), extlinks=dict(
                         issue=('https://github.com/sdpython/cpyquickhelper/issues/%s', 'issue')),
                     title="cpyquickhelper", book=True)

blog_root = "http://www.xavierdupre.fr/app/cpyquickhelper/helpsphinx/"

html_context = {
    'css_files': get_default_stylesheet() + ['_static/my-styles.css'],
}

html_logo = "phdoc_static/project_ico.png"

html_sidebars = {}

language = "en"
custom_preamble = """\n
\\newcommand{\\vecteur}[2]{\\pa{#1,\\dots,#2}}
\\newcommand{\\N}[0]{\\mathbb{N}}
\\newcommand{\\indicatrice}[1]{\\mathbf{1\\!\\!1}_{\\acc{#1}}}
\\usepackage[all]{xy}
\\newcommand{\\infegal}[0]{\\leqslant}
\\newcommand{\\supegal}[0]{\\geqslant}
\\newcommand{\\ensemble}[2]{\\acc{#1,\\dots,#2}}
\\newcommand{\\fleche}[1]{\\overrightarrow{ #1 }}
\\newcommand{\\intervalle}[2]{\\left\\{#1,\\cdots,#2\\right\\}}
\\newcommand{\\loinormale}[2]{{\\cal N}\\pa{#1,#2}}
\\newcommand{\\independant}[0]{\\;\\makebox[3ex]{\\makebox[0ex]{\\rule[-0.2ex]{3ex}{.1ex}}\\!\\!\\!\\!\\makebox[.5ex][l]{\\rule[-.2ex]{.1ex}{2ex}}\\makebox[.5ex][l]{\\rule[-.2ex]{.1ex}{2ex}}} \\,\\,}
\\newcommand{\\esp}{\\mathbb{E}}
\\newcommand{\\var}{\\mathbb{V}}
\\newcommand{\\pr}[1]{\\mathbb{P}\\pa{#1}}
\\newcommand{\\loi}[0]{{\\cal L}}
\\newcommand{\\vecteurno}[2]{#1,\\dots,#2}
\\newcommand{\\norm}[1]{\\left\\Vert#1\\right\\Vert}
\\newcommand{\\dans}[0]{\\rightarrow}
\\newcommand{\\partialfrac}[2]{\\frac{\\partial #1}{\\partial #2}}
\\newcommand{\\partialdfrac}[2]{\\dfrac{\\partial #1}{\\partial #2}}
\\newcommand{\\loimultinomiale}[1]{{\\cal M}\\pa{#1}}
\\newcommand{\\trace}[1]{tr\\pa{#1}}
\\newcommand{\\abs}[1]{\\left|#1\\right|}
"""
# \\usepackage{eepic}

imgmath_latex_preamble += custom_preamble
latex_elements['preamble'] += custom_preamble
mathdef_link_only = True

epkg_dictionary.update({
    'AVX': 'https://en.wikipedia.org/wiki/Advanced_Vector_Extensions',
    'BLAS': 'http://www.netlib.org/blas/explore-html',
    'C': 'https://en.wikipedia.org/wiki/C_(programming_language)',
    'C++': 'https://en.wikipedia.org/wiki/C%2B%2B',
    'DataFrame': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html',
    'ExtensionArray': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.api.extensions.ExtensionArray.html',
    'JIT': 'https://en.wikipedia.org/wiki/Just-in-time_compilation',
    'LAPACK': 'http://www.netlib.org/lapack/explore-html',
    'llvmlite': 'https://llvmlite.readthedocs.io/en/latest/index.html',
    'nuitka': 'http://nuitka.net/',
    'numba': 'https://numba.pydata.org/',
    'PandasArray': 'https://pandas.pydata.org/docs/reference/api/pandas.arrays.PandasArray.html',
    'pybind11': 'https://github.com/pybind/pybind11',
    'Pythran': 'https://pythran.readthedocs.io/en/latest/',
    'pythran': 'https://pythran.readthedocs.io/en/latest/',
    'Series': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.html',
    'SSE': 'https://fr.wikipedia.org/wiki/Streaming_SIMD_Extensions',
})
<commit_msg>documentation<commit_after># -*- coding: utf-8 -*-
import sys
import os
import alabaster

sys.path.insert(0, os.path.abspath(os.path.join(os.path.split(__file__)[0])))
local_template = os.path.join(os.path.abspath(
    os.path.dirname(__file__)), "phdoc_templates")

from pyquickhelper.helpgen.default_conf import set_sphinx_variables, get_default_stylesheet
set_sphinx_variables(__file__, "cpyquickhelper", "Xavier Dupré", 2021,
                     "alabaster", alabaster.get_path(),
                     locals(), extlinks=dict(
                         issue=('https://github.com/sdpython/cpyquickhelper/issues/%s', 'issue')),
                     title="cpyquickhelper", book=True)

blog_root = "http://www.xavierdupre.fr/app/cpyquickhelper/helpsphinx/"

html_context = {
    'css_files': get_default_stylesheet(['_static/my-styles.css']),
}

html_logo = "phdoc_static/project_ico.png"

html_sidebars = {}

language = "en"
custom_preamble = """\n
\\newcommand{\\vecteur}[2]{\\pa{#1,\\dots,#2}}
\\newcommand{\\N}[0]{\\mathbb{N}}
\\newcommand{\\indicatrice}[1]{\\mathbf{1\\!\\!1}_{\\acc{#1}}}
\\usepackage[all]{xy}
\\newcommand{\\infegal}[0]{\\leqslant}
\\newcommand{\\supegal}[0]{\\geqslant}
\\newcommand{\\ensemble}[2]{\\acc{#1,\\dots,#2}}
\\newcommand{\\fleche}[1]{\\overrightarrow{ #1 }}
\\newcommand{\\intervalle}[2]{\\left\\{#1,\\cdots,#2\\right\\}}
\\newcommand{\\loinormale}[2]{{\\cal N}\\pa{#1,#2}}
\\newcommand{\\independant}[0]{\\;\\makebox[3ex]{\\makebox[0ex]{\\rule[-0.2ex]{3ex}{.1ex}}\\!\\!\\!\\!\\makebox[.5ex][l]{\\rule[-.2ex]{.1ex}{2ex}}\\makebox[.5ex][l]{\\rule[-.2ex]{.1ex}{2ex}}} \\,\\,}
\\newcommand{\\esp}{\\mathbb{E}}
\\newcommand{\\var}{\\mathbb{V}}
\\newcommand{\\pr}[1]{\\mathbb{P}\\pa{#1}}
\\newcommand{\\loi}[0]{{\\cal L}}
\\newcommand{\\vecteurno}[2]{#1,\\dots,#2}
\\newcommand{\\norm}[1]{\\left\\Vert#1\\right\\Vert}
\\newcommand{\\dans}[0]{\\rightarrow}
\\newcommand{\\partialfrac}[2]{\\frac{\\partial #1}{\\partial #2}}
\\newcommand{\\partialdfrac}[2]{\\dfrac{\\partial #1}{\\partial #2}}
\\newcommand{\\loimultinomiale}[1]{{\\cal M}\\pa{#1}}
\\newcommand{\\trace}[1]{tr\\pa{#1}}
\\newcommand{\\abs}[1]{\\left|#1\\right|}
"""
# \\usepackage{eepic}

imgmath_latex_preamble += custom_preamble
latex_elements['preamble'] += custom_preamble
mathdef_link_only = True

epkg_dictionary.update({
    'AVX': 'https://en.wikipedia.org/wiki/Advanced_Vector_Extensions',
    'BLAS': 'http://www.netlib.org/blas/explore-html',
    'C': 'https://en.wikipedia.org/wiki/C_(programming_language)',
    'C++': 'https://en.wikipedia.org/wiki/C%2B%2B',
    'DataFrame': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html',
    'ExtensionArray': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.api.extensions.ExtensionArray.html',
    'JIT': 'https://en.wikipedia.org/wiki/Just-in-time_compilation',
    'LAPACK': 'http://www.netlib.org/lapack/explore-html',
    'llvmlite': 'https://llvmlite.readthedocs.io/en/latest/index.html',
    'nuitka': 'http://nuitka.net/',
    'numba': 'https://numba.pydata.org/',
    'PandasArray': 'https://pandas.pydata.org/docs/reference/api/pandas.arrays.PandasArray.html',
    'pybind11': 'https://github.com/pybind/pybind11',
    'Pythran': 'https://pythran.readthedocs.io/en/latest/',
    'pythran': 'https://pythran.readthedocs.io/en/latest/',
    'Series': 'https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.html',
    'SSE': 'https://fr.wikipedia.org/wiki/Streaming_SIMD_Extensions',
})
<|endoftext|>


<commit_before>############################################################################
##
## Copyright (C) 2006-2007 University of Utah. All rights reserved.
##
## This file is part of VisTrails.
##
## This file may be used under the terms of the GNU General Public
## License version 2.0 as published by the Free Software Foundation
## and appearing in the file LICENSE.GPL included in the packaging of
## this file.  Please review the following to ensure GNU General Public
## Licensing requirements will be met:
## http://www.opensource.org/licenses/gpl-license.php
##
## If you are unsure which license is appropriate for your use (for
## instance, you are interested in developing a commercial derivative
## of VisTrails), please contact us at vistrails@sci.utah.edu.
##
## This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
## WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
##
############################################################################
import core.modules
import core.modules.module_registry
from core.modules.vistrails_module import Module, ModuleError
from SciPy import SciPy
from Matrix import *
from scipy import sparse, fft

#################################################################

class DSP(SciPy):
    def compute(self):
        pass

class FFT(DSP):
    def compute(self):
        mat = self.getInputFromPort("Signals")
        pts = self.getInputFromPort("FFT Samples")
        if hasattr(fft,"fft"):
            phasors = fft.fft(mat.matrix.data, pts)
        else:
            phasors = fft(mat.matrix.data, pts)
        outmat = sparse.csc_matrix(phasors)
        out = SparseMatrix()
        out.matrix = outmat
        self.setResult("FFT Output", out)
<commit_msg>Added support for 2D FFT and automatic Windowed Fourier Transform.  The 2D FFT module is FFT2 and provides output in the form of a 2D matrix of the appropriate size.  THe WindowedFourierTransform module takes as inputs the Signal and Sampling Rate of the input and outputs a SparseMatrix where the rows are the timeslices and the columns are the FFT output for that timeslice.  If the user wants more control over this, there are optional inputs Window Size and Stride.  The Window Size input controls the size of the sliding window and the Stride input controls the number of samples the window moves per time-slice.  The default to the sampling rate and the samplingrate/2 respectively.<commit_after>############################################################################
##
## Copyright (C) 2006-2007 University of Utah. All rights reserved.
##
## This file is part of VisTrails.
##
## This file may be used under the terms of the GNU General Public
## License version 2.0 as published by the Free Software Foundation
## and appearing in the file LICENSE.GPL included in the packaging of
## this file.  Please review the following to ensure GNU General Public
## Licensing requirements will be met:
## http://www.opensource.org/licenses/gpl-license.php
##
## If you are unsure which license is appropriate for your use (for
## instance, you are interested in developing a commercial derivative
## of VisTrails), please contact us at vistrails@sci.utah.edu.
##
## This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
## WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
##
############################################################################
import core.modules
import core.modules.module_registry
from core.modules.vistrails_module import Module, ModuleError
from SciPy import SciPy
from Matrix import *
from scipy import sparse, fftpack
import numpy

#################################################################

class DSP(SciPy):
    def compute(self):
        pass

class FFT(DSP):
    def compute(self):
        mat = self.getInputFromPort("Signals")
        pts = self.getInputFromPort("FFT Samples")

        phasors = fftpack.fft(mat.matrix.data, pts)
        outmat = sparse.csc_matrix(phasors)
        out = SparseMatrix()
        out.matrix = outmat
        self.setResult("FFT Output", out)

class FFT2(DSP):
    def compute(self):
        mat = self.getInputFromPort("Signals")

        phasors = fftpack.fftn(mat.matrix.data)
        outmat = sparse.csc_matrix(phasors)
        out = SparseMatrix()
        out.matrix = outmat
        self.setResult("FFT Output", out)

class WindowedFourierTransform(DSP):
    def compute(self):
        mat = self.getInputFromPort("Signal")

        sr = self.getInputFromPort("Sampling Rate")
        if self.hasInputFromPort("Window Size"):
            window = self.getInputFromPort("Window Size")
        else:
            window = sr

        if self.hasInputFromPort("Stride"):
            stride = self.getInputFromPort("Stride")
        else:
            stride = int(sr / 2)

        signal_array = mat.matrix.toarray().ravel()

        # We now have a 1-D array that we can have good indexing into
        pad = signal_array[0:int(window/2)]
        signal_array = numpy.concatenate((pad,signal_array))
        win_low = 0
        win_hi = window - 1
        phasors = fftpack.fft(signal_array[win_low:win_hi])
        out_array = phasors.ravel()

        win_low += stride
        win_hi += stride

        while win_hi < signal_array.shape[0]:
            phasors = fftpack.fft(signal_array[win_low:win_hi])
            win_low += stride
            win_hi += stride
            out_array = numpy.vstack([out_array, phasors.ravel()])

        out = SparseMatrix()
        out.matrix = sparse.csc_matrix(out_array)
        self.setResult("FFT Output", out)
<|endoftext|>


<commit_before>import models

from django import forms
from django.forms.models import modelformset_factory
from django.forms.formsets import BaseFormSet
from django.utils.translation import ugettext as _

source_fields = ('name', 'publisher', 'frequency', 'category',
                 'sub_category', 'suggested_by', 'open_license', 'comment')

class SourceForm(forms.ModelForm):
    open_license = forms.BooleanField(
        required=False,
        help_text=_('Is text distributed under open license?')
    )

    comment = forms.CharField(widget=forms.Textarea(attrs={'rows': 2}),
                              required=False)

    class Meta:
        model = models.Source
        fields = source_fields

class ManagementSourceForm(SourceForm):
    """
    This is pretty much the same as SourceForm with the difference that it
    allows to select owner=curator of the source.
    """

    class Meta:
        model = models.Source
        fields = ('owner',) + source_fields

class DuplicityForm(forms.Form):
    """
        This is very simple form that requires user to check that he
        is not creating duplicities.
    """
    unique_record = forms.BooleanField(
        required=True,
        help_text=_('Check if this is really unique source.'))

class BaseSeedFormset(BaseFormSet):
    def clean(self):
        """
        Checks that there is always at least one valid seed
        """
        urls = [frm.cleaned_data.get('url', None) for frm in self.forms]
        if not any(urls):
            raise forms.ValidationError(_('There must be at least one seed!'))

SeedFormset = modelformset_factory(models.Seed, fields=('url',), extra=7,
                                   formset=BaseSeedFormset)

class SourceEditForm(forms.ModelForm):
    class Meta:
        model = models.Source
        fields = ('name', 'state', 'frequency', 'category', 'sub_category',
                  'comment', 'aleph_id')
<commit_msg>fixing #77<commit_after>import models

from django import forms
from django.forms.models import modelformset_factory
from django.forms.formsets import BaseFormSet
from django.utils.translation import ugettext as _

source_fields = ('name', 'publisher', 'frequency', 'category',
                 'sub_category', 'suggested_by', 'open_license', 'comment')

class SourceForm(forms.ModelForm):
    open_license = forms.BooleanField(
        required=False,
        help_text=_('Is text distributed under open license?')
    )

    comment = forms.CharField(widget=forms.Textarea(attrs={'rows': 2}),
                              required=False)

    class Meta:
        model = models.Source
        fields = source_fields

class ManagementSourceForm(SourceForm):
    """
    This is pretty much the same as SourceForm with the difference that it
    allows to select owner=curator of the source.
    """

    class Meta:
        model = models.Source
        fields = ('owner',) + source_fields

class DuplicityForm(forms.Form):
    """
        This is very simple form that requires user to check that he
        is not creating duplicities.
    """
    unique_record = forms.BooleanField(
        required=True,
        help_text=_('Check if this is really unique source.'))

class BaseSeedFormset(BaseFormSet):
    def clean(self):
        """
        Checks that there is always at least one valid seed
        """
        urls = [frm.cleaned_data.get('url', None) for frm in self.forms]
        valid_urls = filter(None, urls)
        if not valid_urls:
            raise forms.ValidationError(_('There must be at least one seed.'))
        if len(set(valid_urls)) < len(valid_urls):
            raise forms.ValidationError(_('All urls must be unique.'))

SeedFormset = modelformset_factory(models.Seed, fields=('url',), extra=7,
                                   formset=BaseSeedFormset)

class SourceEditForm(forms.ModelForm):
    class Meta:
        model = models.Source
        fields = ('name', 'state', 'frequency', 'category', 'sub_category',
                  'comment', 'aleph_id')
<|endoftext|>


<commit_before># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
import json
import numpy
import os

from django.contrib.gis import geos

from openquake import kvs
from openquake.db import models
from openquake.shapes import Site, GridPoint
from openquake.engine import JobContext
from openquake.db.models import DmgDistPerAsset, DmgDistPerAssetData
from openquake.kvs.tokens import ground_motion_values_key
from openquake.calculators.risk.general import Block
from openquake.calculators.risk.scenario_damage.core import (
    ScenarioDamageRiskCalculator, compute_gmv_fractions)

from tests.utils import helpers

BLOCK_ID = 1

class ScenarioDamageRiskCalculatorTestCase(
    unittest.TestCase, helpers.DbTestCase):

    def setUp(self):
        inputs = [("fragility", ""), ("exposure", "")]
        self.job = self.setup_classic_job(inputs=inputs)

        kvs.mark_job_as_current(self.job.id)
        kvs.cache_gc(self.job.id)

        self.site = Site(1.0, 1.0)
        block = Block(self.job.id, BLOCK_ID, [self.site])
        block.to_kvs()

        # this region contains a single site, that is exactly
        # a site with longitude == 1.0 and latitude == 1.0
        params = {"REGION_VERTEX": "1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0",
                "REGION_GRID_SPACING": "0.5", "BASE_PATH": ".",
                "OUTPUT_DIR": "."}

        self.job_ctxt = JobContext(params, self.job.id, oq_job=self.job)

        self.em = self._store_em()
        self._store_gmvs([0.40, 0.30, 0.45, 0.35, 0.40])

        self.calculator = ScenarioDamageRiskCalculator(self.job_ctxt)

        # just stubbing out some preprocessing stuff...
        ScenarioDamageRiskCalculator.store_exposure_assets = lambda self: None
        ScenarioDamageRiskCalculator.store_fragility_model = lambda self: None
        ScenarioDamageRiskCalculator.partition = lambda self: None

    def test_compute_risk_dda_con(self):
        # test the damage distribution per asset with a continuous
        # fragility model
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        [dda] = DmgDistPerAsset.objects.filter(output__oq_job=self.job.id,
                output__output_type="dmg_dist_per_asset")

        self.assertEquals(["no_damage", "LS1", "LS2"], dda.dmg_states)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="A")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(1.00563, data.mean)
        self._close_to(1.61341, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(34.80851, data.mean)
        self._close_to(18.34906, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(64.18586, data.mean)
        self._close_to(19.83963, data.stddev)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="B")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(3.64527, data.mean)
        self._close_to(3.35330, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(17.10494, data.mean)
        self._close_to(4.63604, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(19.24979, data.mean)
        self._close_to(7.78725, data.stddev)

    def test_compute_risk_dda_dsc(self):
        # test the damage distribution per asset with a discrete
        # fragility model
        fm = self._store_dsc_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        [dda] = DmgDistPerAsset.objects.filter(output__oq_job=self.job.id,
                output__output_type="dmg_dist_per_asset")

        self.assertEquals(["no_damage", "LS1", "LS2"], dda.dmg_states)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="A")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(68.0, data.mean)
        self._close_to(8.5513157, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(21.0, data.mean)
        self._close_to(4.2756578, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(11.0, data.mean)
        self._close_to(4.2756578, data.stddev)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="B")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(30.4, data.mean)
        self._close_to(3.4, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(3.9, data.mean)
        self._close_to(1.4, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(5.7, data.mean)
        self._close_to(2.1, data.stddev)

    def test_dda_iml_above_range(self):
        # corner case where we have a ground motion value
        # (that corresponds to the intensity measure level in the
        # fragility function) that is higher than the highest
        # intensity measure level defined in the model (in this
        # particular case 0.7). Given this condition, to compute
        # the fractions of buildings we use the highest intensity
        # measure level defined in the model (0.7 in this case)

        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1"], format="discrete")

        fmodel.save()

        func = models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1)

        func.save()

        self._close_to(compute_gmv_fractions([func], 0.7),
                compute_gmv_fractions([func], 0.8))

    def test_dda_iml_below_range(self):
        # corner case where we have a ground motion value
        # (that corresponds to the intensity measure level in the
        # fragility function) that is lower than the lowest
        # intensity measure level defined in the model (in this
        # particular case 0.1). Given this condition, the
        # fractions of buildings is 100% no_damage and 0% for the
        # remaining limit states defined in the model

        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1"], format="discrete")

        fmodel.save()

        func = models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1)

        func.save()

        self._close_to([1.0, 0.0],
                compute_gmv_fractions([func], 0.05))

    def test_post_execute_serialization(self):
        # when --output-type=xml is specified, we serialize results
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        self.job_ctxt.serialize_results_to = ["xml"]

        self.calculator.post_execute()
        file_path = self._results_file()

        self.assertTrue(os.path.exists(file_path))
        os.unlink(file_path)

    def test_post_execute_no_serialization(self):
        # otherwise, just on database (default)
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        self.calculator.post_execute()
        file_path = self._results_file()

        self.assertFalse(os.path.exists(file_path))

    def _results_file(self):
        target_dir = os.path.join(self.job_ctxt.params.get("BASE_PATH"),
                self.job_ctxt.params.get("OUTPUT_DIR"))

        return os.path.join(target_dir, "dmg-dist-asset-%s.xml" % self.job.id)

    def _close_to(self, expected, actual):
        self.assertTrue(numpy.allclose(actual, expected, atol=0.0, rtol=0.05))

    def _store_gmvs(self, gmvs):
        client = kvs.get_client()
        encoder = json.JSONEncoder()

        key = ground_motion_values_key(self.job.id, GridPoint(None, 0, 0))

        for gmv in gmvs:
            client.rpush(key, encoder.encode({"mag": gmv}))

    def _store_con_fmodel(self):
        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism,
            lss=["LS1", "LS2"], format="continuous")

        fmodel.save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS2", mean="0.35", stddev="0.10", lsi=2).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", mean="0.20", stddev="0.05", lsi=1).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS2", mean="0.40", stddev="0.12", lsi=2).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS1", mean="0.25", stddev="0.08", lsi=1).save()

        return fmodel

    def _store_dsc_fmodel(self):
        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1", "LS2"], format="discrete")

        fmodel.save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS2", poes=[0.00, 0.05, 0.20, 0.50], lsi=2).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS2", poes=[0.02, 0.07, 0.25, 0.60], lsi=2).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS1", poes=[0.03, 0.12, 0.42, 0.90], lsi=1).save()

        return fmodel

    def _store_em(self):
        [ism] = models.inputs4job(self.job.id, input_type="exposure")

        em = models.ExposureModel(
            owner=ism.owner, input=ism,
            name="AAA", category="single_asset",
            reco_type="aggregated", reco_unit="USD",
            stco_type="aggregated", stco_unit="USD")

        em.save()

        models.ExposureData(
            exposure_model=em, taxonomy="RC",
            asset_ref="A", number_of_units=100, stco=1,
            site=geos.GEOSGeometry(self.site.point.to_wkt()), reco=1).save()

        models.ExposureData(
            exposure_model=em, taxonomy="RM",
            asset_ref="B", number_of_units=40, stco=1,
            site=geos.GEOSGeometry(self.site.point.to_wkt()), reco=1).save()

        return em
<commit_msg>Added test to check distribution per taxonomy export in post_execute()<commit_after># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest
import json
import numpy
import os

from django.contrib.gis import geos

from openquake import kvs
from openquake.db import models
from openquake.shapes import Site, GridPoint
from openquake.engine import JobContext
from openquake.db.models import DmgDistPerAsset, DmgDistPerAssetData
from openquake.kvs.tokens import ground_motion_values_key
from openquake.calculators.risk.general import Block
from openquake.calculators.risk.scenario_damage.core import (
    ScenarioDamageRiskCalculator, compute_gmv_fractions)

from tests.utils import helpers

BLOCK_ID = 1

class ScenarioDamageRiskCalculatorTestCase(
    unittest.TestCase, helpers.DbTestCase):

    def setUp(self):
        inputs = [("fragility", ""), ("exposure", "")]
        self.job = self.setup_classic_job(inputs=inputs)

        kvs.mark_job_as_current(self.job.id)
        kvs.cache_gc(self.job.id)

        self.site = Site(1.0, 1.0)
        block = Block(self.job.id, BLOCK_ID, [self.site])
        block.to_kvs()

        # this region contains a single site, that is exactly
        # a site with longitude == 1.0 and latitude == 1.0
        params = {"REGION_VERTEX": "1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0",
                "REGION_GRID_SPACING": "0.5", "BASE_PATH": ".",
                "OUTPUT_DIR": "."}

        self.job_ctxt = JobContext(params, self.job.id, oq_job=self.job)

        self.em = self._store_em()
        self._store_gmvs([0.40, 0.30, 0.45, 0.35, 0.40])

        self.calculator = ScenarioDamageRiskCalculator(self.job_ctxt)

        # just stubbing out some preprocessing stuff...
        ScenarioDamageRiskCalculator.store_exposure_assets = lambda self: None
        ScenarioDamageRiskCalculator.store_fragility_model = lambda self: None
        ScenarioDamageRiskCalculator.partition = lambda self: None

    def test_compute_risk_dda_con(self):
        # test the damage distribution per asset with a continuous
        # fragility model
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        [dda] = DmgDistPerAsset.objects.filter(output__oq_job=self.job.id,
                output__output_type="dmg_dist_per_asset")

        self.assertEquals(["no_damage", "LS1", "LS2"], dda.dmg_states)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="A")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(1.00563, data.mean)
        self._close_to(1.61341, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(34.80851, data.mean)
        self._close_to(18.34906, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(64.18586, data.mean)
        self._close_to(19.83963, data.stddev)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="B")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(3.64527, data.mean)
        self._close_to(3.35330, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(17.10494, data.mean)
        self._close_to(4.63604, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(19.24979, data.mean)
        self._close_to(7.78725, data.stddev)

    def test_compute_risk_dda_dsc(self):
        # test the damage distribution per asset with a discrete
        # fragility model
        fm = self._store_dsc_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        [dda] = DmgDistPerAsset.objects.filter(output__oq_job=self.job.id,
                output__output_type="dmg_dist_per_asset")

        self.assertEquals(["no_damage", "LS1", "LS2"], dda.dmg_states)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="A")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(68.0, data.mean)
        self._close_to(8.5513157, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(21.0, data.mean)
        self._close_to(4.2756578, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(11.0, data.mean)
        self._close_to(4.2756578, data.stddev)

        [exposure] = self.em.exposuredata_set.filter(asset_ref="B")
        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="no_damage")

        self._close_to(30.4, data.mean)
        self._close_to(3.4, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS1")

        self._close_to(3.9, data.mean)
        self._close_to(1.4, data.stddev)

        [data] = DmgDistPerAssetData.objects.filter(dmg_dist_per_asset=dda,
                exposure_data=exposure, dmg_state="LS2")

        self._close_to(5.7, data.mean)
        self._close_to(2.1, data.stddev)

    def test_dda_iml_above_range(self):
        # corner case where we have a ground motion value
        # (that corresponds to the intensity measure level in the
        # fragility function) that is higher than the highest
        # intensity measure level defined in the model (in this
        # particular case 0.7). Given this condition, to compute
        # the fractions of buildings we use the highest intensity
        # measure level defined in the model (0.7 in this case)

        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1"], format="discrete")

        fmodel.save()

        func = models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1)

        func.save()

        self._close_to(compute_gmv_fractions([func], 0.7),
                compute_gmv_fractions([func], 0.8))

    def test_dda_iml_below_range(self):
        # corner case where we have a ground motion value
        # (that corresponds to the intensity measure level in the
        # fragility function) that is lower than the lowest
        # intensity measure level defined in the model (in this
        # particular case 0.1). Given this condition, the
        # fractions of buildings is 100% no_damage and 0% for the
        # remaining limit states defined in the model

        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1"], format="discrete")

        fmodel.save()

        func = models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1)

        func.save()

        self._close_to([1.0, 0.0],
                compute_gmv_fractions([func], 0.05))

    def test_post_execute_serialization(self):
        # when --output-type=xml is specified, we serialize results
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)

        # since the taxonomy data is computed and aggregated
        # per block in the execute() method and we are not
        # testing it here, just stubbing out some values to
        # produce the taxonomy distribution
        self.calculator.ddt_fractions = {"RC": numpy.array(
                [[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]])}

        self.job_ctxt.serialize_results_to = ["xml"]
        self.calculator.post_execute()

        paths = self._results_files()

        for path in paths:
            self.assertTrue(os.path.exists(path))
            os.unlink(path)

    def test_post_execute_no_serialization(self):
        # otherwise, just on database (default)
        fm = self._store_con_fmodel()

        self.calculator.pre_execute()
        self.calculator.compute_risk(BLOCK_ID, fmodel=fm)
        self.calculator.post_execute()

        paths = self._results_files()

        for path in paths:
            self.assertFalse(os.path.exists(path))

    def _results_files(self):
        target_dir = os.path.join(self.job_ctxt.params.get("BASE_PATH"),
                self.job_ctxt.params.get("OUTPUT_DIR"))

        return [os.path.join(
            target_dir, "dmg-dist-asset-%s.xml" % self.job.id), os.path.join(
            target_dir, "dmg-dist-taxonomy-%s.xml" % self.job.id)]

    def _close_to(self, expected, actual):
        self.assertTrue(numpy.allclose(actual, expected, atol=0.0, rtol=0.05))

    def _store_gmvs(self, gmvs):
        client = kvs.get_client()
        encoder = json.JSONEncoder()

        key = ground_motion_values_key(self.job.id, GridPoint(None, 0, 0))

        for gmv in gmvs:
            client.rpush(key, encoder.encode({"mag": gmv}))

    def _store_con_fmodel(self):
        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism,
            lss=["LS1", "LS2"], format="continuous")

        fmodel.save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS2", mean="0.35", stddev="0.10", lsi=2).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", mean="0.20", stddev="0.05", lsi=1).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS2", mean="0.40", stddev="0.12", lsi=2).save()

        models.Ffc(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS1", mean="0.25", stddev="0.08", lsi=1).save()

        return fmodel

    def _store_dsc_fmodel(self):
        [ism] = models.inputs4job(self.job.id, input_type="fragility")

        fmodel = models.FragilityModel(
            owner=ism.owner, input=ism, imls=[0.1, 0.3, 0.5, 0.7],
            imt="mmi", lss=["LS1", "LS2"], format="discrete")

        fmodel.save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS2", poes=[0.00, 0.05, 0.20, 0.50], lsi=2).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RC",
            ls="LS1", poes=[0.05, 0.20, 0.50, 1.00], lsi=1).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS2", poes=[0.02, 0.07, 0.25, 0.60], lsi=2).save()

        models.Ffd(
            fragility_model=fmodel, taxonomy="RM",
            ls="LS1", poes=[0.03, 0.12, 0.42, 0.90], lsi=1).save()

        return fmodel

    def _store_em(self):
        [ism] = models.inputs4job(self.job.id, input_type="exposure")

        em = models.ExposureModel(
            owner=ism.owner, input=ism,
            name="AAA", category="single_asset",
            reco_type="aggregated", reco_unit="USD",
            stco_type="aggregated", stco_unit="USD")

        em.save()

        models.ExposureData(
            exposure_model=em, taxonomy="RC",
            asset_ref="A", number_of_units=100, stco=1,
            site=geos.GEOSGeometry(self.site.point.to_wkt()), reco=1).save()

        models.ExposureData(
            exposure_model=em, taxonomy="RM",
            asset_ref="B", number_of_units=40, stco=1,
            site=geos.GEOSGeometry(self.site.point.to_wkt()), reco=1).save()

        return em
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from os.path import basename, join, dirname
from project_generator_definitions.definitions import ProGenDef

from workspace_tools.export.exporters import Exporter
from workspace_tools.targets import TARGET_MAP, TARGET_NAMES

# If you wish to add a new target, add it to project_generator_definitions, and then
# define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
# There are 2 default mbed templates (predefined settings) uvision.uvproj and uvproj_microlib.uvproj.tmpl
class Uvision4(Exporter):
    """
    Exporter class for uvision. This class uses project generator.
    """
    # These 2 are currently for exporters backward compatiblity
    NAME = 'uVision4'
    TOOLCHAIN = 'ARM'
    # PROGEN_ACTIVE contains information for exporter scripts that this is using progen
    PROGEN_ACTIVE = True

    # backward compatibility with our scripts
    TARGETS = []
    for target in TARGET_NAMES:
        try:
            if (ProGenDef('uvision').is_supported(str(TARGET_MAP[target])) or
                ProGenDef('uvision').is_supported(TARGET_MAP[target].progen['target'])):
                TARGETS.append(target)
        except AttributeError:
            # target is not supported yet
            continue

    def generate(self):
        """ Generates the project files """
        project_data = self.progen_get_project_data()
        tool_specific = {}
        # Expand tool specific settings by uvision specific settings which are required
        try:
            if TARGET_MAP[self.target].progen['uvision']['template']:
                tool_specific['uvision'] = TARGET_MAP[self.target].progen['uvision']
        except KeyError:
            # use default template
            # by the mbed projects
            tool_specific['uvision'] = {
                    'template': [join(dirname(__file__),  'uvision.uvproj.tmpl')],
            }

        project_data['tool_specific'] = {}
        project_data['tool_specific'].update(tool_specific)
        # armasm does not like floating numbers in macros, timestamp to int
        i = 0
        for macro in project_data['common']['macros']:
            if macro.startswith('MBED_BUILD_TIMESTAMP'):
                timestamp = macro[len('MBED_BUILD_TIMESTAMP='):]
                project_data['common']['macros'][i] = 'MBED_BUILD_TIMESTAMP=' + str(int(float(timestamp)))
                break
            i += 1
        project_data['common']['macros'].append('__ASSERT_MSG')
        self.progen_gen_file('uvision', project_data)

<commit_msg>uvision exporter - fix - asm does not like string defined macros<commit_after>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from os.path import basename, join, dirname
from project_generator_definitions.definitions import ProGenDef

from workspace_tools.export.exporters import Exporter
from workspace_tools.targets import TARGET_MAP, TARGET_NAMES

# If you wish to add a new target, add it to project_generator_definitions, and then
# define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
# There are 2 default mbed templates (predefined settings) uvision.uvproj and uvproj_microlib.uvproj.tmpl
class Uvision4(Exporter):
    """
    Exporter class for uvision. This class uses project generator.
    """
    # These 2 are currently for exporters backward compatiblity
    NAME = 'uVision4'
    TOOLCHAIN = 'ARM'
    # PROGEN_ACTIVE contains information for exporter scripts that this is using progen
    PROGEN_ACTIVE = True

    # backward compatibility with our scripts
    TARGETS = []
    for target in TARGET_NAMES:
        try:
            if (ProGenDef('uvision').is_supported(str(TARGET_MAP[target])) or
                ProGenDef('uvision').is_supported(TARGET_MAP[target].progen['target'])):
                TARGETS.append(target)
        except AttributeError:
            # target is not supported yet
            continue

    def generate(self):
        """ Generates the project files """
        project_data = self.progen_get_project_data()
        tool_specific = {}
        # Expand tool specific settings by uvision specific settings which are required
        try:
            if TARGET_MAP[self.target].progen['uvision']['template']:
                tool_specific['uvision'] = TARGET_MAP[self.target].progen['uvision']
        except KeyError:
            # use default template
            # by the mbed projects
            tool_specific['uvision'] = {
                    'template': [join(dirname(__file__),  'uvision.uvproj.tmpl')],
            }

        project_data['tool_specific'] = {}
        project_data['tool_specific'].update(tool_specific)
        i = 0
        for macro in project_data['common']['macros']:
            # armasm does not like floating numbers in macros, timestamp to int
            if macro.startswith('MBED_BUILD_TIMESTAMP'):
                timestamp = macro[len('MBED_BUILD_TIMESTAMP='):]
                project_data['common']['macros'][i] = 'MBED_BUILD_TIMESTAMP=' + str(int(float(timestamp)))
            # armasm does not even accept MACRO=string
            if macro.startswith('MBED_USERNAME'):
                project_data['common']['macros'].pop(i)
            i += 1
        project_data['common']['macros'].append('__ASSERT_MSG')
        self.progen_gen_file('uvision', project_data)

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import datetime
import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    ExtractorError,
    InAdvancePagedList,
    orderedSet,
    str_to_int,
    unified_strdate,
)

class MotherlessIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
    _TESTS = [{
        'url': 'http://motherless.com/AC3FFE1',
        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
        'info_dict': {
            'id': 'AC3FFE1',
            'ext': 'mp4',
            'title': 'Fucked in the ass while playing PS3',
            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
            'upload_date': '20100913',
            'uploader_id': 'famouslyfuckedup',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        'url': 'http://motherless.com/532291B',
        'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
        'info_dict': {
            'id': '532291B',
            'ext': 'mp4',
            'title': 'Amazing girl playing the omegle game, PERFECT!',
            'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
                           'game', 'hairy'],
            'upload_date': '20140622',
            'uploader_id': 'Sulivana7x',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        },
        'skip': '404',
    }, {
        'url': 'http://motherless.com/g/cosplay/633979F',
        'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
        'info_dict': {
            'id': '633979F',
            'ext': 'mp4',
            'title': 'Turtlette',
            'categories': ['superheroine heroine  superher'],
            'upload_date': '20140827',
            'uploader_id': 'shade0230',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        # no keywords
        'url': 'http://motherless.com/8B4BBC1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if any(p in webpage for p in (
                '<title>404 - MOTHERLESS.COM<',
                ">The page you're looking for cannot be found.<")):
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        if '>The content you are trying to view is for friends only.' in webpage:
            raise ExtractorError('Video %s is for friends only' % video_id, expected=True)

        title = self._html_search_regex(
            r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
        video_url = self._html_search_regex(
            r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video URL')
        age_limit = self._rta_search(webpage)
        view_count = str_to_int(self._html_search_regex(
            r'Views\s+([^<]+)<',
            webpage, 'view count', fatal=False))
        like_count = str_to_int(self._html_search_regex(
            r'Favorited\s+([^<]+)<',
            webpage, 'like count', fatal=False))

        upload_date = self._html_search_regex(
            r'Uploaded\s+([^<]+)<', webpage, 'upload date')
        if 'Ago' in upload_date:
            days = int(re.search(r'([0-9]+)', upload_date).group(1))
            upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
        else:
            upload_date = unified_strdate(upload_date)

        comment_count = webpage.count('class="media-comment-contents"')
        uploader_id = self._html_search_regex(
            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
            webpage, 'uploader_id')

        categories = self._html_search_meta('keywords', webpage, default=None)
        if categories:
            categories = [cat.strip() for cat in categories.split(',')]

        return {
            'id': video_id,
            'title': title,
            'upload_date': upload_date,
            'uploader_id': uploader_id,
            'thumbnail': self._og_search_thumbnail(webpage),
            'categories': categories,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
            'age_limit': age_limit,
            'url': video_url,
        }

class MotherlessGroupIE(InfoExtractor):
    _VALID_URL = 'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
    _TESTS = [{
        'url': 'http://motherless.com/g/movie_scenes',
        'info_dict': {
            'id': 'movie_scenes',
            'title': 'Movie Scenes',
            'description': 'Hot and sexy scenes from "regular" movies... '
                           'Beautiful actresses fully nude... A looot of '
                           'skin! :)Enjoy!',
        },
        'playlist_mincount': 662,
    }, {
        'url': 'http://motherless.com/gv/sex_must_be_funny',
        'info_dict': {
            'id': 'sex_must_be_funny',
            'title': 'Sex must be funny',
            'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
                           'any kind!'
        },
        'playlist_mincount': 9,
    }]

    @classmethod
    def suitable(cls, url):
        return (False if MotherlessIE.suitable(url)
                else super(MotherlessGroupIE, cls).suitable(url))

    def _extract_entries(self, webpage, base):
        return [
            self.url_result(
                compat_urlparse.urljoin(base, video_path),
                MotherlessIE.ie_key(), video_title=title)
            for video_path, title in orderedSet(re.findall(
                r'href="/([^"]+)"[^>]+>\s+]+alt="[^-]+-\s([^"]+)"',
                webpage))
        ]

    def _real_extract(self, url):
        group_id = self._match_id(url)
        page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
        webpage = self._download_webpage(page_url, group_id)
        title = self._search_regex(
            r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
        description = self._html_search_meta(
            'description', webpage, fatal=False)
        page_count = self._int(self._search_regex(
            r'(\d+)</(?:a|span)><(?:a|span)[^>]+>\s*NEXT',
            webpage, 'page_count'), 'page_count')
        PAGE_SIZE = 80

        def _get_page(idx):
            webpage = self._download_webpage(
                page_url, group_id, query={'page': idx + 1},
                note='Downloading page %d/%d' % (idx + 1, page_count)
            )
            for entry in self._extract_entries(webpage, url):
                yield entry

        playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)

        return {
            '_type': 'playlist',
            'id': group_id,
            'title': title,
            'description': description,
            'entries': playlist
        }
<commit_msg>[motherless:group] Capture leading slash of video path<commit_after>from __future__ import unicode_literals

import datetime
import re

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    ExtractorError,
    InAdvancePagedList,
    orderedSet,
    str_to_int,
    unified_strdate,
)

class MotherlessIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
    _TESTS = [{
        'url': 'http://motherless.com/AC3FFE1',
        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
        'info_dict': {
            'id': 'AC3FFE1',
            'ext': 'mp4',
            'title': 'Fucked in the ass while playing PS3',
            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
            'upload_date': '20100913',
            'uploader_id': 'famouslyfuckedup',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        'url': 'http://motherless.com/532291B',
        'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
        'info_dict': {
            'id': '532291B',
            'ext': 'mp4',
            'title': 'Amazing girl playing the omegle game, PERFECT!',
            'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
                           'game', 'hairy'],
            'upload_date': '20140622',
            'uploader_id': 'Sulivana7x',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        },
        'skip': '404',
    }, {
        'url': 'http://motherless.com/g/cosplay/633979F',
        'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
        'info_dict': {
            'id': '633979F',
            'ext': 'mp4',
            'title': 'Turtlette',
            'categories': ['superheroine heroine  superher'],
            'upload_date': '20140827',
            'uploader_id': 'shade0230',
            'thumbnail': r're:http://.*\.jpg',
            'age_limit': 18,
        }
    }, {
        # no keywords
        'url': 'http://motherless.com/8B4BBC1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        if any(p in webpage for p in (
                '<title>404 - MOTHERLESS.COM<',
                ">The page you're looking for cannot be found.<")):
            raise ExtractorError('Video %s does not exist' % video_id, expected=True)

        if '>The content you are trying to view is for friends only.' in webpage:
            raise ExtractorError('Video %s is for friends only' % video_id, expected=True)

        title = self._html_search_regex(
            r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
        video_url = self._html_search_regex(
            r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video URL')
        age_limit = self._rta_search(webpage)
        view_count = str_to_int(self._html_search_regex(
            r'Views\s+([^<]+)<',
            webpage, 'view count', fatal=False))
        like_count = str_to_int(self._html_search_regex(
            r'Favorited\s+([^<]+)<',
            webpage, 'like count', fatal=False))

        upload_date = self._html_search_regex(
            r'Uploaded\s+([^<]+)<', webpage, 'upload date')
        if 'Ago' in upload_date:
            days = int(re.search(r'([0-9]+)', upload_date).group(1))
            upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
        else:
            upload_date = unified_strdate(upload_date)

        comment_count = webpage.count('class="media-comment-contents"')
        uploader_id = self._html_search_regex(
            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
            webpage, 'uploader_id')

        categories = self._html_search_meta('keywords', webpage, default=None)
        if categories:
            categories = [cat.strip() for cat in categories.split(',')]

        return {
            'id': video_id,
            'title': title,
            'upload_date': upload_date,
            'uploader_id': uploader_id,
            'thumbnail': self._og_search_thumbnail(webpage),
            'categories': categories,
            'view_count': view_count,
            'like_count': like_count,
            'comment_count': comment_count,
            'age_limit': age_limit,
            'url': video_url,
        }

class MotherlessGroupIE(InfoExtractor):
    _VALID_URL = 'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
    _TESTS = [{
        'url': 'http://motherless.com/g/movie_scenes',
        'info_dict': {
            'id': 'movie_scenes',
            'title': 'Movie Scenes',
            'description': 'Hot and sexy scenes from "regular" movies... '
                           'Beautiful actresses fully nude... A looot of '
                           'skin! :)Enjoy!',
        },
        'playlist_mincount': 662,
    }, {
        'url': 'http://motherless.com/gv/sex_must_be_funny',
        'info_dict': {
            'id': 'sex_must_be_funny',
            'title': 'Sex must be funny',
            'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
                           'any kind!'
        },
        'playlist_mincount': 9,
    }]

    @classmethod
    def suitable(cls, url):
        return (False if MotherlessIE.suitable(url)
                else super(MotherlessGroupIE, cls).suitable(url))

    def _extract_entries(self, webpage, base):
        return [
            self.url_result(
                compat_urlparse.urljoin(base, video_path),
                MotherlessIE.ie_key(), video_title=title)
            for video_path, title in orderedSet(re.findall(
                r'href="(/[^"]+)"[^>]+>\s+]+alt="[^-]+-\s([^"]+)"',
                webpage))
        ]

    def _real_extract(self, url):
        group_id = self._match_id(url)
        page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
        webpage = self._download_webpage(page_url, group_id)
        title = self._search_regex(
            r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
        description = self._html_search_meta(
            'description', webpage, fatal=False)
        page_count = self._int(self._search_regex(
            r'(\d+)</(?:a|span)><(?:a|span)[^>]+>\s*NEXT',
            webpage, 'page_count'), 'page_count')
        PAGE_SIZE = 80

        def _get_page(idx):
            webpage = self._download_webpage(
                page_url, group_id, query={'page': idx + 1},
                note='Downloading page %d/%d' % (idx + 1, page_count)
            )
            for entry in self._extract_entries(webpage, url):
                yield entry

        playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)

        return {
            '_type': 'playlist',
            'id': group_id,
            'title': title,
            'description': description,
            'entries': playlist
        }
<|endoftext|>


<commit_before># Copyright (c) 2013 Rackspace Hosting, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""pools: a resource to handle storage pool management

A pool is added by an operator by interacting with the
pooling-related endpoints. When specifying a pool, the
following fields are required:

::

    {
        "name": string,
        "weight": integer,
        "uri": string::uri
    }

Furthermore, depending on the underlying storage type of pool being
registered, there is an optional field:
::

    {
        "options": {...}
    }
"""

import falcon
import jsonschema
from oslo_log import log
import six

from zaqar.common.api.schemas import pools as schema
from zaqar.common import decorators
from zaqar.common import utils as common_utils
from zaqar.i18n import _
from zaqar.storage import errors
from zaqar.storage import utils as storage_utils
from zaqar.transport import acl
from zaqar.transport import utils as transport_utils
from zaqar.transport import validation
from zaqar.transport.wsgi import errors as wsgi_errors
from zaqar.transport.wsgi import utils as wsgi_utils

LOG = log.getLogger(__name__)

class Listing(object):
    """A resource to list registered pools

    :param pools_controller: means to interact with storage
    """

    def __init__(self, pools_controller, validate):
        self._ctrl = pools_controller
        self._validate = validate

    @decorators.TransportLog("Pools collection")
    @acl.enforce("pools:get_all")
    def on_get(self, request, response, project_id):
        """Returns a pool listing as objects embedded in an object:

        ::

            {
                "pools": [
                    {"href": "", "weight": 100, "uri": ""},
                    ...
                ],
                "links": [
                    {"href": "", "rel": "next"}
                ]
            }

        :returns: HTTP | 200
        """

        LOG.debug(u'LIST pools')

        store = {}
        request.get_param('marker', store=store)
        request.get_param_as_int('limit', store=store)
        request.get_param_as_bool('detailed', store=store)

        try:
            self._validate.pool_listing(**store)
        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        cursor = self._ctrl.list(**store)
        pools = list(next(cursor))

        results = {'links': []}

        if pools:
            store['marker'] = next(cursor)

            for entry in pools:
                entry['href'] = request.path + '/' + entry['name']

            results['links'] = [
                {
                    'rel': 'next',
                    'href': request.path + falcon.to_query_str(store)
                }
            ]

        results['pools'] = pools

        response.content_location = request.relative_uri
        response.body = transport_utils.to_json(results)
        response.status = falcon.HTTP_200

class Resource(object):
    """A handler for individual pool.

    :param pools_controller: means to interact with storage
    """

    def __init__(self, pools_controller):
        self._ctrl = pools_controller
        validator_type = jsonschema.Draft4Validator
        self._validators = {
            'weight': validator_type(schema.patch_weight),
            'uri': validator_type(schema.patch_uri),
            'group': validator_type(schema.patch_uri),
            'options': validator_type(schema.patch_options),
            'create': validator_type(schema.create)
        }

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:get")
    def on_get(self, request, response, project_id, pool):
        """Returns a JSON object for a single pool entry:

        ::

            {"weight": 100, "uri": "", options: {...}}

        :returns: HTTP | [200, 404]
        """

        LOG.debug(u'GET pool - name: %s', pool)
        data = None
        detailed = request.get_param_as_bool('detailed') or False

        try:
            data = self._ctrl.get(pool, detailed)

        except errors.PoolDoesNotExist as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPNotFound(six.text_type(ex))

        data['href'] = request.path

        response.body = transport_utils.to_json(data)

    @acl.enforce("pools:create")
    def on_put(self, request, response, project_id, pool):
        """Registers a new pool. Expects the following input:

        ::

            {"weight": 100, "uri": ""}

        An options object may also be provided.

        :returns: HTTP | [201, 204]
        """

        LOG.debug(u'PUT pool - name: %s', pool)

        conf = self._ctrl.driver.conf
        data = wsgi_utils.load(request)
        wsgi_utils.validate(self._validators['create'], data)
        if not storage_utils.can_connect(data['uri'], conf=conf):
            raise wsgi_errors.HTTPBadRequestBody(
                'cannot connect to %s' % data['uri']
            )
        try:
            self._ctrl.create(pool, weight=data['weight'],
                              uri=data['uri'],
                              group=data.get('group'),
                              options=data.get('options', {}))
            response.status = falcon.HTTP_201
            response.location = request.path
        except errors.PoolCapabilitiesMismatch as e:
            LOG.exception(e)
            title = _(u'Unable to create pool')
            raise falcon.HTTPBadRequest(title, six.text_type(e))
        except errors.PoolAlreadyExists as e:
            LOG.exception(e)
            raise wsgi_errors.HTTPConflict(six.text_type(e))

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:delete")
    def on_delete(self, request, response, project_id, pool):
        """Deregisters a pool.

        :returns: HTTP | [204, 403]
        """

        LOG.debug(u'DELETE pool - name: %s', pool)

        try:
            self._ctrl.delete(pool)
        except errors.PoolInUseByFlavor as ex:
            LOG.exception(ex)
            title = _(u'Unable to delete')
            description = _(u'This pool is used by flavors {flavor}; '
                            u'It cannot be deleted.')
            description = description.format(flavor=ex.flavor)
            raise falcon.HTTPForbidden(title, description)

        response.status = falcon.HTTP_204

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:update")
    def on_patch(self, request, response, project_id, pool):
        """Allows one to update a pool's weight, uri, and/or options.

        This method expects the user to submit a JSON object
        containing at least one of: 'uri', 'weight', 'group', 'options'. If
        none are found, the request is flagged as bad. There is also
        strict format checking through the use of
        jsonschema. Appropriate errors are returned in each case for
        badly formatted input.

        :returns: HTTP | 200,400
        """

        LOG.debug(u'PATCH pool - name: %s', pool)
        data = wsgi_utils.load(request)

        EXPECT = ('weight', 'uri', 'group', 'options')
        if not any([(field in data) for field in EXPECT]):
            LOG.debug(u'PATCH pool, bad params')
            raise wsgi_errors.HTTPBadRequestBody(
                'One of `uri`, `weight`, `group`, or `options` needs '
                'to be specified'
            )

        for field in EXPECT:
            wsgi_utils.validate(self._validators[field], data)

        conf = self._ctrl.driver.conf
        if 'uri' in data and not storage_utils.can_connect(data['uri'],
                                                           conf=conf):
            raise wsgi_errors.HTTPBadRequestBody(
                'cannot connect to %s' % data['uri']
            )
        fields = common_utils.fields(data, EXPECT,
                                     pred=lambda v: v is not None)
        resp_data = None
        try:
            self._ctrl.update(pool, **fields)
            resp_data = self._ctrl.get(pool, False)
        except errors.PoolDoesNotExist as ex:
            LOG.exception(ex)
            raise wsgi_errors.HTTPNotFound(six.text_type(ex))

        resp_data['href'] = request.path
        response.body = transport_utils.to_json(resp_data)
<commit_msg>Add the missed decorator to pools<commit_after># Copyright (c) 2013 Rackspace Hosting, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""pools: a resource to handle storage pool management

A pool is added by an operator by interacting with the
pooling-related endpoints. When specifying a pool, the
following fields are required:

::

    {
        "name": string,
        "weight": integer,
        "uri": string::uri
    }

Furthermore, depending on the underlying storage type of pool being
registered, there is an optional field:
::

    {
        "options": {...}
    }
"""

import falcon
import jsonschema
from oslo_log import log
import six

from zaqar.common.api.schemas import pools as schema
from zaqar.common import decorators
from zaqar.common import utils as common_utils
from zaqar.i18n import _
from zaqar.storage import errors
from zaqar.storage import utils as storage_utils
from zaqar.transport import acl
from zaqar.transport import utils as transport_utils
from zaqar.transport import validation
from zaqar.transport.wsgi import errors as wsgi_errors
from zaqar.transport.wsgi import utils as wsgi_utils

LOG = log.getLogger(__name__)

class Listing(object):
    """A resource to list registered pools

    :param pools_controller: means to interact with storage
    """

    def __init__(self, pools_controller, validate):
        self._ctrl = pools_controller
        self._validate = validate

    @decorators.TransportLog("Pools collection")
    @acl.enforce("pools:get_all")
    def on_get(self, request, response, project_id):
        """Returns a pool listing as objects embedded in an object:

        ::

            {
                "pools": [
                    {"href": "", "weight": 100, "uri": ""},
                    ...
                ],
                "links": [
                    {"href": "", "rel": "next"}
                ]
            }

        :returns: HTTP | 200
        """

        LOG.debug(u'LIST pools')

        store = {}
        request.get_param('marker', store=store)
        request.get_param_as_int('limit', store=store)
        request.get_param_as_bool('detailed', store=store)

        try:
            self._validate.pool_listing(**store)
        except validation.ValidationFailed as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))

        cursor = self._ctrl.list(**store)
        pools = list(next(cursor))

        results = {'links': []}

        if pools:
            store['marker'] = next(cursor)

            for entry in pools:
                entry['href'] = request.path + '/' + entry['name']

            results['links'] = [
                {
                    'rel': 'next',
                    'href': request.path + falcon.to_query_str(store)
                }
            ]

        results['pools'] = pools

        response.content_location = request.relative_uri
        response.body = transport_utils.to_json(results)
        response.status = falcon.HTTP_200

class Resource(object):
    """A handler for individual pool.

    :param pools_controller: means to interact with storage
    """

    def __init__(self, pools_controller):
        self._ctrl = pools_controller
        validator_type = jsonschema.Draft4Validator
        self._validators = {
            'weight': validator_type(schema.patch_weight),
            'uri': validator_type(schema.patch_uri),
            'group': validator_type(schema.patch_uri),
            'options': validator_type(schema.patch_options),
            'create': validator_type(schema.create)
        }

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:get")
    def on_get(self, request, response, project_id, pool):
        """Returns a JSON object for a single pool entry:

        ::

            {"weight": 100, "uri": "", options: {...}}

        :returns: HTTP | [200, 404]
        """

        LOG.debug(u'GET pool - name: %s', pool)
        data = None
        detailed = request.get_param_as_bool('detailed') or False

        try:
            data = self._ctrl.get(pool, detailed)

        except errors.PoolDoesNotExist as ex:
            LOG.debug(ex)
            raise wsgi_errors.HTTPNotFound(six.text_type(ex))

        data['href'] = request.path

        response.body = transport_utils.to_json(data)

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:create")
    def on_put(self, request, response, project_id, pool):
        """Registers a new pool. Expects the following input:

        ::

            {"weight": 100, "uri": ""}

        An options object may also be provided.

        :returns: HTTP | [201, 204]
        """

        LOG.debug(u'PUT pool - name: %s', pool)

        conf = self._ctrl.driver.conf
        data = wsgi_utils.load(request)
        wsgi_utils.validate(self._validators['create'], data)
        if not storage_utils.can_connect(data['uri'], conf=conf):
            raise wsgi_errors.HTTPBadRequestBody(
                'cannot connect to %s' % data['uri']
            )
        try:
            self._ctrl.create(pool, weight=data['weight'],
                              uri=data['uri'],
                              group=data.get('group'),
                              options=data.get('options', {}))
            response.status = falcon.HTTP_201
            response.location = request.path
        except errors.PoolCapabilitiesMismatch as e:
            LOG.exception(e)
            title = _(u'Unable to create pool')
            raise falcon.HTTPBadRequest(title, six.text_type(e))
        except errors.PoolAlreadyExists as e:
            LOG.exception(e)
            raise wsgi_errors.HTTPConflict(six.text_type(e))

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:delete")
    def on_delete(self, request, response, project_id, pool):
        """Deregisters a pool.

        :returns: HTTP | [204, 403]
        """

        LOG.debug(u'DELETE pool - name: %s', pool)

        try:
            self._ctrl.delete(pool)
        except errors.PoolInUseByFlavor as ex:
            LOG.exception(ex)
            title = _(u'Unable to delete')
            description = _(u'This pool is used by flavors {flavor}; '
                            u'It cannot be deleted.')
            description = description.format(flavor=ex.flavor)
            raise falcon.HTTPForbidden(title, description)

        response.status = falcon.HTTP_204

    @decorators.TransportLog("Pools item")
    @acl.enforce("pools:update")
    def on_patch(self, request, response, project_id, pool):
        """Allows one to update a pool's weight, uri, and/or options.

        This method expects the user to submit a JSON object
        containing at least one of: 'uri', 'weight', 'group', 'options'. If
        none are found, the request is flagged as bad. There is also
        strict format checking through the use of
        jsonschema. Appropriate errors are returned in each case for
        badly formatted input.

        :returns: HTTP | 200,400
        """

        LOG.debug(u'PATCH pool - name: %s', pool)
        data = wsgi_utils.load(request)

        EXPECT = ('weight', 'uri', 'group', 'options')
        if not any([(field in data) for field in EXPECT]):
            LOG.debug(u'PATCH pool, bad params')
            raise wsgi_errors.HTTPBadRequestBody(
                'One of `uri`, `weight`, `group`, or `options` needs '
                'to be specified'
            )

        for field in EXPECT:
            wsgi_utils.validate(self._validators[field], data)

        conf = self._ctrl.driver.conf
        if 'uri' in data and not storage_utils.can_connect(data['uri'],
                                                           conf=conf):
            raise wsgi_errors.HTTPBadRequestBody(
                'cannot connect to %s' % data['uri']
            )
        fields = common_utils.fields(data, EXPECT,
                                     pred=lambda v: v is not None)
        resp_data = None
        try:
            self._ctrl.update(pool, **fields)
            resp_data = self._ctrl.get(pool, False)
        except errors.PoolDoesNotExist as ex:
            LOG.exception(ex)
            raise wsgi_errors.HTTPNotFound(six.text_type(ex))

        resp_data['href'] = request.path
        response.body = transport_utils.to_json(resp_data)
<|endoftext|>


<commit_before>"""
Check for updates in a background process. If we can start a program immediately, but some of our information
is rather old (longer that the L{policy.Policy.freshness} threshold) then we run it anyway, and check for updates using a new
process that runs quietly in the background.

This avoids the need to annoy people with a 'checking for updates' box when they're trying to run things.
"""

import sys, os
from logging import info, warn
from zeroinstall.support import tasks
from zeroinstall.injector.iface_cache import iface_cache
from zeroinstall.injector import handler, namespaces

# Copyright (C) 2009, Thomas Leonard
# See the README file for details, or visit http://0install.net.

def _escape_xml(s):
	return s.replace('&', '&').replace('<', '<')

def _exec_gui(uri, *args):
	os.execvp('0launch', ['0launch', '--download-only', '--gui'] + list(args) + [uri])

class _NetworkState:
	NM_STATE_UNKNOWN = 0
	NM_STATE_ASLEEP = 1
	NM_STATE_CONNECTING = 2
	NM_STATE_CONNECTED = 3
	NM_STATE_DISCONNECTED = 4

class BackgroundHandler(handler.Handler):
	"""A Handler for non-interactive background updates. Runs the GUI if interaction is required."""
	def __init__(self, title, root):
		handler.Handler.__init__(self)
		self.title = title
		self.notification_service = None
		self.network_manager = None
		self.notification_service_caps = []
		self.root = root	# If we need to confirm any keys, run the GUI on this

		try:
			import dbus
			import dbus.glib
		except Exception, ex:
			info("Failed to import D-BUS bindings: %s", ex)
			return

		try:
			session_bus = dbus.SessionBus()
			remote_object = session_bus.get_object('org.freedesktop.Notifications',
								'/org/freedesktop/Notifications')

			self.notification_service = dbus.Interface(remote_object,
							'org.freedesktop.Notifications')

			# The Python bindings insist on printing a pointless introspection
			# warning to stderr if the service is missing. Force it to be done
			# now so we can skip it
			old_stderr = sys.stderr
			sys.stderr = None
			try:
				self.notification_service_caps = [str(s) for s in
						self.notification_service.GetCapabilities()]
			finally:
				sys.stderr = old_stderr
		except Exception, ex:
			info("No D-BUS notification service available: %s", ex)

		try:
			system_bus = dbus.SystemBus()
			remote_object = system_bus.get_object('org.freedesktop.NetworkManager',
								'/org/freedesktop/NetworkManager')

			self.network_manager = dbus.Interface(remote_object,
							'org.freedesktop.NetworkManager')
		except Exception, ex:
			info("No D-BUS network manager service available: %s", ex)

	def get_network_state(self):
		if self.network_manager:
			try:
				return self.network_manager.state()
			except Exception, ex:
				warn("Error getting network state: %s", ex)
		return _NetworkState.NM_STATE_UNKNOWN

	def confirm_trust_keys(self, interface, sigs, iface_xml):
		"""Run the GUI if we need to confirm any keys."""
		info("Can't update interface; signature not yet trusted. Running GUI...")
		_exec_gui(self.root, '--refresh', '--download-only', '--systray')

	def report_error(self, exception, tb = None):
		self.notify("Zero Install", "Error updating %s: %s" % (self.title, str(exception)))

	def notify(self, title, message, timeout = 0, actions = []):
		"""Send a D-BUS notification message if possible. If there is no notification
		service available, log the message instead."""
		if not self.notification_service:
			info('%s: %s', title, message)
			return None

		LOW = 0
		NORMAL = 1
		CRITICAL = 2

		import dbus.types

		hints = {}
		if actions:
			hints['urgency'] = dbus.types.Byte(NORMAL)
		else:
			hints['urgency'] = dbus.types.Byte(LOW)

		return self.notification_service.Notify('Zero Install',
			0,		# replaces_id,
			'',		# icon
			_escape_xml(title),
			_escape_xml(message),
			actions,
			hints,
			timeout * 1000)

	def have_actions_support(self):
		return 'actions' in self.notification_service_caps

def _detach():
	"""Fork a detached grandchild.
	@return: True if we are the original."""
	child = os.fork()
	if child:
		pid, status = os.waitpid(child, 0)
		assert pid == child
		return True

	# The calling process might be waiting for EOF from its child.
	# Close our stdout so we don't keep it waiting.
	# Note: this only fixes the most common case; it could be waiting
	# on any other FD as well. We should really use gobject.spawn_async
	# to close *all* FDs.
	null = os.open('/dev/null', os.O_RDWR)
	os.dup2(null, 1)
	os.close(null)

	grandchild = os.fork()
	if grandchild:
		os._exit(0)	# Parent's waitpid returns and grandchild continues

	return False

def _check_for_updates(policy, verbose):
	root_iface = iface_cache.get_interface(policy.root).get_name()

	policy.handler = BackgroundHandler(root_iface, policy.root)

	info("Checking for updates to '%s' in a background process", root_iface)
	if verbose:
		policy.handler.notify("Zero Install", "Checking for updates to '%s'..." % root_iface, timeout = 1)

	network_state = policy.handler.get_network_state()
	if network_state != _NetworkState.NM_STATE_CONNECTED:
		info("Not yet connected to network (status = %d). Sleeping for a bit...", network_state)
		import time
		time.sleep(120)
		if network_state in (_NetworkState.NM_STATE_DISCONNECTED, _NetworkState.NM_STATE_ASLEEP):
			info("Still not connected to network. Giving up.")
			sys.exit(1)
	else:
		info("NetworkManager says we're on-line. Good!")

	policy.freshness = 0			# Don't bother trying to refresh when getting the interface
	refresh = policy.refresh_all()		# (causes confusing log messages)
	policy.handler.wait_for_blocker(refresh)

	# We could even download the archives here, but for now just
	# update the interfaces.

	if not policy.need_download():
		if verbose:
			policy.handler.notify("Zero Install", "No updates to download.", timeout = 1)
		sys.exit(0)

	if not policy.handler.have_actions_support():
		# Can't ask the user to choose, so just notify them
		# In particular, Ubuntu/Jaunty doesn't support actions
		policy.handler.notify("Zero Install",
				      "Updates ready to download for '%s'." % root_iface,
				      timeout = 1)
		_exec_gui(policy.root, '--refresh', '--download-only', '--systray')
		sys.exit(1)

	notification_closed = tasks.Blocker("wait for notification response")

	def _NotificationClosed(nid, *unused):
		if nid != our_question: return
		notification_closed.trigger()

	def _ActionInvoked(nid, action):
		if nid != our_question: return
		if action == 'download':
			_exec_gui(policy.root)
		notification_closed.trigger()

	policy.handler.notification_service.connect_to_signal('NotificationClosed', _NotificationClosed)
	policy.handler.notification_service.connect_to_signal('ActionInvoked', _ActionInvoked)

	our_question = policy.handler.notify("Zero Install", "Updates ready to download for '%s'." % root_iface,
				actions = ['download', 'Download'])

	policy.handler.wait_for_blocker(notification_closed)

def spawn_background_update(policy, verbose):
	"""Spawn a detached child process to check for updates.
	@param policy: policy containing interfaces to update
	@type policy: L{policy.Policy}
	@param verbose: whether to notify the user about minor events
	@type verbose: bool"""
	# Mark all feeds as being updated. Do this before forking, so that if someone is
	# running lots of 0launch commands in series on the same program we don't start
	# huge numbers of processes.
	for x in policy.implementation:
		iface_cache.mark_as_checking(x.uri)			# Main feed
		for f in policy.usable_feeds(x):
			iface_cache.mark_as_checking(f.uri)		# Extra feeds

	if _detach():
		return

	try:
		try:
			_check_for_updates(policy, verbose)
		except SystemExit:
			raise
		except:
			import traceback
			traceback.print_exc()
			sys.stdout.flush()
	finally:
		os._exit(1)
<commit_msg>Include full stack trace for unexpected errors during background updates<commit_after>"""
Check for updates in a background process. If we can start a program immediately, but some of our information
is rather old (longer that the L{policy.Policy.freshness} threshold) then we run it anyway, and check for updates using a new
process that runs quietly in the background.

This avoids the need to annoy people with a 'checking for updates' box when they're trying to run things.
"""

import sys, os
from logging import info, warn
from zeroinstall.support import tasks
from zeroinstall.injector.iface_cache import iface_cache
from zeroinstall.injector import handler, namespaces

# Copyright (C) 2009, Thomas Leonard
# See the README file for details, or visit http://0install.net.

def _escape_xml(s):
	return s.replace('&', '&').replace('<', '<')

def _exec_gui(uri, *args):
	os.execvp('0launch', ['0launch', '--download-only', '--gui'] + list(args) + [uri])

class _NetworkState:
	NM_STATE_UNKNOWN = 0
	NM_STATE_ASLEEP = 1
	NM_STATE_CONNECTING = 2
	NM_STATE_CONNECTED = 3
	NM_STATE_DISCONNECTED = 4

class BackgroundHandler(handler.Handler):
	"""A Handler for non-interactive background updates. Runs the GUI if interaction is required."""
	def __init__(self, title, root):
		handler.Handler.__init__(self)
		self.title = title
		self.notification_service = None
		self.network_manager = None
		self.notification_service_caps = []
		self.root = root	# If we need to confirm any keys, run the GUI on this

		try:
			import dbus
			import dbus.glib
		except Exception, ex:
			info("Failed to import D-BUS bindings: %s", ex)
			return

		try:
			session_bus = dbus.SessionBus()
			remote_object = session_bus.get_object('org.freedesktop.Notifications',
								'/org/freedesktop/Notifications')

			self.notification_service = dbus.Interface(remote_object,
							'org.freedesktop.Notifications')

			# The Python bindings insist on printing a pointless introspection
			# warning to stderr if the service is missing. Force it to be done
			# now so we can skip it
			old_stderr = sys.stderr
			sys.stderr = None
			try:
				self.notification_service_caps = [str(s) for s in
						self.notification_service.GetCapabilities()]
			finally:
				sys.stderr = old_stderr
		except Exception, ex:
			info("No D-BUS notification service available: %s", ex)

		try:
			system_bus = dbus.SystemBus()
			remote_object = system_bus.get_object('org.freedesktop.NetworkManager',
								'/org/freedesktop/NetworkManager')

			self.network_manager = dbus.Interface(remote_object,
							'org.freedesktop.NetworkManager')
		except Exception, ex:
			info("No D-BUS network manager service available: %s", ex)

	def get_network_state(self):
		if self.network_manager:
			try:
				return self.network_manager.state()
			except Exception, ex:
				warn("Error getting network state: %s", ex)
		return _NetworkState.NM_STATE_UNKNOWN

	def confirm_trust_keys(self, interface, sigs, iface_xml):
		"""Run the GUI if we need to confirm any keys."""
		info("Can't update interface; signature not yet trusted. Running GUI...")
		_exec_gui(self.root, '--refresh', '--download-only', '--systray')

	def report_error(self, exception, tb = None):
		from zeroinstall.injector import download
		if isinstance(exception, download.DownloadError):
			tb = None

		if tb:
			import traceback
			details = '\n' + '\n'.join(traceback.format_exception(type(exception), exception, tb))
		else:
			details = str(exception)
		self.notify("Zero Install", "Error updating %s: %s" % (self.title, details))

	def notify(self, title, message, timeout = 0, actions = []):
		"""Send a D-BUS notification message if possible. If there is no notification
		service available, log the message instead."""
		if not self.notification_service:
			info('%s: %s', title, message)
			return None

		LOW = 0
		NORMAL = 1
		CRITICAL = 2

		import dbus.types

		hints = {}
		if actions:
			hints['urgency'] = dbus.types.Byte(NORMAL)
		else:
			hints['urgency'] = dbus.types.Byte(LOW)

		return self.notification_service.Notify('Zero Install',
			0,		# replaces_id,
			'',		# icon
			_escape_xml(title),
			_escape_xml(message),
			actions,
			hints,
			timeout * 1000)

	def have_actions_support(self):
		return 'actions' in self.notification_service_caps

def _detach():
	"""Fork a detached grandchild.
	@return: True if we are the original."""
	child = os.fork()
	if child:
		pid, status = os.waitpid(child, 0)
		assert pid == child
		return True

	# The calling process might be waiting for EOF from its child.
	# Close our stdout so we don't keep it waiting.
	# Note: this only fixes the most common case; it could be waiting
	# on any other FD as well. We should really use gobject.spawn_async
	# to close *all* FDs.
	null = os.open('/dev/null', os.O_RDWR)
	os.dup2(null, 1)
	os.close(null)

	grandchild = os.fork()
	if grandchild:
		os._exit(0)	# Parent's waitpid returns and grandchild continues

	return False

def _check_for_updates(policy, verbose):
	root_iface = iface_cache.get_interface(policy.root).get_name()

	policy.handler = BackgroundHandler(root_iface, policy.root)

	info("Checking for updates to '%s' in a background process", root_iface)
	if verbose:
		policy.handler.notify("Zero Install", "Checking for updates to '%s'..." % root_iface, timeout = 1)

	network_state = policy.handler.get_network_state()
	if network_state != _NetworkState.NM_STATE_CONNECTED:
		info("Not yet connected to network (status = %d). Sleeping for a bit...", network_state)
		import time
		time.sleep(120)
		if network_state in (_NetworkState.NM_STATE_DISCONNECTED, _NetworkState.NM_STATE_ASLEEP):
			info("Still not connected to network. Giving up.")
			sys.exit(1)
	else:
		info("NetworkManager says we're on-line. Good!")

	policy.freshness = 0			# Don't bother trying to refresh when getting the interface
	refresh = policy.refresh_all()		# (causes confusing log messages)
	policy.handler.wait_for_blocker(refresh)

	# We could even download the archives here, but for now just
	# update the interfaces.

	if not policy.need_download():
		if verbose:
			policy.handler.notify("Zero Install", "No updates to download.", timeout = 1)
		sys.exit(0)

	if not policy.handler.have_actions_support():
		# Can't ask the user to choose, so just notify them
		# In particular, Ubuntu/Jaunty doesn't support actions
		policy.handler.notify("Zero Install",
				      "Updates ready to download for '%s'." % root_iface,
				      timeout = 1)
		_exec_gui(policy.root, '--refresh', '--download-only', '--systray')
		sys.exit(1)

	notification_closed = tasks.Blocker("wait for notification response")

	def _NotificationClosed(nid, *unused):
		if nid != our_question: return
		notification_closed.trigger()

	def _ActionInvoked(nid, action):
		if nid != our_question: return
		if action == 'download':
			_exec_gui(policy.root)
		notification_closed.trigger()

	policy.handler.notification_service.connect_to_signal('NotificationClosed', _NotificationClosed)
	policy.handler.notification_service.connect_to_signal('ActionInvoked', _ActionInvoked)

	our_question = policy.handler.notify("Zero Install", "Updates ready to download for '%s'." % root_iface,
				actions = ['download', 'Download'])

	policy.handler.wait_for_blocker(notification_closed)

def spawn_background_update(policy, verbose):
	"""Spawn a detached child process to check for updates.
	@param policy: policy containing interfaces to update
	@type policy: L{policy.Policy}
	@param verbose: whether to notify the user about minor events
	@type verbose: bool"""
	# Mark all feeds as being updated. Do this before forking, so that if someone is
	# running lots of 0launch commands in series on the same program we don't start
	# huge numbers of processes.
	for x in policy.implementation:
		iface_cache.mark_as_checking(x.uri)			# Main feed
		for f in policy.usable_feeds(x):
			iface_cache.mark_as_checking(f.uri)		# Extra feeds

	if _detach():
		return

	try:
		try:
			_check_for_updates(policy, verbose)
		except SystemExit:
			raise
		except:
			import traceback
			traceback.print_exc()
			sys.stdout.flush()
	finally:
		os._exit(1)
<|endoftext|>