<commit_before># Copyright 2016 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
import requests
import json

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

class Measurement(object):
    def __init__(self, metric=None, value=None, source=None, timestamp=None, properties=None):
        self._metric = None
        self._value = None
        self._source = None
        self._timestamp = None
        self._properties = None
        self.metric = metric
        self.value = value
        self.source = source
        self.timestamp = timestamp
        self.properties = properties

    def __repr__(self):
        return "Measurement(metric='{0}', value={1}, source='{2}', timestamp={3})".format(self.metric, self.value, self.source, self.timestamp)

    @property
    def metric(self):
        return self._metric

    @metric.setter
    def metric(self, metric):
        self._metric = metric

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    @property
    def source(self):
        return self._source

    @source.setter
    def source(self, source):
        self._source = source

    @property
    def timestamp(self):
        return self._timestamp

    @timestamp.setter
    def timestamp(self, timestamp):
        self._timestamp = timestamp

    @property
    def properties(self):
        return self._properties

    @properties.setter
    def properties(self, properties):
        self._properties = properties

def serialize_instance(obj):
    logging.debug(type(obj))
    logging.debug(obj)
    d = []
    d.append(obj.source)
    d.append(obj.metric)
    d.append(obj.value)
    d.append(obj.timestamp)
    d.append(obj.properties)
    return d

def measurement_get_handle_results(api_result, context):
    logging.debug("measurement_get_handle_results")
    # Only process if we get HTTP result of 200
    measurements = None
    metric = context
    if api_result.status_code == requests.codes.ok:
        results = json.loads(api_result.text)
        measurements = []
        log.debug(api_result.text)
        for aggregate in results['result']['aggregates']['key']:
            timestamp = aggregate[0][0]
            for row in aggregate[1]:
                measurements.append(Measurement(metric=metric,
                                                source=row[0],
                                                value=row[1],
                                                timestamp=timestamp))
    return measurements
<commit_msg>Change logging to use module instance<commit_after># Copyright 2016 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
import requests
import json

log = logging.getLogger(__name__)

class Measurement(object):
    def __init__(self, metric=None, value=None, source=None, timestamp=None, properties=None):
        self._metric = None
        self._value = None
        self._source = None
        self._timestamp = None
        self._properties = None
        self.metric = metric
        self.value = value
        self.source = source
        self.timestamp = timestamp
        self.properties = properties

    def __repr__(self):
        return "Measurement(metric='{0}', value={1}, source='{2}', timestamp={3})".format(self.metric, self.value, self.source, self.timestamp)

    @property
    def metric(self):
        return self._metric

    @metric.setter
    def metric(self, metric):
        self._metric = metric

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    @property
    def source(self):
        return self._source

    @source.setter
    def source(self, source):
        self._source = source

    @property
    def timestamp(self):
        return self._timestamp

    @timestamp.setter
    def timestamp(self, timestamp):
        self._timestamp = timestamp

    @property
    def properties(self):
        return self._properties

    @properties.setter
    def properties(self, properties):
        self._properties = properties

def serialize_instance(obj):
    log.debug(type(obj))
    log.debug(obj)
    d = []
    d.append(obj.source)
    d.append(obj.metric)
    d.append(obj.value)
    d.append(obj.timestamp)
    d.append(obj.properties)
    return d

def measurement_get_handle_results(api_result, context):
    log.debug("measurement_get_handle_results")
    # Only process if we get HTTP result of 200
    measurements = None
    metric = context
    if api_result.status_code == requests.codes.ok:
        results = json.loads(api_result.text)
        measurements = []
        log.debug(api_result.text)
        for aggregate in results['result']['aggregates']['key']:
            timestamp = aggregate[0][0]
            for row in aggregate[1]:
                measurements.append(Measurement(metric=metric,
                                                source=row[0],
                                                value=row[1],
                                                timestamp=timestamp))
    return measurements
<|endoftext|>


<commit_before># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import StringIO
import numpy
import os
import shutil
import tempfile

from lxml import etree
from nose.plugins.attrib import attr

from openquake.nrmllib import PARSE_NS_MAP

from openquake.engine.db import models
from openquake.engine.export import hazard as haz_export

from qa_tests import _utils as qa_utils
from qa_tests.hazard.disagg.case_1 import _test_data as test_data

aac = lambda a, b: numpy.testing.assert_allclose(a, b, atol=1e-5)

class DisaggHazardCase1TestCase(qa_utils.BaseQATestCase):

    @attr('qa', 'hazard', 'disagg')
    def test(self):
        # TODO(LB): This is a temporary test case which tests for stability
        # until we can write proper QA tests.

        cfg = os.path.join(os.path.dirname(__file__), 'job.ini')

        job = self.run_hazard(cfg)

        results = models.DisaggResult.objects.filter(output__oq_job=job)

        poe_002_pga = results.filter(imt='PGA', poe=0.02)
        rlz1, rlz2 = poe_002_pga.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_002_PGA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_002_PGA, rlz2.matrix)

        poe_002_sa = results.filter(imt='SA', poe=0.02)
        rlz1, rlz2 = poe_002_sa.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_002_SA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_002_SA, rlz2.matrix)

        poe_01_pga = results.filter(imt='PGA', poe=0.1)
        rlz1, rlz2 = poe_01_pga.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_01_PGA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_01_PGA, rlz2.matrix)

        poe_01_sa = results.filter(imt='SA', poe=0.1)
        rlz1, rlz2 = poe_01_sa.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_01_SA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_01_SA, rlz2.matrix)

        # Lastly, we should check an export of at least one of these results to
        # ensure that the disagg export/serialization is working properly.
        # The export isn't just a simple dump from the database; it requires
        # extraction of PMFs (Probability Mass Function) from a 6d matrix,
        # which are then serialized to XML.
        # This is not a trivial operation.
        try:
            target_dir = tempfile.mkdtemp()
            result_file = haz_export.export(rlz1.output.id, target_dir)
            expected = StringIO.StringIO(test_data.EXPECTED_XML_DISAGG)
            self.assert_disagg_xml_almost_equal(expected, result_file)
            self.assertTrue(qa_utils.validates_against_xml_schema(result_file))
        finally:
            shutil.rmtree(target_dir)

    def assert_disagg_xml_almost_equal(self, expected, actual):
        """
        A special helper function to test that values in the ``expected`` and
        ``actual`` XML are almost equal to a certain precision.

        :param expected, actual:
            Paths to XML files, or file-like objects containing the XML
            contents.
        """
        exp_tree = etree.parse(expected)
        act_tree = etree.parse(actual)

        # First, compare the <disaggMatrices> container element, check attrs,
        # etc.
        [exp_dms] = exp_tree.xpath(
            '//nrml:disaggMatrices', namespaces=PARSE_NS_MAP
        )
        [act_dms] = act_tree.xpath(
            '//nrml:disaggMatrices', namespaces=PARSE_NS_MAP
        )
        self.assertEqual(exp_dms.attrib, act_dms.attrib)

        # Then, loop over each <disaggMatrix>, check attrs, then loop over each
        # <prob> and compare indices and values.
        exp_dm = exp_tree.xpath(
            '//nrml:disaggMatrix', namespaces=PARSE_NS_MAP
        )
        act_dm = act_tree.xpath(
            '//nrml:disaggMatrix', namespaces=PARSE_NS_MAP
        )
        self.assertEqual(len(exp_dm), len(act_dm))

        for i, matrix in enumerate(exp_dm):
            act_matrix = act_dm[i]

            self.assertEqual(matrix.attrib['type'], act_matrix.attrib['type'])
            self.assertEqual(matrix.attrib['dims'], act_matrix.attrib['dims'])
            self.assertEqual(matrix.attrib['poE'], act_matrix.attrib['poE'])
            aac(float(act_matrix.attrib['iml']), float(matrix.attrib['iml']))

            # compare probabilities
            exp_probs = matrix.xpath('./nrml:prob', namespaces=PARSE_NS_MAP)
            act_probs = act_matrix.xpath(
                './nrml:prob', namespaces=PARSE_NS_MAP
            )
            self.assertEqual(len(exp_probs), len(act_probs))

            for j, prob in enumerate(exp_probs):
                act_prob = act_probs[j]

                self.assertEqual(prob.attrib['index'],
                                 act_prob.attrib['index'])
                aac(float(act_prob.attrib['value']),
                    float(prob.attrib['value']))
<commit_msg>Increased the tolerance in the disaggregation test<commit_after># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import StringIO
import numpy
import os
import shutil
import tempfile

from lxml import etree
from nose.plugins.attrib import attr

from openquake.nrmllib import PARSE_NS_MAP

from openquake.engine.db import models
from openquake.engine.export import hazard as haz_export

from qa_tests import _utils as qa_utils
from qa_tests.hazard.disagg.case_1 import _test_data as test_data

aac = lambda a, b: numpy.testing.assert_allclose(a, b, atol=5e-3)

class DisaggHazardCase1TestCase(qa_utils.BaseQATestCase):

    @attr('qa', 'hazard', 'disagg')
    def test(self):
        # TODO(LB): This is a temporary test case which tests for stability
        # until we can write proper QA tests.

        cfg = os.path.join(os.path.dirname(__file__), 'job.ini')

        job = self.run_hazard(cfg)

        results = models.DisaggResult.objects.filter(output__oq_job=job)

        poe_002_pga = results.filter(imt='PGA', poe=0.02)
        rlz1, rlz2 = poe_002_pga.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_002_PGA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_002_PGA, rlz2.matrix)

        poe_002_sa = results.filter(imt='SA', poe=0.02)
        rlz1, rlz2 = poe_002_sa.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_002_SA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_002_SA, rlz2.matrix)

        poe_01_pga = results.filter(imt='PGA', poe=0.1)
        rlz1, rlz2 = poe_01_pga.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_01_PGA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_01_PGA, rlz2.matrix)

        poe_01_sa = results.filter(imt='SA', poe=0.1)
        rlz1, rlz2 = poe_01_sa.order_by('lt_realization')

        aac(test_data.RLZ_1_POE_01_SA, rlz1.matrix)
        aac(test_data.RLZ_2_POE_01_SA, rlz2.matrix)

        # Lastly, we should check an export of at least one of these results to
        # ensure that the disagg export/serialization is working properly.
        # The export isn't just a simple dump from the database; it requires
        # extraction of PMFs (Probability Mass Function) from a 6d matrix,
        # which are then serialized to XML.
        # This is not a trivial operation.
        try:
            target_dir = tempfile.mkdtemp()
            result_file = haz_export.export(rlz1.output.id, target_dir)
            expected = StringIO.StringIO(test_data.EXPECTED_XML_DISAGG)
            self.assert_disagg_xml_almost_equal(expected, result_file)
            self.assertTrue(qa_utils.validates_against_xml_schema(result_file))
        finally:
            shutil.rmtree(target_dir)

    def assert_disagg_xml_almost_equal(self, expected, actual):
        """
        A special helper function to test that values in the ``expected`` and
        ``actual`` XML are almost equal to a certain precision.

        :param expected, actual:
            Paths to XML files, or file-like objects containing the XML
            contents.
        """
        exp_tree = etree.parse(expected)
        act_tree = etree.parse(actual)

        # First, compare the <disaggMatrices> container element, check attrs,
        # etc.
        [exp_dms] = exp_tree.xpath(
            '//nrml:disaggMatrices', namespaces=PARSE_NS_MAP
        )
        [act_dms] = act_tree.xpath(
            '//nrml:disaggMatrices', namespaces=PARSE_NS_MAP
        )
        self.assertEqual(exp_dms.attrib, act_dms.attrib)

        # Then, loop over each <disaggMatrix>, check attrs, then loop over each
        # <prob> and compare indices and values.
        exp_dm = exp_tree.xpath(
            '//nrml:disaggMatrix', namespaces=PARSE_NS_MAP
        )
        act_dm = act_tree.xpath(
            '//nrml:disaggMatrix', namespaces=PARSE_NS_MAP
        )
        self.assertEqual(len(exp_dm), len(act_dm))

        for i, matrix in enumerate(exp_dm):
            act_matrix = act_dm[i]

            self.assertEqual(matrix.attrib['type'], act_matrix.attrib['type'])
            self.assertEqual(matrix.attrib['dims'], act_matrix.attrib['dims'])
            self.assertEqual(matrix.attrib['poE'], act_matrix.attrib['poE'])
            aac(float(act_matrix.attrib['iml']), float(matrix.attrib['iml']))

            # compare probabilities
            exp_probs = matrix.xpath('./nrml:prob', namespaces=PARSE_NS_MAP)
            act_probs = act_matrix.xpath(
                './nrml:prob', namespaces=PARSE_NS_MAP
            )
            self.assertEqual(len(exp_probs), len(act_probs))

            for j, prob in enumerate(exp_probs):
                act_prob = act_probs[j]

                self.assertEqual(prob.attrib['index'],
                                 act_prob.attrib['index'])
                aac(float(act_prob.attrib['value']),
                    float(prob.attrib['value']))
<|endoftext|>


<commit_before>"""Module contains classes for handling GAMS symbols"""

from typing import List, Dict

class _GAMSSymbol(object):
    """Abstarct class for handling GAMS symbols

    Attributes:
        name (str): Symbol name
        expl_text (str): Symbol explanatory text
    """
    def __init__(self, name: str, expl_text: str):
        """Class constructor

        Args:
            name (str): Symbol name
            expl_text (str, optional): Symbol explanatory text
        """
        self.name = name
        self.expl_text = expl_text
        self._type = None
        self.domain = None
        self.dimension = None

    def __str__(self):
        return self.expl_text

    def __repr__(self):
        text = f'{self._type} {self.name}'
        if self.domain:
            text += f'({",".join(str(d) for d in self.domain)})'  # pylint: disable=not-an-iterable
        if self.expl_text:
            text += f' "{self.expl_text}"'
        return text

class _GAMSNDimSymbol(_GAMSSymbol):
    """Abstarct class for N-dimensional GAMS symbols
    """
    def __init__(self, name: str, keys: List[tuple], domain: List[str], expl_text: str):
        """Constructor for GAMSNDimSymbol

        Args:
            name
            keys
            domain
            expl_text

        Raises:
            ValueError
        """

        super().__init__(name, expl_text)

        # Check arguments
        if not isinstance(keys, list):
            raise ValueError("Keys must be a list")
        if domain and not isinstance(domain, list):
            raise ValueError("Domain must be a list")

        # Calculate dimension
        first_key = keys[0]
        if isinstance(first_key, tuple):
            dimension = len(first_key)
            # Check consistency of the keys
            if not all(len(key) == dimension for key in keys):
                raise ValueError("Check consistency of the keys argument")
            else:
                self.dimension = dimension
        else:
            self.dimension = 1

        # Store keys
        self._keys = keys
        self._size = len(keys)

        # Check domain length
        if domain and (len(domain) != self.dimension):
            raise ValueError("Domain is inconsistent with the given values")
        else:
            self.domain = domain

    def __len__(self):
        return self._size

class GAMSSet(_GAMSNDimSymbol):
    """Class for GAMS Sets
    """
    def __init__(self, name: str, keys: List[tuple], domain: List[str] = None, expl_text: str = ''):
        """Constructor for GAMSSet

        Args:
            name: Name of the symbol
            keys: List of tuples of strings for the keys
            domain (optional): List of domain set names
            expl_text (optional): Explanatory text

        Raises:
            ValueError
        """

        super().__init__(name, keys, domain, expl_text)
        self._type = 'Set'

    @property
    def elements(self):
        return self._keys

    def __iter__(self):
        self._iterator = iter(self._keys)
        return self

    def __next__(self):
        return next(self._iterator)

class GAMSScalar(_GAMSSymbol):
    """Class for GAMS Scalars (0-dimensional Parameters)
    """
    def __init__(self, name: str, value: float, expl_text: str = ''):
        """Class constructor

        Args:
            name: Symbol name
            value: Value

        Raises:
            ValueError
        """

        super().__init__(name, expl_text)
        self._type = 'Scalar'
        self.dimension = 0
        self._value = float(value)

    def __float__(self):
        return self._value

    def __int__(self):
        return int(self._value)

class GAMSParameter(_GAMSNDimSymbol):
    """Class for GAMS Parameters
    """

    def __init__(self, name: str, data: Dict[tuple, float], domain: List[str] = None, expl_text: str = ''):
        """Constructor for GAMSParameter

        Args:
            name: Name of the symbol
            data: Dictionay of keys and values
            domain (optional): List of domain set names
            expl_text (optional): Explanatory text

        Raises:
            ValueError
        """

        # Check arguments
        if not isinstance(data, dict):
            raise ValueError("Data must be a dictionary")
        else:
            super().__init__(name, list(data.keys()), domain, expl_text)
            self._type = 'Parameter'
            self._data = data

    def keys(self):
        return self._data.keys()

    def values(self):
        return self._data.values()

    def __getitem__(self, key):
        return self._data[key]

    def __iter__(self):
        self._iterator = iter(self._data.items())
        return self

    def __next__(self):
        key, val = next(self._iterator)
        return key, float(val)
<commit_msg>Change argument type annotations to more generic ones<commit_after>"""Module contains classes for handling GAMS symbols"""

from typing import Sequence, Mapping

class _GAMSSymbol(object):
    """Abstarct class for handling GAMS symbols

    Attributes:
        name (str): Symbol name
        expl_text (str): Symbol explanatory text
    """
    def __init__(self, name: str, expl_text: str):
        """Class constructor

        Args:
            name (str): Symbol name
            expl_text (str, optional): Symbol explanatory text
        """
        self.name = name
        self.expl_text = expl_text
        self._type = None
        self.domain = None
        self.dimension = None

    def __str__(self):
        return self.expl_text

    def __repr__(self):
        text = f'{self._type} {self.name}'
        if self.domain:
            text += f'({",".join(str(d) for d in self.domain)})'  # pylint: disable=not-an-iterable
        if self.expl_text:
            text += f' "{self.expl_text}"'
        return text

class _GAMSNDimSymbol(_GAMSSymbol):
    """Abstarct class for N-dimensional GAMS symbols
    """
    def __init__(self, name: str, keys: Sequence[tuple], domain: Sequence[str], expl_text: str):
        """Constructor for GAMSNDimSymbol

        Args:
            name
            keys
            domain
            expl_text

        Raises:
            ValueError
        """

        super().__init__(name, expl_text)

        # Calculate dimension
        try:
            first_key = keys[0]
        except TypeError:
            raise ValueError("Keys must be a sequence")
        if isinstance(first_key, tuple):
            dimension = len(first_key)
            # Check consistency of the keys
            if not all(len(key) == dimension for key in keys):
                raise ValueError("Check consistency of the keys argument")
            else:
                self.dimension = dimension
        else:
            self.dimension = 1

        # Store keys
        self._keys = keys
        self._size = len(keys)

        # Check domain length
        if domain:
            try:
                if len(domain) != self.dimension:
                    raise ValueError("Domain is inconsistent with the given values")
            except TypeError:
                raise ValueError("Domain must be a sequence")
            self.domain = domain

    def __len__(self):
        return self._size

class GAMSSet(_GAMSNDimSymbol):
    """Class for GAMS Sets
    """
    def __init__(self, name: str, keys: Sequence[tuple], domain: Sequence[str] = None,
                 expl_text: str = ''):
        """Constructor for GAMSSet

        Args:
            name: Name of the symbol
            keys: Sequence of tuples of strings for the keys
            domain (optional): Sequence of domain set names
            expl_text (optional): Explanatory text

        Raises:
            ValueError
        """

        super().__init__(name, keys, domain, expl_text)
        self._type = 'Set'

    @property
    def elements(self):
        return self._keys

    def __iter__(self):
        self._iterator = iter(self._keys)
        return self

    def __next__(self):
        return next(self._iterator)

class GAMSScalar(_GAMSSymbol):
    """Class for GAMS Scalars (0-dimensional Parameters)
    """
    def __init__(self, name: str, value: float, expl_text: str = ''):
        """Class constructor

        Args:
            name: Symbol name
            value: Value

        Raises:
            ValueError
        """

        super().__init__(name, expl_text)
        self._type = 'Scalar'
        self.dimension = 0
        self._value = float(value)

    def __float__(self):
        return self._value

    def __int__(self):
        return int(self._value)

class GAMSParameter(_GAMSNDimSymbol):
    """Class for GAMS Parameters
    """

    def __init__(self, name: str, data: Mapping[tuple, float], domain: Sequence[str] = None, expl_text: str = ''):
        """Constructor for GAMSParameter

        Args:
            name: Name of the symbol
            data: Dictionay of keys and values
            domain (optional): List of domain set names
            expl_text (optional): Explanatory text

        Raises:
            ValueError
        """

        # Check arguments
        if not isinstance(data, dict):
            raise ValueError("Data must be a dictionary")
        else:
            super().__init__(name, list(data.keys()), domain, expl_text)
            self._type = 'Parameter'
            self._data = data

    def keys(self):
        return self._data.keys()

    def values(self):
        return self._data.values()

    def __getitem__(self, key):
        return self._data[key]

    def __iter__(self):
        self._iterator = iter(self._data.items())
        return self

    def __next__(self):
        key, val = next(self._iterator)
        return key, float(val)
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

from nose.plugins.attrib import attr

from qa_tests import risk
from openquake.engine.tests.utils import helpers
from openquake.engine.db import models
from openquake.engine.tools.import_gmf_scenario import import_gmf_scenario

class ScenarioRiskCase1TestCase(risk.BaseRiskQATestCase):
    output_type = "gmf_scenario"

    @attr('qa', 'risk', 'scenario')
    def test(self):
        self._run_test()

    def get_hazard_job(self):
        job = helpers.get_hazard_job(
            helpers.get_data_path("scenario_hazard/job.ini"))
        fname = self._test_path('gmf_scenario.csv')
        helpers.populate_gmf_data_from_csv(job, fname)
        return job

    def actual_data(self, job):
        maps = models.LossMapData.objects.filter(
            loss_map__output__oq_job=job,
            loss_map__insured=False).order_by('asset_ref', 'value')
        agg = models.AggregateLoss.objects.get(output__oq_job=job,
                                               insured=False)
        insured_maps = models.LossMapData.objects.filter(
            loss_map__output__oq_job=job,
            loss_map__insured=True).order_by('asset_ref', 'value')
        insured_agg = models.AggregateLoss.objects.get(
            output__oq_job=job,
            insured=True)

        return [[[m.value, m.std_dev] for m in maps],
                [agg.mean, agg.std_dev],
                [[m.value, m.std_dev] for m in insured_maps],
                [insured_agg.mean, insured_agg.std_dev]]

    def expected_data(self):
        return [[[440.14707, 182.6159],
                 [432.2254, 186.8644],
                 [180.7175, 92.2122]],
                [1053.09, 246.62],
                [[147.49208753, 141.02819028],
                 [104.85957932, 145.36984417],
                 [76.75091081, 69.96900115]],
                [329.10257766, 193.67786848]]

class ImportGmfScenarioTestCase(risk.BaseRiskQATestCase):
    output_type = "gmf_scenario"

    @attr('qa', 'risk', 'scenario')
    def test(self):
        # check that the imported GMFs can be read by the risk calculator
        self._run_test()

    def get_hazard_job(self):
        with open('gmf-scenario.xml') as data:
            output = import_gmf_scenario(data)
        return output.oq_job
<commit_msg>Changed from relative path to full path<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

from nose.plugins.attrib import attr

from qa_tests import risk
from openquake.engine.tests.utils import helpers
from openquake.engine.db import models
from openquake.engine.tools.import_gmf_scenario import import_gmf_scenario

class ScenarioRiskCase1TestCase(risk.BaseRiskQATestCase):
    output_type = "gmf_scenario"

    @attr('qa', 'risk', 'scenario')
    def test(self):
        self._run_test()

    def get_hazard_job(self):
        job = helpers.get_hazard_job(
            helpers.get_data_path("scenario_hazard/job.ini"))
        fname = self._test_path('gmf_scenario.csv')
        helpers.populate_gmf_data_from_csv(job, fname)
        return job

    def actual_data(self, job):
        maps = models.LossMapData.objects.filter(
            loss_map__output__oq_job=job,
            loss_map__insured=False).order_by('asset_ref', 'value')
        agg = models.AggregateLoss.objects.get(output__oq_job=job,
                                               insured=False)
        insured_maps = models.LossMapData.objects.filter(
            loss_map__output__oq_job=job,
            loss_map__insured=True).order_by('asset_ref', 'value')
        insured_agg = models.AggregateLoss.objects.get(
            output__oq_job=job,
            insured=True)

        return [[[m.value, m.std_dev] for m in maps],
                [agg.mean, agg.std_dev],
                [[m.value, m.std_dev] for m in insured_maps],
                [insured_agg.mean, insured_agg.std_dev]]

    def expected_data(self):
        return [[[440.14707, 182.6159],
                 [432.2254, 186.8644],
                 [180.7175, 92.2122]],
                [1053.09, 246.62],
                [[147.49208753, 141.02819028],
                 [104.85957932, 145.36984417],
                 [76.75091081, 69.96900115]],
                [329.10257766, 193.67786848]]

class ImportGmfScenarioTestCase(risk.BaseRiskQATestCase):
    output_type = "gmf_scenario"

    @attr('qa', 'risk', 'scenario')
    def test(self):
        # check that the imported GMFs can be read by the risk calculator
        self._run_test()

    def get_hazard_job(self):
        with open(self._test_path('gmf-scenario.xml')) as data:
            output = import_gmf_scenario(data)
        return output.oq_job
<|endoftext|>


<commit_before># Copyright (c) 2014 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
"""PLY-based parser for TuLiP LTL syntax,
using AST classes from spec.ast
"""
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import warnings
import ply.lex
import ply.yacc
import tulip.spec.ast

TABMODULE = 'tulip.spec.parsetab'
LEX_LOGGER = 'tulip.ltl_lex_log'
YACC_LOGGER = 'tulip.ltl_yacc_log'
PARSER_LOGGER = 'tulip.ltl_parser_log'
# TODO: add past fragment of LTL

class Lexer(object):
    """Token rules to build LTL lexer."""

    reserved = {
        'next': 'NEXT',
        'X': 'XNEXT',
        'false': 'FALSE',
        'true': 'TRUE',
        'G': 'ALWAYS',
        'F': 'EVENTUALLY',
        'U': 'UNTIL',
        'R': 'RELEASE'}

    delimiters = ['LPAREN', 'RPAREN', 'DQUOTES']

    operators = [
        'NOT', 'AND', 'OR', 'XOR', 'IMP', 'BIMP',
        'EQUALS', 'NEQUALS', 'LT', 'LE', 'GT', 'GE',
        'PLUS', 'MINUS', 'TIMES', 'DIV', 'PRIME']

    misc = ['NAME', 'NUMBER', 'COMMENT', 'NEWLINE']

    def __init__(self, debug=False):
        # for setting the logger, call build explicitly
        self.tokens = (
            self.delimiters + self.operators +
            self.misc + self.reserved.values())
        self.build(debug=debug)

    def t_NAME(self, t):
        r'[A-Za-z_][A-za-z0-9._:]*'
        t.type = self.reserved.get(t.value, 'NAME')
        # special treatment
        if t.value.lower() in {'false', 'true'}:
            t.type = self.reserved[t.value.lower()]
        return t

    # t_PRIME  = r'\''

    def t_ALWAYS(self, t):
        r'\[\]'
        # use single letter as more readable and efficient
        t.value = 'G'
        return t

    def t_EVENTUALLY(self, t):
        r'\<\>'
        t.value = 'F'
        return t

    def t_AND(self, t):
        r'\&\&|\&'
        t.value = '&'
        return t

    def t_OR(self, t):
        r'\|\||\|'
        t.value = '|'
        return t

    t_NOT = r'\!'

    t_XOR = r'\^'

    t_EQUALS = r'\='  # a declarative language has no assignment
    t_NEQUALS = r'\!\='
    t_LT = r'\<'
    t_LE = r'\<\='
    t_GT = r'>\='
    t_GE = r'>'

    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_NUMBER = r'\d+'

    t_IMP = '->'
    t_BIMP = '\<->'

    t_PLUS = r'\+'
    t_MINUS = r'-'
    t_TIMES = r'\*'
    t_DIV = r'/'

    t_DQUOTES = r'\"'
    t_PRIME = r"\'"

    t_ignore = " \t"

    def t_COMMENT(self, t):
        r'\#.*'
        return

    def t_NEWLINE(self, t):
        r'\n+'
        t.lexer.lineno += t.value.count("\n")

    def t_error(self, t):
        warnings.warn('Illegal character "{t}"'.format(t=t.value[0]))
        t.lexer.skip(1)

    def build(self, debug=False, debuglog=None, **kwargs):
        """Create a lexer.

        @param kwargs: Same arguments as C{ply.lex.lex}:

          - except for C{module} (fixed to C{self})
          - C{debuglog} defaults to the logger C{"ltl_lex_log"}.
        """
        if debug and debuglog is None:
            debuglog = logging.getLogger(LEX_LOGGER)

        self.lexer = ply.lex.lex(
            module=self,
            debug=debug,
            debuglog=debuglog,
            **kwargs)

class Parser(object):
    """Production rules to build LTL parser."""

    tabmodule = TABMODULE
    start = 'expression'

    # lowest to highest
    precedence = (
        ('right', 'UNTIL', 'RELEASE'),
        ('right', 'BIMP'),
        ('right', 'IMP'),
        ('left', 'XOR'),
        ('left', 'OR'),
        ('left', 'AND'),
        ('right', 'ALWAYS', 'EVENTUALLY'),
        ('right', 'NOT'),
        ('left', 'EQUALS', 'NEQUALS'),
        ('left', 'LT', 'LE', 'GT', 'GE'),
        ('left', 'PLUS', 'MINUS'),
        ('left', 'TIMES', 'DIV'),
        ('right', 'NEXT'),
        ('left', 'PRIME'),
        ('nonassoc', 'TRUE', 'FALSE'))

    def __init__(self, ast=None, lexer=None):
        if ast is None:
            ast = tulip.spec.ast.nodes
        if lexer is None:
            lexer = Lexer()
        self.ast = ast
        self.lexer = lexer
        self.tokens = self.lexer.tokens
        self.build()

    def build(self):
        self.parser = ply.yacc.yacc(
            module=self,
            tabmodule=self.tabmodule,
            write_tables=False,
            debug=False)

    def rebuild_parsetab(self, tabmodule, outputdir='',
                         debug=True, debuglog=None):
        """Rebuild parsetable in debug mode.

        @param tabmodule: name of table file
        @type tabmodule: C{str}

        @param outputdir: save C{tabmodule} in this directory.
        @type outputdir: c{str}

        @param debuglog: defaults to logger C{"ltl_yacc_log"}.
        @type debuglog: C{logging.Logger}
        """
        if debug and debuglog is None:
            debuglog = logging.getLogger(YACC_LOGGER)
        self.lexer.build(debug=debug)
        self.parser = ply.yacc.yacc(
            module=self,
            tabmodule=tabmodule,
            outputdir=outputdir,
            write_tables=True,
            debug=debug,
            debuglog=debuglog)

    def parse(self, formula, debuglog=None):
        """Parse formula string and create abstract syntax tree (AST).

        @param logger: defaults to logger C{"ltl_parser_log"}.
        @type logger: C{logging.Logger}
        """
        if debuglog is None:
            debuglog = logging.getLogger(PARSER_LOGGER)
        root = self.parser.parse(
            formula,
            lexer=self.lexer.lexer,
            debug=debuglog)
        if root is None:
            raise Exception('failed to parse:\n\t{f}'.format(f=formula))
        return root

    def p_arithmetic(self, p):
        """expression : expression TIMES expression
                      | expression DIV expression
                      | expression PLUS expression
                      | expression MINUS expression
        """
        p[0] = self.ast.Arithmetic(p[2], p[1], p[3])

    def p_comparator(self, p):
        """expression : expression EQUALS expression
                      | expression NEQUALS expression
                      | expression LT expression
                      | expression LE expression
                      | expression GT expression
                      | expression GE expression
        """
        p[0] = self.ast.Comparator(p[2], p[1], p[3])

    def p_binary(self, p):
        """expression : expression AND expression
                      | expression OR expression
                      | expression XOR expression
                      | expression IMP expression
                      | expression BIMP expression
                      | expression UNTIL expression
                      | expression RELEASE expression
        """
        p[0] = self.ast.Binary(p[2], p[1], p[3])

    def p_unary(self, p):
        """expression : NOT expression
                      | ALWAYS expression
                      | EVENTUALLY expression
        """
        p[0] = self.ast.Unary(p[1], p[2])

    def p_prefix_next(self, p):
        """expression : NEXT expression
                      | XNEXT expression
        """
        p[0] = self.ast.Unary('X', p[2])

    def p_postfix_next(self, p):
        """expression : expression PRIME"""
        p[0] = self.ast.Unary('X', p[1])

    def p_group(self, p):
        """expression : LPAREN expression RPAREN"""
        p[0] = p[2]

    def p_number(self, p):
        """expression : NUMBER"""
        p[0] = self.ast.Num(p[1])

    def p_expression_name(self, p):
        """expression : NAME"""
        p[0] = self.ast.Var(p[1])

    def p_expression_str(self, p):
        """expression : DQUOTES NAME DQUOTES"""
        p[0] = self.ast.Str(p[2])

    def p_bool(self, p):
        """expression : TRUE
                      | FALSE
        """
        p[0] = self.ast.Bool(p[1])

    def p_error(self, p):
        print('Syntax error at "{p}"'.format(p=p.value))

def parse(formula):
    warnings.warn('Deprecated: Better to instantiate a Parser once only.')
    parser = Parser()
    return parser.parse(formula)

if __name__ == '__main__':
    s = 'up && !(loc = 29) && X((u_in = 0) || (u_in = 2))'
    parsed_formula = parse(s)
    print('Parsing result: {s}'.format(s=parsed_formula))
<commit_msg>rm comment and newline lexer rules from token list<commit_after># Copyright (c) 2014 by California Institute of Technology
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the California Institute of Technology nor
#    the names of its contributors may be used to endorse or promote
#    products derived from this software without specific prior
#    written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CALTECH
# OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
"""PLY-based parser for TuLiP LTL syntax,
using AST classes from spec.ast
"""
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import warnings
import ply.lex
import ply.yacc
import tulip.spec.ast

TABMODULE = 'tulip.spec.parsetab'
LEX_LOGGER = 'tulip.ltl_lex_log'
YACC_LOGGER = 'tulip.ltl_yacc_log'
PARSER_LOGGER = 'tulip.ltl_parser_log'
# TODO: add past fragment of LTL

class Lexer(object):
    """Token rules to build LTL lexer."""

    reserved = {
        'next': 'NEXT',
        'X': 'XNEXT',
        'false': 'FALSE',
        'true': 'TRUE',
        'G': 'ALWAYS',
        'F': 'EVENTUALLY',
        'U': 'UNTIL',
        'R': 'RELEASE'}

    delimiters = ['LPAREN', 'RPAREN', 'DQUOTES']

    operators = [
        'NOT', 'AND', 'OR', 'XOR', 'IMP', 'BIMP',
        'EQUALS', 'NEQUALS', 'LT', 'LE', 'GT', 'GE',
        'PLUS', 'MINUS', 'TIMES', 'DIV', 'PRIME']

    misc = ['NAME', 'NUMBER']

    def __init__(self, debug=False):
        # for setting the logger, call build explicitly
        self.tokens = (
            self.delimiters + self.operators +
            self.misc + self.reserved.values())
        self.build(debug=debug)

    def t_NAME(self, t):
        r'[A-Za-z_][A-za-z0-9._:]*'
        t.type = self.reserved.get(t.value, 'NAME')
        # special treatment
        if t.value.lower() in {'false', 'true'}:
            t.type = self.reserved[t.value.lower()]
        return t

    # t_PRIME  = r'\''

    def t_ALWAYS(self, t):
        r'\[\]'
        # use single letter as more readable and efficient
        t.value = 'G'
        return t

    def t_EVENTUALLY(self, t):
        r'\<\>'
        t.value = 'F'
        return t

    def t_AND(self, t):
        r'\&\&|\&'
        t.value = '&'
        return t

    def t_OR(self, t):
        r'\|\||\|'
        t.value = '|'
        return t

    t_NOT = r'\!'

    t_XOR = r'\^'

    t_EQUALS = r'\='  # a declarative language has no assignment
    t_NEQUALS = r'\!\='
    t_LT = r'\<'
    t_LE = r'\<\='
    t_GT = r'>\='
    t_GE = r'>'

    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_NUMBER = r'\d+'

    t_IMP = '->'
    t_BIMP = '\<->'

    t_PLUS = r'\+'
    t_MINUS = r'-'
    t_TIMES = r'\*'
    t_DIV = r'/'

    t_DQUOTES = r'\"'
    t_PRIME = r"\'"

    t_ignore = " \t"

    def t_comment(self, t):
        r'\#.*'
        return

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += t.value.count("\n")

    def t_error(self, t):
        warnings.warn('Illegal character "{t}"'.format(t=t.value[0]))
        t.lexer.skip(1)

    def build(self, debug=False, debuglog=None, **kwargs):
        """Create a lexer.

        @param kwargs: Same arguments as C{ply.lex.lex}:

          - except for C{module} (fixed to C{self})
          - C{debuglog} defaults to the logger C{"ltl_lex_log"}.
        """
        if debug and debuglog is None:
            debuglog = logging.getLogger(LEX_LOGGER)

        self.lexer = ply.lex.lex(
            module=self,
            debug=debug,
            debuglog=debuglog,
            **kwargs)

class Parser(object):
    """Production rules to build LTL parser."""

    tabmodule = TABMODULE
    start = 'expression'

    # lowest to highest
    precedence = (
        ('right', 'UNTIL', 'RELEASE'),
        ('right', 'BIMP'),
        ('right', 'IMP'),
        ('left', 'XOR'),
        ('left', 'OR'),
        ('left', 'AND'),
        ('right', 'ALWAYS', 'EVENTUALLY'),
        ('right', 'NOT'),
        ('left', 'EQUALS', 'NEQUALS'),
        ('left', 'LT', 'LE', 'GT', 'GE'),
        ('left', 'PLUS', 'MINUS'),
        ('left', 'TIMES', 'DIV'),
        ('right', 'NEXT'),
        ('left', 'PRIME'),
        ('nonassoc', 'TRUE', 'FALSE'))

    def __init__(self, ast=None, lexer=None):
        if ast is None:
            ast = tulip.spec.ast.nodes
        if lexer is None:
            lexer = Lexer()
        self.ast = ast
        self.lexer = lexer
        self.tokens = self.lexer.tokens
        self.build()

    def build(self):
        self.parser = ply.yacc.yacc(
            module=self,
            tabmodule=self.tabmodule,
            write_tables=False,
            debug=False)

    def rebuild_parsetab(self, tabmodule, outputdir='',
                         debug=True, debuglog=None):
        """Rebuild parsetable in debug mode.

        @param tabmodule: name of table file
        @type tabmodule: C{str}

        @param outputdir: save C{tabmodule} in this directory.
        @type outputdir: c{str}

        @param debuglog: defaults to logger C{"ltl_yacc_log"}.
        @type debuglog: C{logging.Logger}
        """
        if debug and debuglog is None:
            debuglog = logging.getLogger(YACC_LOGGER)
        self.lexer.build(debug=debug)
        self.parser = ply.yacc.yacc(
            module=self,
            tabmodule=tabmodule,
            outputdir=outputdir,
            write_tables=True,
            debug=debug,
            debuglog=debuglog)

    def parse(self, formula, debuglog=None):
        """Parse formula string and create abstract syntax tree (AST).

        @param logger: defaults to logger C{"ltl_parser_log"}.
        @type logger: C{logging.Logger}
        """
        if debuglog is None:
            debuglog = logging.getLogger(PARSER_LOGGER)
        root = self.parser.parse(
            formula,
            lexer=self.lexer.lexer,
            debug=debuglog)
        if root is None:
            raise Exception('failed to parse:\n\t{f}'.format(f=formula))
        return root

    def p_arithmetic(self, p):
        """expression : expression TIMES expression
                      | expression DIV expression
                      | expression PLUS expression
                      | expression MINUS expression
        """
        p[0] = self.ast.Arithmetic(p[2], p[1], p[3])

    def p_comparator(self, p):
        """expression : expression EQUALS expression
                      | expression NEQUALS expression
                      | expression LT expression
                      | expression LE expression
                      | expression GT expression
                      | expression GE expression
        """
        p[0] = self.ast.Comparator(p[2], p[1], p[3])

    def p_binary(self, p):
        """expression : expression AND expression
                      | expression OR expression
                      | expression XOR expression
                      | expression IMP expression
                      | expression BIMP expression
                      | expression UNTIL expression
                      | expression RELEASE expression
        """
        p[0] = self.ast.Binary(p[2], p[1], p[3])

    def p_unary(self, p):
        """expression : NOT expression
                      | ALWAYS expression
                      | EVENTUALLY expression
        """
        p[0] = self.ast.Unary(p[1], p[2])

    def p_prefix_next(self, p):
        """expression : NEXT expression
                      | XNEXT expression
        """
        p[0] = self.ast.Unary('X', p[2])

    def p_postfix_next(self, p):
        """expression : expression PRIME"""
        p[0] = self.ast.Unary('X', p[1])

    def p_group(self, p):
        """expression : LPAREN expression RPAREN"""
        p[0] = p[2]

    def p_number(self, p):
        """expression : NUMBER"""
        p[0] = self.ast.Num(p[1])

    def p_expression_name(self, p):
        """expression : NAME"""
        p[0] = self.ast.Var(p[1])

    def p_expression_str(self, p):
        """expression : DQUOTES NAME DQUOTES"""
        p[0] = self.ast.Str(p[2])

    def p_bool(self, p):
        """expression : TRUE
                      | FALSE
        """
        p[0] = self.ast.Bool(p[1])

    def p_error(self, p):
        print('Syntax error at "{p}"'.format(p=p.value))

def parse(formula):
    warnings.warn('Deprecated: Better to instantiate a Parser once only.')
    parser = Parser()
    return parser.parse(formula)

if __name__ == '__main__':
    s = 'up && !(loc = 29) && X((u_in = 0) || (u_in = 2))'
    parsed_formula = parse(s)
    print('Parsing result: {s}'.format(s=parsed_formula))
<|endoftext|>


<commit_before>'''
    NeuroLearn Mask Classes
    =======================
    Classes to represent masks

'''

__all__ = ['create_sphere', 'expand_mask', 'collapse_mask']
__author__ = ["Luke Chang", "Sam Greydanus"]
__license__ = "MIT"

import os
import nibabel as nib
from nltools.utils import get_resource_path
from nilearn.input_data import NiftiMasker
from copy import deepcopy
import pandas as pd
import numpy as np
import warnings
from nilearn.masking import intersect_masks

# from neurosynth.masks import Masker

def create_sphere(coordinates, radius=5, mask=None):
    """ Generate a set of spheres in the brain mask space

    Args:
        radius: vector of radius.  Will create multiple spheres if len(radius) > 1
        centers: a vector of sphere centers of the form [px, py, pz] or [[px1, py1, pz1], ..., [pxn, pyn, pzn]]

    """

    if mask is not None:
        if not isinstance(mask,nib.Nifti1Image):
            if type(mask) is str:
                if os.path.isfile(mask):
                    data = nib.load(mask)
            else:
                raise ValueError("mask is not a nibabel instance or a valid file name")
    else:
        mask = nib.load(os.path.join(get_resource_path(),'MNI152_T1_2mm_brain_mask.nii.gz'))
    dims = mask.get_data().shape

    def sphere(r, p, mask):
        """ create a sphere of given radius at some point p in the brain mask

        Args:
            r: radius of the sphere
            p: point (in coordinates of the brain mask) of the center of the sphere

        """
        dims = mask.shape

        x, y, z = np.ogrid[-p[0]:dims[0]-p[0], -p[1]:dims[1]-p[1], -p[2]:dims[2]-p[2]]
        mask_r = x*x + y*y + z*z <= r*r

        activation = np.zeros(dims)
        activation[mask_r] = 1
        activation = np.multiply(activation, mask.get_data())
        activation = nib.Nifti1Image(activation, affine=np.eye(4))

        #return the 3D numpy matrix of zeros containing the sphere as a region of ones
        return activation.get_data()

    # Initialize Spheres with options for multiple radii and centers of the spheres (or just an int and a 3D list)
    if type(radius) is int:
        radius = [radius]
    if coordinates is None:
        coordinates = [[dims[0]/2, dims[1]/2, dims[2]/2] * len(radius)] #default value for centers
    elif type(coordinates) is list and type(coordinates[0]) is int and len(radius) is 1:
        coordinates = [coordinates]
    if (type(radius)) is list and (type(coordinates) is list) and (len(radius) == len(coordinates)):
        A = np.zeros_like(mask.get_data())
        for i in xrange(len(radius)):
            A = np.add(A, sphere(radius[i], coordinates[i], mask))
        nifti_sphere = nib.Nifti1Image(A.astype(np.float32), affine=mask.get_affine())
        return nifti_sphere
    else:
        raise ValueError("Data type for sphere or radius(ii) or center(s) not recognized.")

def expand_mask(mask):
    """ expand a mask with multiple integers into separate binary masks

    Args:
        mask: nibabel or Brain_Data instance

    Returns:
        out: Brain_Data instance of multiple binary masks

    """

    from nltools.data import Brain_Data
    if isinstance(mask,nib.Nifti1Image):
        mask = Brain_Data(mask)
    if not isinstance(mask,Brain_Data):
        raise ValueError('Make sure mask is a nibabel or Brain_Data instance.')
    mask.data = np.round(mask.data).astype(int)
    tmp = []
    for i in np.nonzero(np.unique(mask.data))[0]:
        tmp.append((mask.data==i)*1)
    out = mask.empty()
    out.data = np.array(tmp)
    return out

def collapse_mask(mask, auto_label=True):
    """ collapse separate masks into one mask with multiple integers overlapping areas are ignored

    Args:
        mask: nibabel or Brain_Data instance

    Returns:
        out: Brain_Data instance of a mask with different integers indicating different masks

    """

    from nltools.data import Brain_Data
    if not isinstance(mask,Brain_Data):
        if isinstance(mask,nib.Nifti1Image):
            mask = Brain_Data(mask)
        else:
            raise ValueError('Make sure mask is a nibabel or Brain_Data instance.')

    if len(mask.shape()) > 1:
        if len(mask) > 1:
            out = mask.empty()

            # Create list of masks and find any overlaps
            m_list = []
            for x in range(len(mask)):
                m_list.append(mask[x].to_nifti())
            intersect = intersect_masks(m_list, threshold=1, connected=False)
            intersect = Brain_Data(nib.Nifti1Image(np.abs(intersect.get_data()-1),intersect.get_affine()))

            merge = []
            if auto_label:
                # Combine all masks into sequential order ignoring any areas of overlap
                for i in range(len(m_list)):
                    merge.append(np.multiply(Brain_Data(m_list[i]).data,intersect.data)*(i+1))
                out.data = np.sum(np.array(merge).T,1).astype(int)
            else:
                # Collapse masks using value as label
                for i in range(len(m_list)):
                    merge.append(np.multiply(Brain_Data(m_list[i]).data,intersect.data))
                out.data = np.sum(np.array(merge).T,1)
            return out
    else:
        warnings.warn("Doesn't need to be collapased")

<commit_msg>Updated create_sphere<commit_after>'''
    NeuroLearn Mask Classes
    =======================
    Classes to represent masks

'''

__all__ = ['create_sphere', 'expand_mask', 'collapse_mask']
__author__ = ["Luke Chang", "Sam Greydanus"]
__license__ = "MIT"

import os
import nibabel as nib
from nltools.utils import get_resource_path
from nilearn.input_data import NiftiMasker
from copy import deepcopy
import pandas as pd
import numpy as np
import warnings
from nilearn.masking import intersect_masks
# from neurosynth.masks import Masker

def create_sphere(coordinates, radius=5, mask=None):
    """ Generate a set of spheres in the brain mask space

    Args:
        radius: vector of radius.  Will create multiple spheres if len(radius) > 1
        centers: a vector of sphere centers of the form [px, py, pz] or [[px1, py1, pz1], ..., [pxn, pyn, pzn]]

    """
    from nltools.data import Brain_Data

    if mask is not None:
        if not isinstance(mask,nib.Nifti1Image):
            if type(mask) is str:
                if os.path.isfile(mask):
                    data = nib.load(mask)
            else:
                raise ValueError("mask is not a nibabel instance or a valid file name")
    else:
        mask = nib.load(os.path.join(get_resource_path(),'MNI152_T1_2mm_brain_mask.nii.gz'))
    dims = mask.get_data().shape

    def sphere(r, p, mask):
        """ create a sphere of given radius at some point p in the brain mask

        Args:
            r: radius of the sphere
            p: point (in coordinates of the brain mask) of the center of the sphere

        """
        dims = mask.shape
        m = [dims[0]/2, dims[1]/2, dims[2]/2] # JC edit: default value for centers
        x, y, z = np.ogrid[-m[0]:dims[0]-m[0], -m[1]:dims[1]-m[1], -m[2]:dims[2]-m[2]] #JC edit: creates sphere
        # x, y, z = np.ogrid[-p[0]:dims[0]-p[0], -p[1]:dims[1]-p[1], -p[2]:dims[2]-p[2]]
        mask_r = x*x + y*y + z*z <= r*r

        activation = np.zeros(dims)
        activation[mask_r] = 1
        # JC edit shift mask to proper location
        translation_affine= np.array([[1, 0, 0, p[0]-m[0]],
                                [0, 1, 0, p[1]-m[1]],
                                [0, 0, 1, p[2]-m[2]],
                                 [0, 0, 0, 1]])

        # activation = np.multiply(activation, mask.get_data())
        # activation = nib.Nifti1Image(activation, affine=np.eye(4))
        activation = nib.Nifti1Image(activation,affine=translation_affine)
        #return the 3D numpy matrix of zeros containing the sphere as a region of ones
        # return activation.get_data(), translation_affine
        return activation

    # Initialize Spheres with options for multiple radii and centers of the spheres (or just an int and a 3D list)
    # return sphere(radius,coordinates,mask)
    if type(radius) is int:
        radius = [radius]
    if coordinates is None:
        coordinates = [[dims[0]/2, dims[1]/2, dims[2]/2] * len(radius)] #default value for centers
    elif type(coordinates) is list and type(coordinates[0]) is int and len(radius) is 1:
        coordinates = [coordinates]
    if (type(radius)) is list and (type(coordinates) is list) and (len(radius) == len(coordinates)):
        A = np.zeros_like(mask.get_data())
        A = Brain_Data(nib.Nifti1Image(A,affine=mask.get_affine()))
        for i in xrange(len(radius)):
            A = A + Brain_Data(sphere(radius[i], coordinates[i], mask))
            # B,translation_affine = sphere(radius[i], coordinates[i], mask)
            # A = np.add(A, B)
            # A = np.add(A, sphere(radius[i], coordinates[i], mask))
        # nifti_sphere = nib.Nifti1Image(A.astype(np.float32), affine=translation_affine)
        # return nifti_sphere
        A = A.to_nifti()
        A.get_data()[A.get_data()>0.5]=1
        A.get_data()[A.get_data()<0.5]=0
        return A
    else:
        raise ValueError("Data type for sphere or radius(ii) or center(s) not recognized.")

def expand_mask(mask):
    """ expand a mask with multiple integers into separate binary masks

    Args:
        mask: nibabel or Brain_Data instance

    Returns:
        out: Brain_Data instance of multiple binary masks

    """

    from nltools.data import Brain_Data
    if isinstance(mask,nib.Nifti1Image):
        mask = Brain_Data(mask)
    if not isinstance(mask,Brain_Data):
        raise ValueError('Make sure mask is a nibabel or Brain_Data instance.')
    mask.data = np.round(mask.data).astype(int)
    tmp = []
    for i in np.nonzero(np.unique(mask.data))[0]:
        tmp.append((mask.data==i)*1)
    out = mask.empty()
    out.data = np.array(tmp)
    return out

def collapse_mask(mask, auto_label=True):
    """ collapse separate masks into one mask with multiple integers overlapping areas are ignored

    Args:
        mask: nibabel or Brain_Data instance

    Returns:
        out: Brain_Data instance of a mask with different integers indicating different masks

    """

    from nltools.data import Brain_Data
    if not isinstance(mask,Brain_Data):
        if isinstance(mask,nib.Nifti1Image):
            mask = Brain_Data(mask)
        else:
            raise ValueError('Make sure mask is a nibabel or Brain_Data instance.')

    if len(mask.shape()) > 1:
        if len(mask) > 1:
            out = mask.empty()

            # Create list of masks and find any overlaps
            m_list = []
            for x in range(len(mask)):
                m_list.append(mask[x].to_nifti())
            intersect = intersect_masks(m_list, threshold=1, connected=False)
            intersect = Brain_Data(nib.Nifti1Image(np.abs(intersect.get_data()-1),intersect.get_affine()))

            merge = []
            if auto_label:
                # Combine all masks into sequential order ignoring any areas of overlap
                for i in range(len(m_list)):
                    merge.append(np.multiply(Brain_Data(m_list[i]).data,intersect.data)*(i+1))
                out.data = np.sum(np.array(merge).T,1).astype(int)
            else:
                # Collapse masks using value as label
                for i in range(len(m_list)):
                    merge.append(np.multiply(Brain_Data(m_list[i]).data,intersect.data))
                out.data = np.sum(np.array(merge).T,1)
            return out
    else:
        warnings.warn("Doesn't need to be collapased")

<|endoftext|>


<commit_before>from pymc3.model import modelcontext
from pymc3.distributions.distribution import Distribution
from functools import partial

class DistSpec(object):
    def __init__(self, distcls, *args, **kwargs):
        if not issubclass(distcls, Distribution):
            raise ValueError('We can deal with pymc3 distributions only')
        self.args = args
        self.kwargs = kwargs
        self.distcls = distcls

    def __call__(self, shape, name=None):
        model = modelcontext(None)
        if name is None:
            name = 'w{}'.format(len(model.vars))
        called_args = self.called_args(self.args, name, shape)
        called_kwargs = self.called_kwargs(self.kwargs, name, shape)
        called_kwargs.update(shape=shape)
        val = model.Var(
                name, self.distcls.dist(
                    *called_args,
                    **called_kwargs
                ),
            )
        val.tag.test_value = val.random()
        return val

    def with_name(self, name):
        return partial(self, name=name)

    def called_args(self, args, name, shape):
        return [
            self._call(arg, '{}_arg{}'.format(name, i), shape)
            for i, arg in enumerate(args)
        ]

    def called_kwargs(self, kwargs, name, shape):
        return {
            key: self._call(arg, '{}_{}'.format(name, key), shape)
            for key, arg in kwargs.items()
        }

    @staticmethod
    def _call(arg, label, shape):
        if callable(arg):
            if isinstance(arg, DistSpec):
                return arg(shape, label)
            else:
                raise TypeError(
                    'Cannot proceed type {} in DistSpec'
                    .format(type(arg))
                )
        else:
            return arg
<commit_msg>tini changes in spec<commit_after>from pymc3.model import modelcontext
from pymc3.distributions.distribution import Distribution
from functools import partial

class DistSpec(object):
    """Spec based on pymc3 distributions

    Parameters
    ----------
    distcls : pymc3.Distribution
    args : args for `distcls`
    kwargs : kwargs for `distcls`

    Returns
    -------
    callable spec

    Usage
    -----
    spec = DistSpec(Normal, mu=0, sd=DistSpec(Lognormal, 0, 1))
    """

    def __init__(self, distcls, *args, **kwargs):
        if not issubclass(distcls, Distribution):
            raise ValueError('We can deal with pymc3 distributions only')
        self.args = args
        self.kwargs = kwargs
        self.distcls = distcls

    def __call__(self, shape, name=None):
        model = modelcontext(None)
        if name is None:
            name = 'w{}'.format(len(model.vars))
        called_args = self._call_args(self.args, name, shape)
        called_kwargs = self._call_kwargs(self.kwargs, name, shape)
        called_kwargs.update(shape=shape)
        val = model.Var(
                name, self.distcls.dist(
                    *called_args,
                    **called_kwargs
                ),
            )
        val.tag.test_value = val.random()
        return val

    def with_name(self, name):
        return partial(self, name=name)

    def _call_args(self, args, name, shape):
        return [
            self._call(arg, '{}_arg{}'.format(name, i), shape)
            for i, arg in enumerate(args)
        ]

    def _call_kwargs(self, kwargs, name, shape):
        return {
            key: self._call(arg, '{}_{}'.format(name, key), shape)
            for key, arg in kwargs.items()
        }

    @staticmethod
    def _call(arg, label, shape):
        if callable(arg):
            if isinstance(arg, DistSpec):
                return arg(shape, label)
            else:
                raise TypeError(
                    'Cannot proceed type {} in DistSpec'
                    .format(type(arg))
                )
        else:
            return arg
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse, Http404
from django.views.decorators.csrf import csrf_protect
from approver.forms import AboutYouForm, ProjectForm
from django.contrib.auth.models import User
from approver.models import Project

def index(request):
    return render(request, 'approver/index.html')
<commit_msg>check if user is logged in before showing landing page<commit_after>from django.shortcuts import render
from django.http import HttpResponse, Http404
from django.views.decorators.csrf import csrf_protect
from approver.forms import AboutYouForm, ProjectForm
from django.contrib.auth.models import User
from approver.models import Project

import approver.constants as constants
import approver.utils as utils
from approver.views.dashboard import dashboard

def index(request):
    if(utils.get_current_user_gatorlink(request.session)):
        return dashboard(request)
    else:
        return render(request, 'approver/index.html')
<|endoftext|>