<commit_before>from crawler.Crawler import Crawler
from indexer.indexer import Indexer
from analyzer.cosinus import CosinusAnalyzer
from utils.string import StringUtil

__author__ = 'pascal'

#  _____                           _
# | ____|_  ____ _ _ __ ___  _ __ | | ___
# |  _| \ \/ / _` | '_ ` _ \| '_ \| |/ _ \
# | |___ >  < (_| | | | | | | |_) | |  __/
# |_____/_/\_\__,_|_| |_| |_| .__/|_|\___|
#                           |_|

def crawler_example():
    # Create a seed
    seed = [
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d01.html',
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d06.html',
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d08.html'
    ]

    # Instatiate the crawler.
    crawler = Crawler()

    # Start the crawler with the seed.
    crawler.start_crawling(seed)

    # Access the pages.
    pages = crawler.pages

    # Create an Indexer
    indexer = Indexer()

    # Index the pages
    indexer.index_pages(pages)

    # Print the link structure
    link_structure_txt = pages.get_link_structure_text()
    print(link_structure_txt)

    # Print your index
    print(indexer.index)

    # Print the content of the pages
    print(pages)

    example_queries = ['tokens', 'index', 'tokens classification' ]
    analyzer = CosinusAnalyzer(indexer.index, crawler.pages)
    print(analyzer.get_length_of_pages_text())

    print(StringUtil.header('cosine_scores.txt'))
    for query in example_queries:
        hits = analyzer.analyze(query)
        print(hits)
    print()

    print(StringUtil.header('Cosinus combined with Page Rank'))
    analyzer = CosinusAnalyzer(indexer.index, crawler.pages, combine_with_page_rank=True)
    for query in example_queries:
        hits = analyzer.analyze(query)
        print(hits)
    print()

    row = 0
    matrix = []
    for page in pages:
        outlinks = []
        for c in range(1, pages.count() + 1):
            title = "d0" + str(c)
            outlinks.append(page.out_pages.has_page_with_title(title))
        matrix.append(outlinks)
        row += 1
    print(matrix)

<commit_msg>Remove matrix calculation from crawler example.py<commit_after>from crawler.Crawler import Crawler
from indexer.indexer import Indexer
from analyzer.cosinus import CosinusAnalyzer
from utils.string import StringUtil

__author__ = 'pascal'

#  _____                           _
# | ____|_  ____ _ _ __ ___  _ __ | | ___
# |  _| \ \/ / _` | '_ ` _ \| '_ \| |/ _ \
# | |___ >  < (_| | | | | | | |_) | |  __/
# |_____/_/\_\__,_|_| |_| |_| .__/|_|\___|
#                           |_|

def crawler_example():
    # Create a seed
    seed = [
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d01.html',
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d06.html',
        'http://people.f4.htw-berlin.de/fileadmin/user_upload/Dozenten/WI-Dozenten/Classen/DAWeb/smdocs/d08.html'
    ]

    # Instatiate the crawler.
    crawler = Crawler()

    # Start the crawler with the seed.
    crawler.start_crawling(seed)

    # Access the pages.
    pages = crawler.pages

    # Create an Indexer
    indexer = Indexer()

    # Index the pages
    indexer.index_pages(pages)

    # Print the link structure
    link_structure_txt = pages.get_link_structure_text()
    print(link_structure_txt)

    # Print your index
    print(indexer.index)

    # Print the content of the pages
    print(pages)

    # Replace me with your Page Rank Calculation :)

    example_queries = ['tokens', 'index', 'tokens classification' ]
    analyzer = CosinusAnalyzer(indexer.index, crawler.pages)
    print(analyzer.get_length_of_pages_text())

    print(StringUtil.header('cosine_scores.txt'))
    for query in example_queries:
        hits = analyzer.analyze(query)
        print(hits)
    print()

    print(StringUtil.header('Cosinus combined with Page Rank'))
    analyzer = CosinusAnalyzer(indexer.index, crawler.pages, combine_with_page_rank=True)
    for query in example_queries:
        hits = analyzer.analyze(query)
        print(hits)
    print()

<|endoftext|>


<commit_before>import scipy as _sp
from openpnm.core import logging as _logging
_logger = _logging.getLogger(__name__)

def straight(target, pore_diameter='pore.diameter', L_negative=1e-9):
    r"""
    Calculate throat length

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    L_negative : float
        The default throat length to use when negative lengths are found.  The
        default is 1 nm.  To accept negative throat lengths, set this value to
        ``None``.
    """
    network = target.project.network
    # Initialize throat_property['length']
    throats = network.throats(target.name)
    pore1 = network['throat.conns'][:, 0]
    pore2 = network['throat.conns'][:, 1]
    C1 = network['pore.coords'][pore1]
    C2 = network['pore.coords'][pore2]
    E = _sp.sqrt(_sp.sum((C1-C2)**2, axis=1))  # Euclidean distance
    D1 = network[pore_diameter][pore1]
    D2 = network[pore_diameter][pore2]
    value = E-(D1+D2)/2.
    value = value[throats]
    if _sp.any(value < 0) and L_negative is not None:
        _logger.warn('Negative throat lengths are calculated. Arbitrary ' +
                     'positive length assigned: ' + str(L_negative))
        Ts = _sp.where(value < 0)[0]
        value[Ts] = L_negative
    return value
<commit_msg>Add ctc model which calculates center-to-center distance between pores<commit_after>import scipy as _sp
from openpnm.core import logging as _logging
_logger = _logging.getLogger(__name__)

def ctc(target, pore_diameter='pore.diameter'):
    r"""
    Calculate throat length assuming point-like pores, i.e. center-to-center
    distance between pores.

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    pore_diameter : string
        Dictionary key of the pore diameter values

    """
    network = target.project.network
    throats = network.throats(target.name)
    cn = network['throat.conns']
    C1 = network['pore.coords'][cn[:, 0]]
    C2 = network['pore.coords'][cn[:, 1]]
    E = _sp.sqrt(_sp.sum((C1-C2)**2, axis=1))
    return E[throats]

def straight(target, pore_diameter='pore.diameter', L_negative=1e-9):
    r"""
    Calculate throat length

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    L_negative : float
        The default throat length to use when negative lengths are found.  The
        default is 1 nm.  To accept negative throat lengths, set this value to
        ``None``.
    """
    network = target.project.network
    # Initialize throat_property['length']
    throats = network.throats(target.name)
    pore1 = network['throat.conns'][:, 0]
    pore2 = network['throat.conns'][:, 1]
    C1 = network['pore.coords'][pore1]
    C2 = network['pore.coords'][pore2]
    E = _sp.sqrt(_sp.sum((C1-C2)**2, axis=1))  # Euclidean distance
    D1 = network[pore_diameter][pore1]
    D2 = network[pore_diameter][pore2]
    value = E-(D1+D2)/2.
    value = value[throats]
    if _sp.any(value < 0) and L_negative is not None:
        _logger.warn('Negative throat lengths are calculated. Arbitrary ' +
                     'positive length assigned: ' + str(L_negative))
        Ts = _sp.where(value < 0)[0]
        value[Ts] = L_negative
    return value
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from time import time, mktime
from datetime import datetime
from retrying import retry
from iso8601 import parse_date
from socket import setdefaulttimeout
from retrying import retry

from openprocurement_client.client import TendersClient
from openprocurement.search.source import BaseSource

from logging import getLogger
logger = getLogger(__name__)

class AuctionSource(BaseSource):
    """Auction Source
    """
    __doc_type__ = 'auction'

    config = {
        'auction_api_key': '',
        'auction_api_url': "",
        'auction_api_version': '0',
        'auction_resource': 'auctions',
        'auction_api_mode': '',
        'auction_skip_until': None,
        'auction_limit': 1000,
        'auction_preload': 500000,
        'auction_resethour': 23,
        'timeout': 30,
    }

    def __init__(self, config={}):
        if config:
            self.config.update(config)
        self.config['auction_limit'] = int(self.config['auction_limit'] or 0) or 100
        self.config['auction_preload'] = int(self.config['auction_preload'] or 0) or 100
        self.config['auction_resethour'] = int(self.config['auction_resethour'] or 0)
        self.client = None

    def procuring_entity(self, item):
        try:
            return item.data.get('procuringEntity', None)
        except (KeyError, AttributeError):
            return None

    def patch_version(self, item):
        """Convert dateModified to long version
        """
        item['doc_type'] = self.__doc_type__
        dt = parse_date(item['dateModified'])
        version = 1e6 * mktime(dt.timetuple()) + dt.microsecond
        item['version'] = long(version)
        return item

    def need_reset(self):
        if self.should_reset:
            return True
        if time() - self.last_reset_time > 3600:
            return datetime.now().hour == int(self.config['auction_resethour'])

    @retry(stop_max_attempt_number=5, wait_fixed=15000)
    def reset(self):
        logger.info("Reset auctions, auction_skip_until=%s",
                    self.config['auction_skip_until'])
        if self.config.get('timeout', None):
            setdefaulttimeout(float(self.config['timeout']))
        params = {}
        if self.config['auction_api_mode']:
            params['mode'] = self.config['auction_api_mode']
        if self.config['auction_limit']:
            params['limit'] = self.config['auction_limit']
        self.client = TendersClient(
            key=self.config['auction_api_key'],
            host_url=self.config['auction_api_url'],
            api_version=self.config['auction_api_version'],
            resource=self.config['auction_resource'],
            params=params)
        self.skip_until = self.config.get('auction_skip_until', None)
        if self.skip_until and self.skip_until[:2] != '20':
            self.skip_until = None
        self.last_reset_time = time()
        self.should_reset = False

    def preload(self):
        preload_items = []
        while True:
            try:
                items = self.client.get_tenders()
            except Exception as e:
                logger.error("AuctionSource.preload error %s", str(e))
                self.reset()
                break
            if self.should_exit:
                return []
            if not items:
                break

            preload_items.extend(items)

            if len(preload_items) >= 100:
                logger.info("Preload %d auctions, last %s",
                    len(preload_items), items[-1]['dateModified'])
            if len(items) < 10:
                break
            if len(preload_items) >= self.config['auction_preload']:
                break

        return preload_items

    def items(self):
        if not self.client:
            self.reset()
        self.last_skipped = None
        for auction in self.preload():
            if self.should_exit:
                raise StopIteration()
            if self.skip_until > auction['dateModified']:
                self.last_skipped = auction['dateModified']
                continue
            yield self.patch_version(auction)

    def get(self, item):
        auction = {}
        retry_count = 0
        while not self.should_exit:
            try:
                auction = self.client.get_tender(item['id'])
                assert auction['data']['id'] == item['id']
                assert auction['data']['dateModified'] >= item['dateModified']
                break
            except Exception as e:
                if retry_count > 3:
                    raise e
                retry_count += 1
                logger.error("get_auction %s retry %d error %s",
                    str(item['id']), retry_count, str(e))
                self.sleep(5)
                if retry_count > 1:
                    self.reset()
        if item['dateModified'] != auction['data']['dateModified']:
            logger.debug("Auction dateModified mismatch %s %s %s",
                item['id'], item['dateModified'],
                auction['data']['dateModified'])
            item['dateModified'] = auction['data']['dateModified']
            item = self.patch_version(item)
        auction['meta'] = item
        return auction
<commit_msg>auction_preload=10000<commit_after># -*- coding: utf-8 -*-
from time import time, mktime
from datetime import datetime
from retrying import retry
from iso8601 import parse_date
from socket import setdefaulttimeout
from retrying import retry

from openprocurement_client.client import TendersClient
from openprocurement.search.source import BaseSource

from logging import getLogger
logger = getLogger(__name__)

class AuctionSource(BaseSource):
    """Auction Source
    """
    __doc_type__ = 'auction'

    config = {
        'auction_api_key': '',
        'auction_api_url': "",
        'auction_api_version': '0',
        'auction_resource': 'auctions',
        'auction_api_mode': '',
        'auction_skip_until': None,
        'auction_limit': 1000,
        'auction_preload': 10000,
        'auction_resethour': 23,
        'timeout': 30,
    }

    def __init__(self, config={}):
        if config:
            self.config.update(config)
        self.config['auction_limit'] = int(self.config['auction_limit'] or 0) or 100
        self.config['auction_preload'] = int(self.config['auction_preload'] or 0) or 100
        self.config['auction_resethour'] = int(self.config['auction_resethour'] or 0)
        self.client = None

    def procuring_entity(self, item):
        try:
            return item.data.get('procuringEntity', None)
        except (KeyError, AttributeError):
            return None

    def patch_version(self, item):
        """Convert dateModified to long version
        """
        item['doc_type'] = self.__doc_type__
        dt = parse_date(item['dateModified'])
        version = 1e6 * mktime(dt.timetuple()) + dt.microsecond
        item['version'] = long(version)
        return item

    def need_reset(self):
        if self.should_reset:
            return True
        if time() - self.last_reset_time > 3600:
            return datetime.now().hour == int(self.config['auction_resethour'])

    @retry(stop_max_attempt_number=5, wait_fixed=15000)
    def reset(self):
        logger.info("Reset auctions, auction_skip_until=%s",
                    self.config['auction_skip_until'])
        if self.config.get('timeout', None):
            setdefaulttimeout(float(self.config['timeout']))
        params = {}
        if self.config['auction_api_mode']:
            params['mode'] = self.config['auction_api_mode']
        if self.config['auction_limit']:
            params['limit'] = self.config['auction_limit']
        self.client = TendersClient(
            key=self.config['auction_api_key'],
            host_url=self.config['auction_api_url'],
            api_version=self.config['auction_api_version'],
            resource=self.config['auction_resource'],
            params=params)
        self.skip_until = self.config.get('auction_skip_until', None)
        if self.skip_until and self.skip_until[:2] != '20':
            self.skip_until = None
        self.last_reset_time = time()
        self.should_reset = False

    def preload(self):
        preload_items = []
        while True:
            try:
                items = self.client.get_tenders()
            except Exception as e:
                logger.error("AuctionSource.preload error %s", str(e))
                self.reset()
                break
            if self.should_exit:
                return []
            if not items:
                break

            preload_items.extend(items)

            if len(preload_items) >= 100:
                logger.info("Preload %d auctions, last %s",
                    len(preload_items), items[-1]['dateModified'])
            if len(items) < 10:
                break
            if len(preload_items) >= self.config['auction_preload']:
                break

        return preload_items

    def items(self):
        if not self.client:
            self.reset()
        self.last_skipped = None
        for auction in self.preload():
            if self.should_exit:
                raise StopIteration()
            if self.skip_until > auction['dateModified']:
                self.last_skipped = auction['dateModified']
                continue
            yield self.patch_version(auction)

    def get(self, item):
        auction = {}
        retry_count = 0
        while not self.should_exit:
            try:
                auction = self.client.get_tender(item['id'])
                assert auction['data']['id'] == item['id']
                assert auction['data']['dateModified'] >= item['dateModified']
                break
            except Exception as e:
                if retry_count > 3:
                    raise e
                retry_count += 1
                logger.error("get_auction %s retry %d error %s",
                    str(item['id']), retry_count, str(e))
                self.sleep(5)
                if retry_count > 1:
                    self.reset()
        if item['dateModified'] != auction['data']['dateModified']:
            logger.debug("Auction dateModified mismatch %s %s %s",
                item['id'], item['dateModified'],
                auction['data']['dateModified'])
            item['dateModified'] = auction['data']['dateModified']
            item = self.patch_version(item)
        auction['meta'] = item
        return auction
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import mock
import unittest
import numpy
from openquake.baselib import parallel

try:
    import celery
except ImportError:
    celery = None

def get_length(data):
    return {'n': len(data)}

def get_len(data, monitor):
    with monitor:
        result = {'n': len(data)}
    monitor.flush()
    return result

class StarmapTestCase(unittest.TestCase):
    monitor = parallel.Monitor()

    def test_apply(self):
        res = parallel.Starmap.apply(
            get_length, (numpy.arange(10),), concurrent_tasks=3).reduce()
        self.assertEqual(res, {'n': 10})  # chunks [4, 4, 2]

    # this case is non-trivial since there is a key, so two groups are
    # generated even if everything is run in a single core
    def test_apply_no_tasks(self):
        res = parallel.Starmap.apply(
            get_length, ('aaabb',), concurrent_tasks=0,
            key=lambda char: char)
        # chunks [['a', 'a', 'a'], ['b', 'b']]
        partial_sums = sorted(dic['n'] for dic in res)
        self.assertEqual(partial_sums, [2, 3])

    def test_apply_maxweight(self):
        res = parallel.Starmap.apply(
            get_length, ('aaabb',), maxweight=2,
            key=lambda char: char)
        # chunks ['aa', 'ab', 'b']
        partial_sums = sorted(dic['n'] for dic in res)
        self.assertEqual(partial_sums, [1, 2, 2])

    def test_spawn(self):
        all_data = [
            ('a', list(range(10))), ('b', list(range(20))),
            ('c', list(range(15)))]
        res = {key: parallel.Starmap(get_length, [(data,)])
               for key, data in all_data}
        for key, val in res.items():
            res[key] = val.reduce()
        parallel.Starmap.restart()
        self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})

    def test_no_flush(self):
        mon = parallel.Monitor('test')
        res = parallel.safely_call(get_len, ('ab', mon))
        self.assertIn(
            'Monitor(\'test\').flush() must not be called in a worker', res[0])
        self.assertEqual(res[1], RuntimeError)
        self.assertEqual(res[2].operation, mon.operation)

    if celery:
        def test_received(self):
            with mock.patch.dict('os.environ', OQ_DISTRIBUTE='celery'):
                res = parallel.Starmap.apply(
                    get_length, (numpy.arange(10),)).submit_all()
                list(res)  # iterate on the results
                self.assertGreater(len(res.received), 0)
<commit_msg>Removed test [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import mock
import unittest
import numpy
from openquake.baselib import parallel

try:
    import celery
except ImportError:
    celery = None

def get_length(data):
    return {'n': len(data)}

def get_len(data, monitor):
    with monitor:
        result = {'n': len(data)}
    monitor.flush()
    return result

class StarmapTestCase(unittest.TestCase):
    monitor = parallel.Monitor()

    def test_apply(self):
        res = parallel.Starmap.apply(
            get_length, (numpy.arange(10),), concurrent_tasks=3).reduce()
        self.assertEqual(res, {'n': 10})  # chunks [4, 4, 2]

    # this case is non-trivial since there is a key, so two groups are
    # generated even if everything is run in a single core
    def test_apply_no_tasks(self):
        res = parallel.Starmap.apply(
            get_length, ('aaabb',), concurrent_tasks=0,
            key=lambda char: char)
        # chunks [['a', 'a', 'a'], ['b', 'b']]
        partial_sums = sorted(dic['n'] for dic in res)
        self.assertEqual(partial_sums, [2, 3])

    def test_apply_maxweight(self):
        res = parallel.Starmap.apply(
            get_length, ('aaabb',), maxweight=2,
            key=lambda char: char)
        # chunks ['aa', 'ab', 'b']
        partial_sums = sorted(dic['n'] for dic in res)
        self.assertEqual(partial_sums, [1, 2, 2])

    def test_spawn(self):
        all_data = [
            ('a', list(range(10))), ('b', list(range(20))),
            ('c', list(range(15)))]
        res = {key: parallel.Starmap(get_length, [(data,)])
               for key, data in all_data}
        for key, val in res.items():
            res[key] = val.reduce()
        parallel.Starmap.restart()
        self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})

    def test_no_flush(self):
        mon = parallel.Monitor('test')
        res = parallel.safely_call(get_len, ('ab', mon))
        self.assertIn(
            'Monitor(\'test\').flush() must not be called in a worker', res[0])
        self.assertEqual(res[1], RuntimeError)
        self.assertEqual(res[2].operation, mon.operation)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import itertools
import numpy

from openquake.commonlib import riskmodels, calc
from openquake.risklib import scientific
from openquake.calculators import base

F32 = numpy.float32
F64 = numpy.float64

def dist_by_asset(data, multi_stat_dt):
    """
    :param data: array of shape (N, R, L, 2, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (N, R) with records of type multi_stat_dt
    """
    N, R, L = data.shape[:3]
    out = numpy.zeros((N, R), multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for n, r in itertools.product(range(N), range(R)):
            out_lt[n, r] = data[n, r, l]
    return out

def dist_by_taxon(data, multi_stat_dt):
    """
    :param data: array of shape (T, R, L, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (T, R) with records of type multi_stat_dt
    """
    T, R, L = data.shape[:3]
    out = numpy.zeros((T, R), multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for t, r in itertools.product(range(T), range(R)):
            out_lt[t, r] = scientific.mean_std(data[t, r, l])
    return out

def dist_total(data, multi_stat_dt):
    """
    :param data: array of shape (T, R, L, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (R,) with records of type multi_stat_dt
    """
    T, R, L = data.shape[:3]
    total = data.sum(axis=0)
    out = numpy.zeros(R, multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for r in range(R):
            out_lt[r] = scientific.mean_std(total[r, l])
    return out

def scenario_damage(riskinput, riskmodel, param, monitor):
    """
    Core function for a damage computation.

    :param riskinput:
        a :class:`openquake.risklib.riskinput.RiskInput` object
    :param riskmodel:
        a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :param param:
        dictionary of extra parameters
    :returns:
        a dictionary {'d_asset': [(l, r, a, mean-stddev), ...],
                      'd_taxonomy': damage array of shape T, R, L, E, D,
                      'c_asset': [(l, r, a, mean-stddev), ...],
                      'c_taxonomy': damage array of shape T, R, L, E}

    `d_asset` and `d_taxonomy` are related to the damage distributions
    whereas `c_asset` and `c_taxonomy` are the consequence distributions.
    If there is no consequence model `c_asset` is an empty list and
    `c_taxonomy` is a zero-value array.
    """
    c_models = param['consequence_models']
    L = len(riskmodel.loss_types)
    R = len(riskinput.rlzs)
    D = len(riskmodel.damage_states)
    E = param['number_of_ground_motion_fields']
    T = len(param['taxonomies'])
    taxo2idx = {taxo: i for i, taxo in enumerate(param['taxonomies'])}
    result = dict(d_asset=[], d_taxon=numpy.zeros((T, R, L, E, D), F64),
                  c_asset=[], c_taxon=numpy.zeros((T, R, L, E), F64))
    for outputs in riskmodel.gen_outputs(riskinput, monitor):
        r = outputs.r
        for l, damages in enumerate(outputs):
            loss_type = riskmodel.loss_types[l]
            c_model = c_models.get(loss_type)
            for asset, fraction in zip(outputs.assets, damages):
                t = taxo2idx[asset.taxonomy]
                damages = fraction * asset.number
                if c_model:  # compute consequences
                    means = [par[0] for par in c_model[asset.taxonomy].params]
                    # NB: we add a 0 in front for nodamage state
                    c_ratio = numpy.dot(fraction, [0] + means)
                    consequences = c_ratio * asset.value(loss_type)
                    result['c_asset'].append(
                        (l, r, asset.ordinal,
                         scientific.mean_std(consequences)))
                    result['c_taxon'][t, r, l, :] += consequences
                    # TODO: consequences for the occupants
                result['d_asset'].append(
                    (l, r, asset.ordinal, scientific.mean_std(damages)))
                result['d_taxon'][t, r, l, :] += damages
    return result

@base.calculators.add('scenario_damage')
class ScenarioDamageCalculator(base.RiskCalculator):
    """
    Scenario damage calculator
    """
    pre_calculator = 'scenario'
    core_task = scenario_damage
    is_stochastic = True

    def pre_execute(self):
        if 'gmfs' in self.oqparam.inputs:
            self.pre_calculator = None
        base.RiskCalculator.pre_execute(self)
        self.param['consequence_models'] = riskmodels.get_risk_models(
            self.oqparam, 'consequence')
        self.param['number_of_ground_motion_fields'] = (
            self.oqparam.number_of_ground_motion_fields)
        self.datastore['etags'], gmfs = calc.get_gmfs(
            self.datastore, self.precalc)
        rlzs = self.csm_info.get_rlzs_assoc().realizations
        self.riskinputs = self.build_riskinputs(
            'gmf', {rlz: gmf for rlz, gmf in zip(rlzs, gmfs)})
        self.param['taxonomies'] = sorted(self.taxonomies)

    def post_execute(self, result):
        """
        Compute stats for the aggregated distributions and save
        the results on the datastore.
        """
        dstates = self.riskmodel.damage_states
        ltypes = self.riskmodel.loss_types
        L = len(ltypes)
        R = len(self.rlzs_assoc.realizations)
        D = len(dstates)
        N = len(self.assetcol)

        # damage distributions
        dt_list = []
        for ltype in ltypes:
            dt_list.append((ltype, numpy.dtype([('mean', (F32, D)),
                                                ('stddev', (F32, D))])))
        multi_stat_dt = numpy.dtype(dt_list)
        d_asset = numpy.zeros((N, R, L, 2, D), F32)
        for (l, r, a, stat) in result['d_asset']:
            d_asset[a, r, l] = stat
        self.datastore['dmg_by_asset'] = dist_by_asset(
            d_asset, multi_stat_dt)
        self.datastore['dmg_by_taxon'] = dist_by_taxon(
            result['d_taxon'], multi_stat_dt)
        self.datastore['dmg_total'] = dist_total(
            result['d_taxon'], multi_stat_dt)

        # consequence distributions
        if result['c_asset']:
            stat_dt = numpy.dtype([('mean', F32), ('stddev', F32)])
            c_asset = numpy.zeros((N, R, L), stat_dt)
            for (l, r, a, stat) in result['c_asset']:
                c_asset[a, r, l] = stat
            multi_stat_dt = self.oqparam.loss_dt(stat_dt)
            self.datastore['losses_by_asset'] = c_asset
            self.datastore['losses_by_taxon'] = dist_by_taxon(
                result['c_taxon'], multi_stat_dt)
            self.datastore['losses_total'] = dist_total(
                result['c_taxon'], multi_stat_dt)
<commit_msg>Cleanup [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import itertools
import numpy

from openquake.commonlib import riskmodels, calc
from openquake.risklib import scientific
from openquake.calculators import base

F32 = numpy.float32
F64 = numpy.float64

def dist_by_asset(data, multi_stat_dt):
    """
    :param data: array of shape (N, R, L, 2, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (N, R) with records of type multi_stat_dt
    """
    N, R, L = data.shape[:3]
    out = numpy.zeros((N, R), multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for n, r in itertools.product(range(N), range(R)):
            out_lt[n, r] = data[n, r, l]
    return out

def dist_by_taxon(data, multi_stat_dt):
    """
    :param data: array of shape (T, R, L, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (T, R) with records of type multi_stat_dt
    """
    T, R, L = data.shape[:3]
    out = numpy.zeros((T, R), multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for t, r in itertools.product(range(T), range(R)):
            out_lt[t, r] = scientific.mean_std(data[t, r, l])
    return out

def dist_total(data, multi_stat_dt):
    """
    :param data: array of shape (T, R, L, ...)
    :param multi_stat_dt: numpy dtype for statistical outputs
    :returns: array of shape (R,) with records of type multi_stat_dt
    """
    T, R, L = data.shape[:3]
    total = data.sum(axis=0)
    out = numpy.zeros(R, multi_stat_dt)
    for l, lt in enumerate(multi_stat_dt.names):
        out_lt = out[lt]
        for r in range(R):
            out_lt[r] = scientific.mean_std(total[r, l])
    return out

def scenario_damage(riskinput, riskmodel, param, monitor):
    """
    Core function for a damage computation.

    :param riskinput:
        a :class:`openquake.risklib.riskinput.RiskInput` object
    :param riskmodel:
        a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :param param:
        dictionary of extra parameters
    :returns:
        a dictionary {'d_asset': [(l, r, a, mean-stddev), ...],
                      'd_taxonomy': damage array of shape T, R, L, E, D,
                      'c_asset': [(l, r, a, mean-stddev), ...],
                      'c_taxonomy': damage array of shape T, R, L, E}

    `d_asset` and `d_taxonomy` are related to the damage distributions
    whereas `c_asset` and `c_taxonomy` are the consequence distributions.
    If there is no consequence model `c_asset` is an empty list and
    `c_taxonomy` is a zero-value array.
    """
    c_models = param['consequence_models']
    L = len(riskmodel.loss_types)
    R = len(riskinput.rlzs)
    D = len(riskmodel.damage_states)
    E = param['number_of_ground_motion_fields']
    T = len(param['taxonomies'])
    taxo2idx = {taxo: i for i, taxo in enumerate(param['taxonomies'])}
    result = dict(d_asset=[], d_taxon=numpy.zeros((T, R, L, E, D), F64),
                  c_asset=[], c_taxon=numpy.zeros((T, R, L, E), F64))
    for outputs in riskmodel.gen_outputs(riskinput, monitor):
        r = outputs.r
        for l, damages in enumerate(outputs):
            loss_type = riskmodel.loss_types[l]
            c_model = c_models.get(loss_type)
            for asset, fraction in zip(outputs.assets, damages):
                t = taxo2idx[asset.taxonomy]
                damages = fraction * asset.number
                if c_model:  # compute consequences
                    means = [par[0] for par in c_model[asset.taxonomy].params]
                    # NB: we add a 0 in front for nodamage state
                    c_ratio = numpy.dot(fraction, [0] + means)
                    consequences = c_ratio * asset.value(loss_type)
                    result['c_asset'].append(
                        (l, r, asset.ordinal,
                         scientific.mean_std(consequences)))
                    result['c_taxon'][t, r, l, :] += consequences
                    # TODO: consequences for the occupants
                result['d_asset'].append(
                    (l, r, asset.ordinal, scientific.mean_std(damages)))
                result['d_taxon'][t, r, l, :] += damages
    return result

@base.calculators.add('scenario_damage')
class ScenarioDamageCalculator(base.RiskCalculator):
    """
    Scenario damage calculator
    """
    pre_calculator = 'scenario'
    core_task = scenario_damage
    is_stochastic = True

    def pre_execute(self):
        if 'gmfs' in self.oqparam.inputs:
            self.pre_calculator = None
        base.RiskCalculator.pre_execute(self)
        self.param['number_of_ground_motion_fields'] = (
            self.oqparam.number_of_ground_motion_fields)
        self.param['consequence_models'] = riskmodels.get_risk_models(
            self.oqparam, 'consequence')
        self.datastore['etags'], gmfs = calc.get_gmfs(
            self.datastore, self.precalc)
        rlzs = self.csm_info.get_rlzs_assoc().realizations
        self.riskinputs = self.build_riskinputs(
            'gmf', {rlz: gmf for rlz, gmf in zip(rlzs, gmfs)})
        self.param['taxonomies'] = sorted(self.taxonomies)

    def post_execute(self, result):
        """
        Compute stats for the aggregated distributions and save
        the results on the datastore.
        """
        dstates = self.riskmodel.damage_states
        ltypes = self.riskmodel.loss_types
        L = len(ltypes)
        R = len(self.rlzs_assoc.realizations)
        D = len(dstates)
        N = len(self.assetcol)

        # damage distributions
        dt_list = []
        for ltype in ltypes:
            dt_list.append((ltype, numpy.dtype([('mean', (F32, D)),
                                                ('stddev', (F32, D))])))
        multi_stat_dt = numpy.dtype(dt_list)
        d_asset = numpy.zeros((N, R, L, 2, D), F32)
        for (l, r, a, stat) in result['d_asset']:
            d_asset[a, r, l] = stat
        self.datastore['dmg_by_asset'] = dist_by_asset(
            d_asset, multi_stat_dt)
        self.datastore['dmg_by_taxon'] = dist_by_taxon(
            result['d_taxon'], multi_stat_dt)
        self.datastore['dmg_total'] = dist_total(
            result['d_taxon'], multi_stat_dt)

        # consequence distributions
        if result['c_asset']:
            stat_dt = numpy.dtype([('mean', F32), ('stddev', F32)])
            c_asset = numpy.zeros((N, R, L), stat_dt)
            for (l, r, a, stat) in result['c_asset']:
                c_asset[a, r, l] = stat
            multi_stat_dt = self.oqparam.loss_dt(stat_dt)
            self.datastore['losses_by_asset'] = c_asset
            self.datastore['losses_by_taxon'] = dist_by_taxon(
                result['c_taxon'], multi_stat_dt)
            self.datastore['losses_total'] = dist_total(
                result['c_taxon'], multi_stat_dt)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import numpy
from openquake.risklib import scientific
from openquake.calculators import base

U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64

def scenario_damage(riskinputs, crmodel, param, monitor):
    """
    Core function for a damage computation.

    :param riskinputs:
        :class:`openquake.risklib.riskinput.RiskInput` objects
    :param crmodel:
        a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :param param:
        dictionary of extra parameters
    :returns:
        a dictionary {'d_asset': [(l, r, a, mean-stddev), ...],
                      'd_event': damage array of shape R, L, E, D,
                      + optional consequences}

    `d_asset` and `d_tag` are related to the damage distributions.
    """
    L = len(crmodel.loss_types)
    D = len(crmodel.damage_states)
    R = riskinputs[0].hazard_getter.num_rlzs
    result = dict(d_asset=[])
    consequences = crmodel.get_consequences()
    for name in consequences:
        result[name + '_by_asset'] = []
    for ri in riskinputs:
        for out in ri.gen_outputs(crmodel, monitor):
            r = out.rlzi
            E = len(out.eids)
            if 'd_event' not in result:
                result['d_event'] = numpy.zeros((E, R, L, D), F64)
            for name in consequences:
                if name + '_by_event' not in result:
                    result[name + '_by_event'] = numpy.zeros((E, R, L), F64)
            for l, loss_type in enumerate(crmodel.loss_types):
                for asset, fractions in zip(ri.assets, out[loss_type]):
                    dmg = fractions * asset['number']  # shape (E, D)
                    result['d_event'][:, r, l] += dmg
                    result['d_asset'].append(
                        (l, r, asset['ordinal'], scientific.mean_std(dmg)))
                    csq = crmodel.compute_csq(asset, fractions, loss_type)
                    for name, value in csq.items():
                        result[name + '_by_asset'].append(
                            (l, r, asset['ordinal'],
                             scientific.mean_std(value)))
                        result[name + '_by_event'][:, r, l] += value
    return result

@base.calculators.add('scenario_damage')
class ScenarioDamageCalculator(base.RiskCalculator):
    """
    Scenario damage calculator
    """
    core_task = scenario_damage
    is_stochastic = True
    precalc = 'scenario'
    accept_precalc = ['scenario']

    def pre_execute(self):
        super().pre_execute()
        self.riskinputs = self.build_riskinputs('gmf')
        self.param['tags'] = list(self.assetcol.tagcol)

    def post_execute(self, result):
        """
        Compute stats for the aggregated distributions and save
        the results on the datastore.
        """
        if not result:
            self.collapsed()
            return
        dstates = self.crmodel.damage_states
        ltypes = self.crmodel.loss_types
        L = len(ltypes)
        R = len(self.rlzs_assoc.realizations)
        D = len(dstates)
        N = len(self.assetcol)
        F = self.oqparam.number_of_ground_motion_fields

        # damage distributions
        dt_list = []
        mean_std_dt = numpy.dtype([('mean', (F32, D)), ('stddev', (F32, D))])
        for ltype in ltypes:
            dt_list.append((ltype, mean_std_dt))
        d_asset = numpy.zeros((N, R, L, 2, D), F32)
        for (l, r, a, stat) in result['d_asset']:
            d_asset[a, r, l] = stat
        self.datastore['dmg_by_asset'] = d_asset
        dmg_dt = [(ds, F32) for ds in self.crmodel.damage_states]
        d_event = numpy.zeros((F, R, L), dmg_dt)
        for d, ds in enumerate(self.crmodel.damage_states):
            d_event[ds] = result['d_event'][:, :, :, d]
        self.datastore['dmg_by_event'] = d_event

        # consequence distributions
        del result['d_asset']
        del result['d_event']
        dtlist = [('event_id', U32), ('rlz_id', U16), ('loss', (F32, (L,)))]
        stat_dt = numpy.dtype([('mean', F32), ('stddev', F32)])
        for name, csq in result.items():
            if name.endswith('_by_asset'):
                c_asset = numpy.zeros((N, R, L), stat_dt)
                for (l, r, a, stat) in result[name]:
                    c_asset[a, r, l] = stat
                self.datastore[name] = c_asset
            elif name.endswith('_by_event'):
                self.datastore[name] = numpy.fromiter(
                    ((eid + rlzi * F, rlzi, F32(result[name][eid, rlzi]))
                     for rlzi in range(R) for eid in range(F)), dtlist)
<commit_msg>[skip hazardlib]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import numpy
from openquake.risklib import scientific
from openquake.calculators import base

U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
F64 = numpy.float64

def scenario_damage(riskinputs, crmodel, param, monitor):
    """
    Core function for a damage computation.

    :param riskinputs:
        :class:`openquake.risklib.riskinput.RiskInput` objects
    :param crmodel:
        a :class:`openquake.risklib.riskinput.CompositeRiskModel` instance
    :param monitor:
        :class:`openquake.baselib.performance.Monitor` instance
    :param param:
        dictionary of extra parameters
    :returns:
        a dictionary {'d_asset': [(l, r, a, mean-stddev), ...],
                      'd_event': damage array of shape R, L, E, D,
                      + optional consequences}

    `d_asset` and `d_tag` are related to the damage distributions.
    """
    L = len(crmodel.loss_types)
    D = len(crmodel.damage_states)
    F = param['number_of_ground_motion_fields']
    R = riskinputs[0].hazard_getter.num_rlzs
    consequences = crmodel.get_consequences()
    result = dict(d_asset=[])
    if F:
        result['d_event'] = numpy.zeros((F, R, L, D), F64)
        for name in consequences:
            result[name + '_by_event'] = numpy.zeros((F, R, L), F64)
    for name in consequences:
        result[name + '_by_asset'] = []
    for ri in riskinputs:
        for out in ri.gen_outputs(crmodel, monitor):
            r = out.rlzi
            for l, loss_type in enumerate(crmodel.loss_types):
                for asset, fractions in zip(ri.assets, out[loss_type]):
                    dmg = fractions * asset['number']  # shape (F, D)
                    if F:
                        result['d_event'][:, r, l] += dmg
                    result['d_asset'].append(
                        (l, r, asset['ordinal'], scientific.mean_std(dmg)))
                    csq = crmodel.compute_csq(asset, fractions, loss_type)
                    for name, value in csq.items():
                        result[name + '_by_asset'].append(
                            (l, r, asset['ordinal'],
                             scientific.mean_std(value)))
                        if F:
                            result[name + '_by_event'][:, r, l] += value
    return result

@base.calculators.add('scenario_damage')
class ScenarioDamageCalculator(base.RiskCalculator):
    """
    Scenario damage calculator
    """
    core_task = scenario_damage
    is_stochastic = True
    precalc = 'scenario'
    accept_precalc = ['scenario']

    def pre_execute(self):
        super().pre_execute()
        F = self.oqparam.number_of_ground_motion_fields
        self.param['number_of_ground_motion_fields'] = F
        self.riskinputs = self.build_riskinputs('gmf')
        self.param['tags'] = list(self.assetcol.tagcol)

    def post_execute(self, result):
        """
        Compute stats for the aggregated distributions and save
        the results on the datastore.
        """
        if not result:
            self.collapsed()
            return
        dstates = self.crmodel.damage_states
        ltypes = self.crmodel.loss_types
        L = len(ltypes)
        R = len(self.rlzs_assoc.realizations)
        D = len(dstates)
        N = len(self.assetcol)
        F = self.oqparam.number_of_ground_motion_fields

        # damage distributions
        dt_list = []
        mean_std_dt = numpy.dtype([('mean', (F32, D)), ('stddev', (F32, D))])
        for ltype in ltypes:
            dt_list.append((ltype, mean_std_dt))
        d_asset = numpy.zeros((N, R, L, 2, D), F32)
        for (l, r, a, stat) in result['d_asset']:
            d_asset[a, r, l] = stat
        self.datastore['dmg_by_asset'] = d_asset
        dmg_dt = [(ds, F32) for ds in self.crmodel.damage_states]
        d_event = numpy.zeros((F, R, L), dmg_dt)
        for d, ds in enumerate(self.crmodel.damage_states):
            d_event[ds] = result['d_event'][:, :, :, d]
        self.datastore['dmg_by_event'] = d_event

        # consequence distributions
        del result['d_asset']
        del result['d_event']
        dtlist = [('event_id', U32), ('rlz_id', U16), ('loss', (F32, (L,)))]
        stat_dt = numpy.dtype([('mean', F32), ('stddev', F32)])
        for name, csq in result.items():
            if name.endswith('_by_asset'):
                c_asset = numpy.zeros((N, R, L), stat_dt)
                for (l, r, a, stat) in result[name]:
                    c_asset[a, r, l] = stat
                self.datastore[name] = c_asset
            elif name.endswith('_by_event'):
                self.datastore[name] = numpy.fromiter(
                    ((eid + rlzi * F, rlzi, F32(result[name][eid, rlzi]))
                     for rlzi in range(R) for eid in range(F)), dtlist)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Utility to setup a font for addition to Piper.

Generate METADATA.pb files for font families.

METADATA.pb files are used to serve the families on http://fonts.google.com.
Font families are stored in this repo by license type. The following
directories contain font families:

../fonts/ofl
../fonts/apache
../fonts/ufl

Generating a METADATA.pb file for a new family:

1. Determine the family's license type, ofl, ufl or apache
2. Create a new folder under the license type directory
3. Name the folder so it's the family name, all lowercase and no spaces.
4. Run the following: python add_font.py /path/to/new/family
5. Update the category field in the generated METADATA.pb file.

Generating a METADATA.pb file for an existing family:

1. run the following: python add_font.py --update /path/to/existing/family
"""
from __future__ import print_function
from functools import cmp_to_key
import contextlib
import errno
import glob
import os
import sys
import time
from fontTools import ttLib

from absl import flags
import gftools.fonts_public_pb2 as fonts_pb2
from gftools.util import google_fonts as fonts
from gftools.utils import cmp
from absl import app
from google.protobuf import text_format

FLAGS = flags.FLAGS

flags.DEFINE_integer('min_pct', 50,
                     'What percentage of subset codepoints have to be supported'
                     ' for a non-ext subset.')
# if a single glyph from the 81 glyphs in *-ext_unique-glyphs.nam file is present, the font can have the "ext" subset
flags.DEFINE_float('min_pct_ext', 0.01,
                   'What percentage of subset codepoints have to be supported'
                   ' for a -ext subset.')

def _FileFamilyStyleWeights(fontdir):
  """Extracts file, family, style, weight 4-tuples for each font in dir.

  Args:
    fontdir: Directory that supposedly contains font files for a family.
  Returns:
    List of fonts.FileFamilyStyleWeightTuple ordered by weight, style
    (normal first).
  Raises:
    OSError: If the font directory doesn't exist (errno.ENOTDIR) or has no font
    files (errno.ENOENT) in it.
    RuntimeError: If the font directory appears to contain files from multiple
    families.
  """
  if not os.path.isdir(fontdir):
    raise OSError(errno.ENOTDIR, 'No such directory', fontdir)

  files = glob.glob(os.path.join(fontdir, '*.[ot]tf'))
  if not files:
    raise OSError(errno.ENOENT, 'no font files found')

  result = [fonts.FamilyStyleWeight(f) for f in files]
  def _Cmp(r1, r2):
    return cmp(r1.weight, r2.weight) or -cmp(r1.style, r2.style)
  result = sorted(result, key=cmp_to_key(_Cmp))

  family_names = {i.family for i in result}
  if len(family_names) > 1:
    raise RuntimeError('Ambiguous family name; possibilities: %s'
                       % family_names)

  return result

def _MakeMetadata(fontdir, is_new):
  """Builds a dictionary matching a METADATA.pb file.

  Args:
    fontdir: Directory containing font files for which we want metadata.
    is_new: Whether this is an existing or new family.
  Returns:
    OrderedDict of a complete METADATA.pb structure.
  Raises:
    RuntimeError: If the variable font axes info differs between font files of
    same family.
  """
  file_family_style_weights = _FileFamilyStyleWeights(fontdir)

  first_file = file_family_style_weights[0].file
  old_metadata_file = os.path.join(fontdir, 'METADATA.pb')
  font_license = fonts.LicenseFromPath(fontdir)

  metadata = fonts_pb2.FamilyProto()
  metadata.name = file_family_style_weights[0].family

  subsets_in_font = [s[0] for s in fonts.SubsetsInFont(
    first_file, FLAGS.min_pct, FLAGS.min_pct_ext
  )]

  if not is_new:
    old_metadata = fonts_pb2.FamilyProto()
    with open(old_metadata_file, 'rb') as old_meta:
      text_format.Parse(old_meta.read(), old_metadata)
      metadata.designer = old_metadata.designer
      metadata.category = old_metadata.category
      metadata.date_added = old_metadata.date_added
      subsets = set(old_metadata.subsets) | set(subsets_in_font)
  else:
    metadata.designer = 'UNKNOWN'
    metadata.category = 'SANS_SERIF'
    metadata.date_added = time.strftime('%Y-%m-%d')
    subsets = ['menu'] + subsets_in_font

  metadata.license = font_license
  subsets = sorted(subsets)
  for subset in subsets:
    metadata.subsets.append(subset)

  for (fontfile, family, style, weight) in file_family_style_weights:
    filename = os.path.basename(fontfile)
    font_psname = fonts.ExtractName(fontfile, fonts.NAME_PSNAME,
                                    os.path.splitext(filename)[0])
    font_copyright = fonts.ExtractName(fontfile, fonts.NAME_COPYRIGHT,
                                       '???.').strip()

    font_metadata = metadata.fonts.add()
    font_metadata.name = family
    font_metadata.style = style
    font_metadata.weight = weight
    font_metadata.filename = filename
    font_metadata.post_script_name = font_psname
    default_fullname = os.path.splitext(filename)[0].replace('-', ' ')
    font_metadata.full_name = fonts.ExtractName(fontfile, fonts.NAME_FULLNAME,
                                                default_fullname)
    font_metadata.copyright = font_copyright

  axes_info_from_font_files \
    = {_AxisInfo(f.file) for f in file_family_style_weights}
  if len(axes_info_from_font_files) != 1:
    raise RuntimeError('Variable axes info not matching between font files')

  for axes_info in axes_info_from_font_files:
    if axes_info:
      for axes in axes_info:
        var_axes = metadata.axes.add()
        var_axes.tag = axes[0]
        var_axes.min_value = axes[1]
        var_axes.max_value = axes[2]

  return metadata

def _AxisInfo(fontfile):
  """Gets variable axes info.

  Args:
    fontfile: Font file to look at for variation info

  Returns:
    Variable axes info
  """
  with contextlib.closing(ttLib.TTFont(fontfile)) as font:
    if 'fvar' not in font:
      return frozenset()
    else:
      fvar = font['fvar']
      axis_info = [
          (a.axisTag, a.minValue, a.maxValue) for a in fvar.axes
      ]
      return tuple(sorted(axis_info))

def _GetAvgSize(file_family_style_weights):
  """Gets average file size of all font weights.

  Returns:
       average file size.

  Args:
    file_family_style_weights: List of fonts.FileFamilyStyleWeightTuple.
  """
  total_size = 0
  for list_tuple in file_family_style_weights:
    total_size += os.stat(list_tuple.file).st_size
  return total_size / len(file_family_style_weights)

def _WriteTextFile(filename, text):
  """Write text to file.

  Nop if file exists with that exact content. This allows running against files
  that are in Piper and not marked for editing; you will get an error only if
  something changed.

  Args:
    filename: The file to write.
    text: The content to write to the file.
  """
  if os.path.isfile(filename):
    with open(filename, 'r') as f:
      current = f.read()
    if current == text:
      print('No change to %s' % filename)
      return

  with open(filename, 'w') as f:
    f.write(text)
  print('Wrote %s' % filename)

def _AddHumanReadableDateComment(text_proto):
  return re.sub(r'(date_added: \d+)',
                r'\1  # ' + time.strftime('%Y-%m-%d'), text_proto)

def main(argv):
  if len(argv) != 2:
    sys.exit('One argument, a directory containing a font family')
  fontdir = argv[1]

  is_new = True
  old_metadata_file = os.path.join(fontdir, 'METADATA.pb')
  if os.path.isfile(old_metadata_file):
    is_new = False

  metadata = _MakeMetadata(fontdir, is_new)
  text_proto = text_format.MessageToString(metadata, as_utf8=True)

  desc = os.path.join(fontdir, 'DESCRIPTION.en_us.html')
  if os.path.isfile(desc):
    print('DESCRIPTION.en_us.html exists')
  else:
    _WriteTextFile(desc, 'N/A')

  _WriteTextFile(os.path.join(fontdir, 'METADATA.pb'), text_proto)

if __name__ == '__main__':
    app.run(main)
<commit_msg>Update some docstrings in add-font.<commit_after>#!/usr/bin/env python3
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Utility to setup a font for addition to Piper.

Generate METADATA.pb files for font families.

METADATA.pb files are used to serve the families on http://fonts.google.com.
Font families are stored in this repo by license type. The following
directories contain font families:

../fonts/ofl
../fonts/apache
../fonts/ufl

Generating a METADATA.pb file for a new family:

1. Determine the family's license type, ofl, ufl or apache
2. Create a new folder under the license type directory
3. Name the folder so it's the family name, all lowercase and no spaces.
4. Run the following: gftools add-font /path/to/new/family
5. Update the category field in the generated METADATA.pb file.

Generating a METADATA.pb file for an existing family:

1. run the following: gftools add-font /path/to/existing/family
"""
from __future__ import print_function
from functools import cmp_to_key
import contextlib
import errno
import glob
import os
import sys
import time
from fontTools import ttLib

from absl import flags
import gftools.fonts_public_pb2 as fonts_pb2
from gftools.util import google_fonts as fonts
from gftools.utils import cmp
from absl import app
from google.protobuf import text_format

FLAGS = flags.FLAGS

flags.DEFINE_integer('min_pct', 50,
                     'What percentage of subset codepoints have to be supported'
                     ' for a non-ext subset.')
# if a single glyph from the 81 glyphs in *-ext_unique-glyphs.nam file is present, the font can have the "ext" subset
flags.DEFINE_float('min_pct_ext', 0.01,
                   'What percentage of subset codepoints have to be supported'
                   ' for a -ext subset.')

def _FileFamilyStyleWeights(fontdir):
  """Extracts file, family, style, weight 4-tuples for each font in dir.

  Args:
    fontdir: Directory that supposedly contains font files for a family.
  Returns:
    List of fonts.FileFamilyStyleWeightTuple ordered by weight, style
    (normal first).
  Raises:
    OSError: If the font directory doesn't exist (errno.ENOTDIR) or has no font
    files (errno.ENOENT) in it.
    RuntimeError: If the font directory appears to contain files from multiple
    families.
  """
  if not os.path.isdir(fontdir):
    raise OSError(errno.ENOTDIR, 'No such directory', fontdir)

  files = glob.glob(os.path.join(fontdir, '*.[ot]tf'))
  if not files:
    raise OSError(errno.ENOENT, 'no font files found')

  result = [fonts.FamilyStyleWeight(f) for f in files]
  def _Cmp(r1, r2):
    return cmp(r1.weight, r2.weight) or -cmp(r1.style, r2.style)
  result = sorted(result, key=cmp_to_key(_Cmp))

  family_names = {i.family for i in result}
  if len(family_names) > 1:
    raise RuntimeError('Ambiguous family name; possibilities: %s'
                       % family_names)

  return result

def _MakeMetadata(fontdir, is_new):
  """Builds a dictionary matching a METADATA.pb file.

  Args:
    fontdir: Directory containing font files for which we want metadata.
    is_new: Whether this is an existing or new family.
  Returns:
    A fonts_pb2.FamilyProto message, the METADATA.pb structure.
  Raises:
    RuntimeError: If the variable font axes info differs between font files of
    same family.
  """
  file_family_style_weights = _FileFamilyStyleWeights(fontdir)

  first_file = file_family_style_weights[0].file
  old_metadata_file = os.path.join(fontdir, 'METADATA.pb')
  font_license = fonts.LicenseFromPath(fontdir)

  metadata = fonts_pb2.FamilyProto()
  metadata.name = file_family_style_weights[0].family

  subsets_in_font = [s[0] for s in fonts.SubsetsInFont(
    first_file, FLAGS.min_pct, FLAGS.min_pct_ext
  )]

  if not is_new:
    old_metadata = fonts_pb2.FamilyProto()
    with open(old_metadata_file, 'rb') as old_meta:
      text_format.Parse(old_meta.read(), old_metadata)
      metadata.designer = old_metadata.designer
      metadata.category = old_metadata.category
      metadata.date_added = old_metadata.date_added
      subsets = set(old_metadata.subsets) | set(subsets_in_font)
  else:
    metadata.designer = 'UNKNOWN'
    metadata.category = 'SANS_SERIF'
    metadata.date_added = time.strftime('%Y-%m-%d')
    subsets = ['menu'] + subsets_in_font

  metadata.license = font_license
  subsets = sorted(subsets)
  for subset in subsets:
    metadata.subsets.append(subset)

  for (fontfile, family, style, weight) in file_family_style_weights:
    filename = os.path.basename(fontfile)
    font_psname = fonts.ExtractName(fontfile, fonts.NAME_PSNAME,
                                    os.path.splitext(filename)[0])
    font_copyright = fonts.ExtractName(fontfile, fonts.NAME_COPYRIGHT,
                                       '???.').strip()

    font_metadata = metadata.fonts.add()
    font_metadata.name = family
    font_metadata.style = style
    font_metadata.weight = weight
    font_metadata.filename = filename
    font_metadata.post_script_name = font_psname
    default_fullname = os.path.splitext(filename)[0].replace('-', ' ')
    font_metadata.full_name = fonts.ExtractName(fontfile, fonts.NAME_FULLNAME,
                                                default_fullname)
    font_metadata.copyright = font_copyright

  axes_info_from_font_files \
    = {_AxisInfo(f.file) for f in file_family_style_weights}
  if len(axes_info_from_font_files) != 1:
    raise RuntimeError('Variable axes info not matching between font files')

  for axes_info in axes_info_from_font_files:
    if axes_info:
      for axes in axes_info:
        var_axes = metadata.axes.add()
        var_axes.tag = axes[0]
        var_axes.min_value = axes[1]
        var_axes.max_value = axes[2]

  return metadata

def _AxisInfo(fontfile):
  """Gets variable axes info.

  Args:
    fontfile: Font file to look at for variation info

  Returns:
    Variable axes info
  """
  with contextlib.closing(ttLib.TTFont(fontfile)) as font:
    if 'fvar' not in font:
      return frozenset()
    else:
      fvar = font['fvar']
      axis_info = [
          (a.axisTag, a.minValue, a.maxValue) for a in fvar.axes
      ]
      return tuple(sorted(axis_info))

def _GetAvgSize(file_family_style_weights):
  """Gets average file size of all font weights.

  Returns:
       average file size.

  Args:
    file_family_style_weights: List of fonts.FileFamilyStyleWeightTuple.
  """
  total_size = 0
  for list_tuple in file_family_style_weights:
    total_size += os.stat(list_tuple.file).st_size
  return total_size / len(file_family_style_weights)

def _WriteTextFile(filename, text):
  """Write text to file.

  Nop if file exists with that exact content. This allows running against files
  that are in Piper and not marked for editing; you will get an error only if
  something changed.

  Args:
    filename: The file to write.
    text: The content to write to the file.
  """
  if os.path.isfile(filename):
    with open(filename, 'r') as f:
      current = f.read()
    if current == text:
      print('No change to %s' % filename)
      return

  with open(filename, 'w') as f:
    f.write(text)
  print('Wrote %s' % filename)

def _AddHumanReadableDateComment(text_proto):
  return re.sub(r'(date_added: \d+)',
                r'\1  # ' + time.strftime('%Y-%m-%d'), text_proto)

def main(argv):
  if len(argv) != 2:
    sys.exit('One argument, a directory containing a font family')
  fontdir = argv[1]

  is_new = True
  old_metadata_file = os.path.join(fontdir, 'METADATA.pb')
  if os.path.isfile(old_metadata_file):
    is_new = False

  metadata = _MakeMetadata(fontdir, is_new)
  text_proto = text_format.MessageToString(metadata, as_utf8=True)

  desc = os.path.join(fontdir, 'DESCRIPTION.en_us.html')
  if os.path.isfile(desc):
    print('DESCRIPTION.en_us.html exists')
  else:
    _WriteTextFile(desc, 'N/A')

  _WriteTextFile(os.path.join(fontdir, 'METADATA.pb'), text_proto)

if __name__ == '__main__':
    app.run(main)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import time
import logging
import functools
from datetime import datetime
import numpy

from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, AccumDict
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import pmap_from_grp, poe_map
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib import valid, nrml
from openquake.commonlib import source, readinput, config
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base, classical
from openquake.calculators.ucerf_event_based import (
    UCERFControl, DEFAULT_TRT, UcerfSource)
# FIXME: the counting of effective ruptures has to be revised completely

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFControl(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])
SourceConverter.convert_UCERFSource = convert_UCERFSource

def _hazard_curves_per_rupture_subset(
        rupset_idx, ucerf_source, sites, imtls, cmaker,
        truncation_level=None, monitor=Monitor()):
    """
    Calculates the probabilities of exceedence from a set of rupture indices
    """
    imtls = DictArray(imtls)
    ctx_mon = monitor('making contexts', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    pmap = ProbabilityMap(len(imtls.array), len(cmaker.gsims))
    pmap.calc_times = []
    pmap.grp_id = ucerf_source.src_group_id
    nsites = len(sites)
    ucerf_source.rupset_idx = rupset_idx
    ucerf_source.num_ruptures = len(rupset_idx)
    pmap.eff_ruptures = {pmap.grp_id: ucerf_source.num_ruptures}
    # NB: the effective ruptures can be less, some may have zero probability
    t0 = time.time()
    upmap = poe_map(ucerf_source, sites, imtls, cmaker,
                    truncation_level, ctx_mon, pne_mon)
    pmap |= upmap
    pmap.calc_times.append(
        (ucerf_source.source_id, nsites, time.time() - t0))
    return pmap

def ucerf_classical_hazard_by_rupture_set(
        rupset_idx, ucerf_source, src_filter, gsims, monitor):
    """
    :param rupset_idx:
        indices of the rupture sets
    :param ucerf_source:
        an object taking the place of a source for UCERF
    :param src_filter:
        a source filter returning the sites affected by the source
    :param gsims:
        a list of GSIMs
    :param monitor:
        a monitor instance
    :returns:
        a ProbabilityMap
    """
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    max_dist = src_filter.integration_distance[DEFAULT_TRT]
    rupset_idx, s_sites = \
        ucerf_source.filter_sites_by_distance_from_rupture_set(
            rupset_idx, src_filter.sitecol, max_dist)
    if len(s_sites):
        ucerf_source.src_filter = src_filter  # so that .iter_ruptures() work
        cmaker = ContextMaker(gsims, max_dist)
        pm = _hazard_curves_per_rupture_subset(
            rupset_idx, ucerf_source, s_sites, imtls, cmaker,
            truncation_level, monitor=monitor)
    else:  # return an empty probability map
        pm = ProbabilityMap(len(imtls.array), len(gsims))
        pm.calc_times = []  # TODO: fix .calc_times
        pm.eff_ruptures = {ucerf_source.src_group_id: 0}
        pm.grp_id = ucerf_source.src_group_id
    return pm
ucerf_classical_hazard_by_rupture_set.shared_dir_on = config.SHARED_DIR_ON

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(classical.PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = ucerf_classical_hazard_by_rupture_set
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        self.sitecol = readinput.get_site_collection(self.oqparam)
        self.gsim_lt = readinput.get_gsim_lt(self.oqparam, [DEFAULT_TRT])
        self.smlt = readinput.get_source_model_lt(self.oqparam)
        parser = nrml.SourceModelParser(
            SourceConverter(self.oqparam.investigation_time,
                            self.oqparam.rupture_mesh_spacing))
        [self.src_group] = parser.parse_src_groups(
            self.oqparam.inputs["source_model"])
        source_models = []
        for sm in self.smlt.gen_source_models(self.gsim_lt):
            sg = copy.copy(self.src_group)
            sg.id = sm.ordinal
            sm.src_groups = [sg]
            # sm.path[0]=ltbrTrueMean, sm.name=FM0_0/MEANFS/MEANMSR/MeanRates
            sg.sources = [UcerfSource(sg[0], sm.ordinal, sm.path[0], sm.name)]
            source_models.append(sm)
        self.csm = source.CompositeSourceModel(
            self.gsim_lt, self.smlt, source_models, set_weight=True)
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
        self.rup_data = {}
        self.num_tiles = 1

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor.new(self.core_task.__name__)
        monitor.oqparam = oq = self.oqparam
        self.src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.rlzs_assoc.gsims_by_grp_id.items()})
        acc.calc_times = []
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility

        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.rlzs_assoc.gsims_by_grp_id[grp_id]
            [[ucerf_source]] = sm.src_groups
            ucerf_source.nsites = len(self.sitecol)
            self.csm.infos[grp_id, ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            logging.info('Getting the background point sources')
            with self.monitor('getting background sources', autoflush=True):
                background_sids = ucerf_source.get_background_sids(
                    self.src_filter)
                bckgnd_sources = ucerf_source.get_background_sources(
                    background_sids)

            # parallelize on the background sources, small tasks
            args = (bckgnd_sources, self.src_filter, oq.imtls,
                    gsims, self.oqparam.truncation_level, (), monitor)
            bg_res = parallel.Starmap.apply(
                pmap_from_grp, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=self.oqparam.concurrent_tasks).submit_all()

            # parallelize by rupture subsets
            rup_sets = numpy.arange(ucerf_source.num_ruptures)
            taskname = 'ucerf_classical_hazard_by_rupture_set_%d' % grp_id
            rup_res = parallel.Starmap.apply(
                ucerf_classical_hazard_by_rupture_set,
                (rup_sets, ucerf_source, self.src_filter, gsims, monitor),
                concurrent_tasks=self.oqparam.concurrent_tasks, name=taskname
            ).submit_all()

            # compose probabilities from background sources
            for pmap in bg_res:
                acc[grp_id] |= pmap
            self.save_data_transfer(bg_res)

            acc = functools.reduce(self.agg_dicts, rup_res, acc)
            with self.monitor('store source_info', autoflush=True):
                self.store_source_info(self.csm.infos)
                self.save_data_transfer(rup_res)
        self.datastore['csm_info'] = self.csm.info
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
            functools.partial(self.count_eff_ruptures, acc))
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(classical.ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<commit_msg>Halved the number of tasks [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import copy
import time
import logging
import functools
from datetime import datetime
import numpy

from openquake.baselib.performance import Monitor
from openquake.baselib.general import DictArray, AccumDict
from openquake.baselib import parallel
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.calc.hazard_curve import pmap_from_grp, poe_map
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib import valid, nrml
from openquake.commonlib import source, readinput, config
from openquake.hazardlib.sourceconverter import SourceConverter

from openquake.calculators import base, classical
from openquake.calculators.ucerf_event_based import (
    UCERFControl, DEFAULT_TRT, UcerfSource)
# FIXME: the counting of effective ruptures has to be revised completely

def convert_UCERFSource(self, node):
    """
    Converts the Ucerf Source node into an SES Control object
    """
    dirname = os.path.dirname(self.fname)  # where the source_model_file is
    source_file = os.path.join(dirname, node["filename"])
    if "startDate" in node.attrib and "investigationTime" in node.attrib:
        # Is a time-dependent model - even if rates were originally
        # poissonian
        # Verify that the source time span is the same as the TOM time span
        inv_time = float(node["investigationTime"])
        if inv_time != self.tom.time_span:
            raise ValueError("Source investigation time (%s) is not "
                             "equal to configuration investigation time "
                             "(%s)" % (inv_time, self.tom.time_span))
        start_date = datetime.strptime(node["startDate"], "%d/%m/%Y")
    else:
        start_date = None
    return UCERFControl(
        source_file,
        node["id"],
        self.tom.time_span,
        start_date,
        float(node["minMag"]),
        npd=self.convert_npdist(node),
        hdd=self.convert_hpdist(node),
        aspect=~node.ruptAspectRatio,
        upper_seismogenic_depth=~node.pointGeometry.upperSeismoDepth,
        lower_seismogenic_depth=~node.pointGeometry.lowerSeismoDepth,
        msr=valid.SCALEREL[~node.magScaleRel](),
        mesh_spacing=self.rupture_mesh_spacing,
        trt=node["tectonicRegion"])
SourceConverter.convert_UCERFSource = convert_UCERFSource

def _hazard_curves_per_rupture_subset(
        rupset_idx, ucerf_source, sites, imtls, cmaker,
        truncation_level=None, monitor=Monitor()):
    """
    Calculates the probabilities of exceedence from a set of rupture indices
    """
    imtls = DictArray(imtls)
    ctx_mon = monitor('making contexts', measuremem=False)
    pne_mon = monitor('computing poes', measuremem=False)
    pmap = ProbabilityMap(len(imtls.array), len(cmaker.gsims))
    pmap.calc_times = []
    pmap.grp_id = ucerf_source.src_group_id
    nsites = len(sites)
    ucerf_source.rupset_idx = rupset_idx
    ucerf_source.num_ruptures = len(rupset_idx)
    pmap.eff_ruptures = {pmap.grp_id: ucerf_source.num_ruptures}
    # NB: the effective ruptures can be less, some may have zero probability
    t0 = time.time()
    upmap = poe_map(ucerf_source, sites, imtls, cmaker,
                    truncation_level, ctx_mon, pne_mon)
    pmap |= upmap
    pmap.calc_times.append(
        (ucerf_source.source_id, nsites, time.time() - t0))
    return pmap

def ucerf_classical_hazard_by_rupture_set(
        rupset_idx, ucerf_source, src_filter, gsims, monitor):
    """
    :param rupset_idx:
        indices of the rupture sets
    :param ucerf_source:
        an object taking the place of a source for UCERF
    :param src_filter:
        a source filter returning the sites affected by the source
    :param gsims:
        a list of GSIMs
    :param monitor:
        a monitor instance
    :returns:
        a ProbabilityMap
    """
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    max_dist = src_filter.integration_distance[DEFAULT_TRT]
    rupset_idx, s_sites = \
        ucerf_source.filter_sites_by_distance_from_rupture_set(
            rupset_idx, src_filter.sitecol, max_dist)
    if len(s_sites):
        ucerf_source.src_filter = src_filter  # so that .iter_ruptures() work
        cmaker = ContextMaker(gsims, max_dist)
        pm = _hazard_curves_per_rupture_subset(
            rupset_idx, ucerf_source, s_sites, imtls, cmaker,
            truncation_level, monitor=monitor)
    else:  # return an empty probability map
        pm = ProbabilityMap(len(imtls.array), len(gsims))
        pm.calc_times = []  # TODO: fix .calc_times
        pm.eff_ruptures = {ucerf_source.src_group_id: 0}
        pm.grp_id = ucerf_source.src_group_id
    return pm
ucerf_classical_hazard_by_rupture_set.shared_dir_on = config.SHARED_DIR_ON

@base.calculators.add('ucerf_psha')
class UcerfPSHACalculator(classical.PSHACalculator):
    """
    UCERF classical calculator.
    """
    core_task = ucerf_classical_hazard_by_rupture_set
    is_stochastic = False

    def pre_execute(self):
        """
        parse the logic tree and source model input
        """
        logging.warn('%s is still experimental', self.__class__.__name__)
        self.sitecol = readinput.get_site_collection(self.oqparam)
        self.gsim_lt = readinput.get_gsim_lt(self.oqparam, [DEFAULT_TRT])
        self.smlt = readinput.get_source_model_lt(self.oqparam)
        parser = nrml.SourceModelParser(
            SourceConverter(self.oqparam.investigation_time,
                            self.oqparam.rupture_mesh_spacing))
        [self.src_group] = parser.parse_src_groups(
            self.oqparam.inputs["source_model"])
        source_models = []
        for sm in self.smlt.gen_source_models(self.gsim_lt):
            sg = copy.copy(self.src_group)
            sg.id = sm.ordinal
            sm.src_groups = [sg]
            # sm.path[0]=ltbrTrueMean, sm.name=FM0_0/MEANFS/MEANMSR/MeanRates
            sg.sources = [UcerfSource(sg[0], sm.ordinal, sm.path[0], sm.name)]
            source_models.append(sm)
        self.csm = source.CompositeSourceModel(
            self.gsim_lt, self.smlt, source_models, set_weight=True)
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc()
        self.rup_data = {}
        self.num_tiles = 1

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor.new(self.core_task.__name__)
        monitor.oqparam = oq = self.oqparam
        self.src_filter = SourceFilter(self.sitecol, oq.maximum_distance)
        acc = AccumDict({
            grp_id: ProbabilityMap(len(oq.imtls.array), len(gsims))
            for grp_id, gsims in self.rlzs_assoc.gsims_by_grp_id.items()})
        acc.calc_times = []
        acc.eff_ruptures = AccumDict()  # grp_id -> eff_ruptures
        acc.bb_dict = {}  # just for API compatibility

        for sm in self.csm.source_models:  # one branch at the time
            grp_id = sm.ordinal
            gsims = self.rlzs_assoc.gsims_by_grp_id[grp_id]
            [[ucerf_source]] = sm.src_groups
            ucerf_source.nsites = len(self.sitecol)
            self.csm.infos[grp_id, ucerf_source.source_id] = source.SourceInfo(
                ucerf_source)
            logging.info('Getting the background point sources')
            with self.monitor('getting background sources', autoflush=True):
                background_sids = ucerf_source.get_background_sids(
                    self.src_filter)
                bckgnd_sources = ucerf_source.get_background_sources(
                    background_sids)
            ct2 = (self.oqparam.concurrent_tasks // 2) or 1

            # parallelize on the background sources, small tasks
            args = (bckgnd_sources, self.src_filter, oq.imtls,
                    gsims, self.oqparam.truncation_level, (), monitor)
            bg_res = parallel.Starmap.apply(
                pmap_from_grp, args, name='background_sources_%d' % grp_id,
                concurrent_tasks=ct2).submit_all()

            # parallelize by rupture subsets
            rup_sets = numpy.arange(ucerf_source.num_ruptures)
            taskname = 'ucerf_classical_hazard_by_rupture_set_%d' % grp_id
            rup_res = parallel.Starmap.apply(
                ucerf_classical_hazard_by_rupture_set,
                (rup_sets, ucerf_source, self.src_filter, gsims, monitor),
                concurrent_tasks=ct2, name=taskname
            ).submit_all()

            # compose probabilities from background sources
            for pmap in bg_res:
                acc[grp_id] |= pmap
            self.save_data_transfer(bg_res)

            acc = functools.reduce(self.agg_dicts, rup_res, acc)
            with self.monitor('store source_info', autoflush=True):
                self.store_source_info(self.csm.infos)
                self.save_data_transfer(rup_res)
        self.datastore['csm_info'] = self.csm.info
        self.rlzs_assoc = self.csm.info.get_rlzs_assoc(
            functools.partial(self.count_eff_ruptures, acc))
        return acc  # {grp_id: pmap}

@base.calculators.add('ucerf_classical')
class UCERFClassicalCalculator(classical.ClassicalCalculator):
    pre_calculator = 'ucerf_psha'
<|endoftext|>