<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class EdiInvoice(orm.Model):
    """ Model name: Edi Invoice
    """

    _name = 'edi.invoice'
    _description = 'EDI invoice'
    _rec_name = 'name'
    _order = 'name'

    _columns = {
        'name': fields.char('Number', size=20, required=True),
        'date': fields.date('Invoice date', required=True),
        }

class EdiInvoiceLine(orm.Model):
    """ Model name: Edi Invoice Line
    """

    _name = 'edi.invoice.line'
    _description = 'EDI invoice line'
    _rec_name = 'name'
    _order = 'invoice_id,name'

    def import_invoice_line_from_account(self, cr, uid, context=None):
        ''' Import procedure for get all invoice line for check
            Export with sprix: spx780
        '''
        # Pool used:
        invoice_pool = self.pool.get('edi.invoice')

        filename = '~/etl/edi/elior/controllo/fatture.txt'

        # ---------------------------------------------------------------------
        # Clean previous elements:
        # ---------------------------------------------------------------------
        # Details:
        line_ids = self.search(cr, uid, [], context=context)
        self.unlink(cr, uid, line_ids, context=context)

        # Invoice:
        invoice_ids = invoice_pool.search(cr, uid, [], context=context)
        invoice_pool.unlink(cr, uid, invoice_ids, context=context)

        invoice_db = {}
        for row in open(filename, 'r'):
            # -----------------------------------------------------------------
            # Header data:
            # -----------------------------------------------------------------
            invoice_number = row[:6]
            invoice_date = '%s-%s-%s' % (
                row[6:10],
                row[10:12],
                row[12:14],
                )
            invoice_number = 'FT-%s-%s' % (
                invoice_number, row[6:10], )

            if invoice_number not in invoice_db:
                invoice_db['invoice_number'] = invoice_pool.create(cr, uid, {
                    'name': invoice_number,
                    'date': invoice_date,
                    }, context=context)

            # -----------------------------------------------------------------
            # Detail data:
            # -----------------------------------------------------------------
            data = {
                'invoice_id': invoice_db.get(invoice_number, False),
                'order_sequence': int(row[16:20]),
                'name': row[20:36],
                'article': row[36:47],
                'qty': row[47:62],
                'price': row[62:77],
                'uom': row[77:92],
                'description': row[92:127],
                'ddt_number': row[127:133],
                'ddt_date': '%s-%s-%s' % (
                    row[133:137],
                    row[137:139],
                    row[139:141],
                    ),
                'order_number': row[143:159],
                'order_date': '%s-%s-%s' % (
                    row[159:163],
                    row[163:165],
                    row[165:167],
                    ),
                }

            self.create(cr, uid, data, context=context)
        return True

    _columns = {
        'invoice_id': fields.many2one('edi.invoice', 'Invoice'),
        'order_sequence': fields.integer('Order position',
            readonly=True),
        'name': fields.char('Company code', size=16,
            required=True, readonly=True),
        'article': fields.char('Customer code', size=16,
            required=True, readonly=True),
        'qty': fields.float('Q.ty', digits=(16, 3),
            required=True, readonly=True),
        'price': fields.float('Price', digits=(16, 3),
            required=True, readonly=True),
        'uom': fields.char('UOM', size=5,
            required=True, readonly=True),
        'description': fields.char('Description', size=16,
            required=True, readonly=True),
        'ddt_number': fields.char('DDT #', size=20,
            required=True, readonly=True),
        'ddt_date': fields.date('Invoice date',
            required=True, readonly=True),
        'order_number': fields.char('Company order #', size=25,
            required=True, readonly=True),
        'order_date': fields.date('Order date',
            required=True, readonly=True),
        }

class EdiInvoice(orm.Model):
    """ Model name: Edi Invoice
    """

    _inherit = 'edi.invoice'

    _columns = {
        'line_ids': fields.one2many('edi.invoice.line', 'invoice_id', 'Lines'),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _name = 'edi.order'
    _description = 'EDI order'
    _rec_name = 'name'

    _columns = {
        'name': fields.char('Number', size=25, required=True),
        'date': fields.datetime('Date', readonly=True),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _name = 'edi.order.folder'
    _description = 'EDI order folder'
    _rec_name = 'name'

    def load_file_in_folder(self, cr, uid, ids, context=None):
        """ Load file in this folder (used also for scheduled)
        """
        # Parameters:
        extension = 'ASC'

        # Pool used:
        order_pool = self.pool.get('edi.order')
        file_pool = self.pool.get('edi.order.file')

        for folder in self.browse(cr, uid, ids, context=context):
            # Delete all previouse file in folder:
            file_ids = file_pool.search(cr, uid, [
                ('folder_id', '=', folder.id),
                ], context=context)
            file_pool.unlink(cr, uid, file_ids, context=context)

            for root, dirs, files in os.walk(folder.path):
                for f in files:
                    if not f.endswith(estension):
                        _logger.warning('Jump file: %s' % f)
                        continue

                    # ---------------------------------------------------------
                    # Create order file:
                    # ---------------------------------------------------------
                    # Open file for get order number:
                    filename = os.path.join(folder.path, files)
                    f_asc = open(filename, 'r')
                    row = f_asc.readline() # XXX only one row
                    f_asc.close()

                    # Parse name:
                    mode = f[:6]
                    date = f[6:22]

                    date = '%s-%s-%s %s:%s:%s' % (
                        # Date:
                        date[:4],
                        date[4:6],
                        date[6:8],

                        # Time:
                        date[8:10],
                        date[10:12],
                        date[12:14],
                        )

                    # Order info:
                    order = row[20:30]

                    order_date = row[30:38]
                    order_date = '%s-%s-%S' % (
                        order_date[:4],
                        order_date[4:6],
                        order_date[6:8],
                        )

                    order_ids = order_pool.search(cr, uid, [
                        ('name', '=', order),
                        # XXX no date
                        ], context=context)

                    if order_ids: # Nothing:
                        order_id = order_ids
                    else: # Create
                        order_id = order_pool.create(cr, uid, {
                            'name': order,
                            'date': order_date,
                            }, context=context)

                    # Create order line:
                    #for row in open(filename, 'r'):
                    #    data = {
                    #        #'order_id':,
                    #        'order_sequence': row[20:30], # XXX errato
                    #        'name': row[320:355],
                    #        'article': row[355:390],
                    #        'qty': row[595:605],
                    #        'price': row[841:871],
                    #        'uom': row[605:608],
                    #        'description': row[495:595],
                    #        'total': row[871:901],
                    #        }

    def load_scheduled_folder_selected(self, cr, uid, ids, context=None):
        ''' Load all selected folder
        '''
        folder_ids = self.search(cr, uid, [
            ('autoload', '=', True),
            ], context=context)

        return self.load_file_in_folder(cr, uid, folder_ids, context=context)

    _columns = {
        'autoload': fields.boolean('Auto load',
            help='Schedule for auto load operation'),
        'name': fields.char('Name', size=25, required=True),
        'path': fields.char('Path', size=180, required=False),
        'note': fields.text('Note'),
        }

class EdiOrderFile(orm.Model):
    """ Model name: Edi Order File
    """

    _name = 'edi.order.file'
    _description = 'EDI order file'
    _rec_name = 'name'

    def _get_fullname_file_name(
            self, cr, uid, ids, fields, args, context=None):
        ''' Fields function for calculate
        '''
        res = {}
        for item in self.browse(cr, uid, ids, context=context):
            res[item.id] = os.path.join(item.folder_id.path, item.name)
        return res

    _columns = {
        'order_id': fields.many2one('edi.order', 'Order'),
        'folder_id': fields.many2one('edi.order.folder', 'History folder'),
        'name': fields.char('Name', size=40, required=True, readonly=True),
        'fullname': fields.function(
            _get_fullname_file_name, method=True,
            type='char', string='Fullname', size=200, store=False),
        'last': fields.boolean('Last'),
        'datetime': fields.date('Datetime', required=False, readonly=True,
            help='Datetime form customer EDI program (not file datetime)'),
        'mode': fields.selection([
            ('create', 'Create (ELIORD)'),
            ('urgent', 'Urgent (ELIURG)'),
            ('delete', 'Delete (ELICHG)'),
            ], 'Mode', readonly=False),
        }

    _defaults = {
        'mode': lambda *x: 'create',
        }

class EdiOrderLine(orm.Model):
    """ Model name: Edi Invoice Line
    """

    _name = 'edi.order.line'
    _description = 'EDI line'
    _rec_name = 'name'

    _columns = {
        'order_id': fields.many2one('edi.order', 'Order'),
        'sequence': fields.char('Seq.',
            size=10, readonly=True),
        'name': fields.char('Company code', size=16,
            required=True, readonly=True),
        'article': fields.char('Customer code', size=16,
            required=True, readonly=True),
        'uom': fields.char('UOM', size=5,
            required=True, readonly=True),
        'qty': fields.float('Q.ty', digits=(16, 3),
            required=True, readonly=True),
        'price': fields.float('Price', digits=(16, 3),
            required=True, readonly=True),
        'description': fields.char('Description', size=16,
            required=True, readonly=True),
        'total': fields.float('Subtotal', digits=(16, 3),
            required=True, readonly=True),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _inherit = 'edi.order'

    _columns = {
        'file_ids': fields.one2many('edi.order.file', 'order_id',  'Files'),
        'line_ids': fields.one2many('edi.order.line', 'order_id',  'Line'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>import folder daa<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class EdiInvoice(orm.Model):
    """ Model name: Edi Invoice
    """

    _name = 'edi.invoice'
    _description = 'EDI invoice'
    _rec_name = 'name'
    _order = 'name'

    _columns = {
        'name': fields.char('Number', size=20, required=True),
        'date': fields.date('Invoice date', required=True),
        }

class EdiInvoiceLine(orm.Model):
    """ Model name: Edi Invoice Line
    """

    _name = 'edi.invoice.line'
    _description = 'EDI invoice line'
    _rec_name = 'name'
    _order = 'invoice_id,name'

    def import_invoice_line_from_account(self, cr, uid, context=None):
        ''' Import procedure for get all invoice line for check
            Export with sprix: spx780
        '''
        # Pool used:
        invoice_pool = self.pool.get('edi.invoice')

        filename = '~/etl/edi/elior/controllo/fatture.txt'

        # ---------------------------------------------------------------------
        # Clean previous elements:
        # ---------------------------------------------------------------------
        # Details:
        line_ids = self.search(cr, uid, [], context=context)
        self.unlink(cr, uid, line_ids, context=context)

        # Invoice:
        invoice_ids = invoice_pool.search(cr, uid, [], context=context)
        invoice_pool.unlink(cr, uid, invoice_ids, context=context)

        invoice_db = {}
        for row in open(filename, 'r'):
            # -----------------------------------------------------------------
            # Header data:
            # -----------------------------------------------------------------
            invoice_number = row[:6]
            invoice_date = '%s-%s-%s' % (
                row[6:10],
                row[10:12],
                row[12:14],
                )
            invoice_number = 'FT-%s-%s' % (
                invoice_number, row[6:10], )

            if invoice_number not in invoice_db:
                invoice_db['invoice_number'] = invoice_pool.create(cr, uid, {
                    'name': invoice_number,
                    'date': invoice_date,
                    }, context=context)

            # -----------------------------------------------------------------
            # Detail data:
            # -----------------------------------------------------------------
            data = {
                'invoice_id': invoice_db.get(invoice_number, False),
                'order_sequence': int(row[16:20]),
                'name': row[20:36],
                'article': row[36:47],
                'qty': row[47:62],
                'price': row[62:77],
                'uom': row[77:92],
                'description': row[92:127],
                'ddt_number': row[127:133],
                'ddt_date': '%s-%s-%s' % (
                    row[133:137],
                    row[137:139],
                    row[139:141],
                    ),
                'order_number': row[143:159],
                'order_date': '%s-%s-%s' % (
                    row[159:163],
                    row[163:165],
                    row[165:167],
                    ),
                }

            self.create(cr, uid, data, context=context)
        return True

    _columns = {
        'invoice_id': fields.many2one('edi.invoice', 'Invoice'),
        'order_sequence': fields.integer('Order position',
            readonly=True),
        'name': fields.char('Company code', size=16,
            required=True, readonly=True),
        'article': fields.char('Customer code', size=16,
            required=True, readonly=True),
        'qty': fields.float('Q.ty', digits=(16, 3),
            required=True, readonly=True),
        'price': fields.float('Price', digits=(16, 3),
            required=True, readonly=True),
        'uom': fields.char('UOM', size=5,
            required=True, readonly=True),
        'description': fields.char('Description', size=16,
            required=True, readonly=True),
        'ddt_number': fields.char('DDT #', size=20,
            required=True, readonly=True),
        'ddt_date': fields.date('Invoice date',
            required=True, readonly=True),
        'order_number': fields.char('Company order #', size=25,
            required=True, readonly=True),
        'order_date': fields.date('Order date',
            required=True, readonly=True),
        }

class EdiInvoice(orm.Model):
    """ Model name: Edi Invoice
    """

    _inherit = 'edi.invoice'

    _columns = {
        'line_ids': fields.one2many('edi.invoice.line', 'invoice_id', 'Lines'),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _name = 'edi.order'
    _description = 'EDI order'
    _rec_name = 'name'

    _columns = {
        'name': fields.char('Number', size=25, required=True),
        'date': fields.datetime('Date', readonly=True),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _name = 'edi.order.folder'
    _description = 'EDI order folder'
    _rec_name = 'name'

    def load_file_in_folder(self, cr, uid, ids, context=None):
        """ Load file in this folder (used also for scheduled)
        """
        # Parameters:
        extension = 'ASC'

        # Pool used:
        order_pool = self.pool.get('edi.order')
        file_pool = self.pool.get('edi.order.file')

        for folder in self.browse(cr, uid, ids, context=context):
            # Delete all previouse file in folder:
            file_ids = file_pool.search(cr, uid, [
                ('folder_id', '=', folder.id),
                ], context=context)
            file_pool.unlink(cr, uid, file_ids, context=context)

            # TODO order_loaded for speed up search order operations?
            for root, dirs, files in os.walk(folder.path):
                for f in files:
                    if not f.endswith(estension):
                        _logger.warning('Jump file: %s' % f)
                        continue

                    # ---------------------------------------------------------
                    # Read file data (name and content):
                    # ---------------------------------------------------------
                    # Open file for get order number:
                    filename = os.path.join(folder.path, files)
                    f_asc = open(filename, 'r')
                    row = f_asc.readline() # XXX only one row
                    f_asc.close()

                    # Parse name:
                    mode = f[:6]

                    date = f[6:22]

                    date = '%s-%s-%s %s:%s:%s' % (
                        # Date:
                        date[:4],
                        date[4:6],
                        date[6:8],

                        # Time:
                        date[8:10],
                        date[10:12],
                        date[12:14],
                        )

                    if mode == 'ELIORD':
                        mode = 'create'
                    if mode == 'ELIURG':
                        mode = 'urgent'
                    if mode == 'ELICHG':
                        mode = 'delete'

                    # ---------------------------------------------------------
                    # Order info:
                    # ---------------------------------------------------------
                    # Create order record:
                    order = row[20:30]

                    order_date = row[30:38]
                    order_date = '%s-%s-%S' % (
                        order_date[:4],
                        order_date[4:6],
                        order_date[6:8],
                        )

                    order_ids = order_pool.search(cr, uid, [
                        ('name', '=', order),
                        # XXX no date
                        ], context=context)

                    if order_ids: # Nothing:
                        order_id = order_ids
                    else: # Create
                        order_id = order_pool.create(cr, uid, {
                            'name': order,
                            'date': order_date,
                            }, context=context)

                    # ---------------------------------------------------------
                    # Create file item (not write because deleted all):
                    # ---------------------------------------------------------
                    file_pool.create(cr, uid, {
                        'order_id': order_id,
                        'folder_id': folder.id,
                        'name': f,
                        'last': False,
                        'datetime': date,
                        'mode': mode,
                        }, context=context)

                    # Create order line:
                    #for row in open(filename, 'r'):
                    #    data = {
                    #        #'order_id':,
                    #        'order_sequence': row[20:30], # XXX errato
                    #        'name': row[320:355],
                    #        'article': row[355:390],
                    #        'qty': row[595:605],
                    #        'price': row[841:871],
                    #        'uom': row[605:608],
                    #        'description': row[495:595],
                    #        'total': row[871:901],
                    #        }

    def load_scheduled_folder_selected(self, cr, uid, ids, context=None):
        ''' Load all selected folder
        '''
        folder_ids = self.search(cr, uid, [
            ('autoload', '=', True),
            ], context=context)

        return self.load_file_in_folder(cr, uid, folder_ids, context=context)

    _columns = {
        'autoload': fields.boolean('Auto load',
            help='Schedule for auto load operation'),
        'name': fields.char('Name', size=25, required=True),
        'path': fields.char('Path', size=180, required=False),
        'note': fields.text('Note'),
        }

class EdiOrderFile(orm.Model):
    """ Model name: Edi Order File
    """

    _name = 'edi.order.file'
    _description = 'EDI order file'
    _rec_name = 'name'

    def _get_fullname_file_name(
            self, cr, uid, ids, fields, args, context=None):
        ''' Fields function for calculate
        '''
        res = {}
        for item in self.browse(cr, uid, ids, context=context):
            res[item.id] = os.path.join(item.folder_id.path, item.name)
        return res

    _columns = {
        'order_id': fields.many2one('edi.order', 'Order'),
        'folder_id': fields.many2one('edi.order.folder', 'History folder'),
        'name': fields.char('Name', size=40, required=True, readonly=True),
        'fullname': fields.function(
            _get_fullname_file_name, method=True,
            type='char', string='Fullname', size=200, store=False),
        'last': fields.boolean('Last'),
        'datetime': fields.date('Datetime', required=False, readonly=True,
            help='Datetime form customer EDI program (not file datetime)'),
        'mode': fields.selection([
            ('create', 'Create (ELIORD)'),
            ('urgent', 'Urgent (ELIURG)'),
            ('delete', 'Delete (ELICHG)'),
            ], 'Mode', readonly=False),
        }

    _defaults = {
        'mode': lambda *x: 'create',
        }

class EdiOrderLine(orm.Model):
    """ Model name: Edi Invoice Line
    """

    _name = 'edi.order.line'
    _description = 'EDI line'
    _rec_name = 'name'

    _columns = {
        'order_id': fields.many2one('edi.order', 'Order'),
        'sequence': fields.char('Seq.',
            size=10, readonly=True),
        'name': fields.char('Company code', size=16,
            required=True, readonly=True),
        'article': fields.char('Customer code', size=16,
            required=True, readonly=True),
        'uom': fields.char('UOM', size=5,
            required=True, readonly=True),
        'qty': fields.float('Q.ty', digits=(16, 3),
            required=True, readonly=True),
        'price': fields.float('Price', digits=(16, 3),
            required=True, readonly=True),
        'description': fields.char('Description', size=16,
            required=True, readonly=True),
        'total': fields.float('Subtotal', digits=(16, 3),
            required=True, readonly=True),
        }

class EdiOrder(orm.Model):
    """ Model name: Edi Order
    """

    _inherit = 'edi.order'

    _columns = {
        'file_ids': fields.one2many('edi.order.file', 'order_id',  'Files'),
        'line_ids': fields.one2many('edi.order.line', 'order_id',  'Line'),
        }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core calculator functionality for computing stochastic event sets and ground
motion fields using the 'event-based' method.

Stochastic events sets (which can be thought of as collections of ruptures) are
computed iven a set of seismic sources and investigation time span (in years).

For more information on computing stochastic event sets, see
:mod:`openquake.hazardlib.calc.stochastic`.

One can optionally compute a ground motion field (GMF) given a rupture, a site
collection (which is a collection of geographical points with associated soil
parameters), and a ground shaking intensity model (GSIM).

For more information on computing ground motion fields, see
:mod:`openquake.hazardlib.calc.gmf`.
"""

import random
import collections

import numpy.random

from django.db import transaction
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc import gmf
from openquake.hazardlib.imt import from_string

from openquake.engine import writer
from openquake.engine.calculators.hazard import general
from openquake.engine.calculators.hazard.classical import (
    post_processing as cls_post_proc)
from openquake.engine.calculators.hazard.event_based import post_processing
from openquake.engine.db import models
from openquake.engine.utils import tasks
from openquake.engine.performance import EnginePerformanceMonitor, LightMonitor

#: Always 1 for the computation of ground motion fields in the event-based
#: hazard calculator.
DEFAULT_GMF_REALIZATIONS = 1

# NB: beware of large caches
inserter = writer.CacheInserter(models.GmfData, 1000)

@tasks.oqtask
def compute_ses_and_gmfs(job_id, src_seeds, gsims_by_rlz, task_no):
    """
    Celery task for the stochastic event set calculator.

    Samples logic trees and calls the stochastic event set calculator.

    Once stochastic event sets are calculated, results will be saved to the
    database. See :class:`openquake.engine.db.models.SESCollection`.

    Optionally (specified in the job configuration using the
    `ground_motion_fields` parameter), GMFs can be computed from each rupture
    in each stochastic event set. GMFs are also saved to the database.

    :param int job_id:
        ID of the currently running job.
    :param src_seeds:
        List of pairs (source, seed)
    :params gsims_by_rlz:
        dictionary of GSIM
    :param task_no:
        an ordinal so that GMV can be collected in a reproducible order
    """
    rlz_ids = [r.id for r in gsims_by_rlz]
    ses_coll = models.SESCollection.objects.get(lt_realization_ids=rlz_ids)

    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    all_ses = models.SES.objects.filter(ses_collection=ses_coll)
    imts = map(from_string, hc.intensity_measure_types)
    params = dict(
        correl_model=general.get_correl_model(hc),
        truncation_level=hc.truncation_level,
        maximum_distance=hc.maximum_distance)

    gmvs_per_site = collections.defaultdict(list)
    ruptures_per_site = collections.defaultdict(list)

    mon1 = LightMonitor('filtering sites', job_id, compute_ses_and_gmfs)
    mon2 = LightMonitor('generating ruptures', job_id, compute_ses_and_gmfs)
    mon3 = LightMonitor('sampling ruptures', job_id, compute_ses_and_gmfs)
    mon4 = LightMonitor('saving ruptures', job_id, compute_ses_and_gmfs)
    mon5 = LightMonitor('generating gmfs', job_id, compute_ses_and_gmfs)

    # Compute and save stochastic event sets
    rnd = random.Random()
    for src, seed in src_seeds:
        rnd.seed(seed)

        ruptures = []
        ses_ruptures = []
        rupture_seeds = []

        with mon1:
            s_sites = src.filter_sites_by_distance_to_source(
                hc.maximum_distance, hc.site_collection
            ) if hc.maximum_distance else hc.site_collection
            if s_sites is None:
                continue

        with mon2:
            rupts = []
            for r in src.iter_ruptures():
                r_sites = r.source_typology.\
                    filter_sites_by_distance_to_rupture(
                        r, hc.maximum_distance, s_sites
                    ) if hc.maximum_distance else s_sites
                if r_sites is not None:
                    rupts.append(r)
            if not rupts:
                continue

        for ses in all_ses:
            numpy.random.seed(rnd.randint(0, models.MAX_SINT_32))
            for i, r in enumerate(rupts):
                with mon3:
                    for j in xrange(r.sample_number_of_occurrences()):
                        rup = models.SESRupture(
                            ses=ses,
                            rupture=r,
                            tag='smlt=%02d|ses=%04d|src=%s|i=%04d-%02d' % (
                                ses_coll.ordinal,
                                ses.ordinal, src.source_id, i, j),
                            hypocenter=r.hypocenter.wkt2d,
                            magnitude=r.mag,
                        )
                        rup_seed = rnd.randint(0, models.MAX_SINT_32)
                        ruptures.append(r)
                        ses_ruptures.append(rup)
                        rupture_seeds.append(rup_seed)

        if not ruptures:
            continue

        with mon4:
            rupture_ids = _save_ses_ruptures(ses_ruptures)

        if not hc.ground_motion_fields:
            continue

        for rlz, gsims in gsims_by_rlz.items():
            with mon5:
                for imt, site_id, gmv, rup_id in _compute_gmf(
                        params, imts, gsims, s_sites,
                        zip(ruptures, rupture_ids, rupture_seeds)):
                    gmvs_per_site[rlz, imt, site_id].append(gmv)
                    ruptures_per_site[rlz, imt, site_id].append(rup_id)

    mon1.flush()
    mon2.flush()
    mon3.flush()
    mon4.flush()
    mon5.flush()

    if not hc.ground_motion_fields:
        return

    for rlz in gsims_by_rlz:
        with EnginePerformanceMonitor(
                'saving gmfs', job_id, compute_ses_and_gmfs):
            _save_gmfs(gmvs_per_site, ruptures_per_site,
                       hc.site_collection, task_no)

def _save_ses_ruptures(ruptures):
    """
    Helper function for saving stochastic event set ruptures to the database.
    :param ruptures:
        A list of :class:`openquake.engine.db.models.SESRupture` instances.
    """
    # TODO: Possible future optimization:
    # Refactor this to do bulk insertion of ruptures
    rupture_ids = []
    with transaction.commit_on_success(using='job_init'):
        for r in ruptures:
            r.save()
            rupture_ids.append(r.id)
    return rupture_ids

# NB: I tried to return a single dictionary {site_id: [(gmv, rupt_id),...]}
# but it takes a lot more memory (MS)
def _compute_gmf(params, imts, gsims, site_coll, rupture_id_seed_triples):
    """
    Compute a ground motion field value for each rupture, for all the
    points affected by that rupture, for the given IMT. Returns a
    dictionary with the nonzero contributions to each site id, and a dictionary
    with the ids of the contributing ruptures for each site id.
    assert len(ruptures) == len(rupture_seeds)

    :param params:
        a dictionary containing the keys
        correl_model, truncation_level, maximum_distance
    :param imts:
        a list of hazardlib intensity measure types
    :param gsims:
        a dictionary {tectonic region type -> GSIM instance}
    :param site_coll:
        a SiteCollection instance
    :param rupture_id_seed_triple:
        a list of triples with types
        (:class:`openquake.hazardlib.source.rupture.Rupture`, int, int)
    """
    # Compute and save ground motion fields
    for i, (rupture, rup_id, rup_seed) in enumerate(rupture_id_seed_triples):
        gmf_calc_kwargs = {
            'rupture': rupture,
            'sites': site_coll,
            'imts': imts,
            'gsim': gsims[rupture.tectonic_region_type],
            'truncation_level': params['truncation_level'],
            'realizations': DEFAULT_GMF_REALIZATIONS,
            'correlation_model': params['correl_model'],
        }
        numpy.random.seed(rup_seed)
        gmf_dict = gmf.ground_motion_fields(**gmf_calc_kwargs)
        for imt, gmf_1_realiz in gmf_dict.iteritems():
            # since DEFAULT_GMF_REALIZATIONS is 1, gmf_1_realiz is a matrix
            # with n_sites rows and 1 column
            for site, gmv in zip(site_coll, gmf_1_realiz):
                gmv = float(gmv)  # convert a 1x1 matrix into a float
                if gmv:  # nonzero contribution to site
                    yield imt, site.id, gmv, rup_id

@transaction.commit_on_success(using='job_init')
def _save_gmfs(gmvs_per_site, ruptures_per_site, sites, task_no):
    """
    Helper method to save computed GMF data to the database.
    :param gmf_per_site:
        The GMFs per rupture
    :param rupture_per_site:
        The associated rupture ids
    :param sites:
        An :class:`openquake.hazardlib.site.SiteCollection` object,
        representing the sites of interest for a calculation.
    :param task_no:
        The ordinal of the task which generated the current GMFs to save
    """
    for rlz, imt, site_id in gmvs_per_site:
        gmf = models.Gmf.objects.get(lt_realization=rlz)
        imt_name, sa_period, sa_damping = imt
        inserter.add(models.GmfData(
            gmf=gmf,
            task_no=task_no,
            imt=imt_name,
            sa_period=sa_period,
            sa_damping=sa_damping,
            site_id=site_id,
            gmvs=gmvs_per_site[rlz, imt, site_id],
            rupture_ids=ruptures_per_site[rlz, imt, site_id]))
    inserter.flush()

class EventBasedHazardCalculator(general.BaseHazardCalculator):
    """
    Probabilistic Event-Based hazard calculator. Computes stochastic event sets
    and (optionally) ground motion fields.
    """
    core_calc_task = compute_ses_and_gmfs

    def task_arg_gen(self, _block_size=None):
        """
        Loop through realizations and sources to generate a sequence of
        task arg tuples. Each tuple of args applies to a single task.
        Yielded results are tuples of the form job_id, sources, ses, seeds
        (seeds will be used to seed numpy for temporal occurence sampling).
        """
        hc = self.hc
        rnd = random.Random()
        rnd.seed(hc.random_seed)
        task_no = 0
        for job_id, block, gsims_by_rlz in super(
                EventBasedHazardCalculator, self).task_arg_gen():
            ss = [(src, rnd.randint(0, models.MAX_SINT_32))
                  for src in block]  # source, seed pairs
            yield job_id, ss, gsims_by_rlz, task_no
            task_no += 1

        # now the source_blocks_per_ltpath dictionary can be cleared
        self.source_blocks_per_ltpath.clear()

    def initialize_ses_db_records(self, ordinal, rlzs):
        """
        Create :class:`~openquake.engine.db.models.Output`,
        :class:`~openquake.engine.db.models.SESCollection` and
        :class:`~openquake.engine.db.models.SES` "container" records for
        a single realization.

        Stochastic event set ruptures computed for this realization will be
        associated to these containers.

        NOTE: Many tasks can contribute ruptures to the same SES.
        """
        rlz_ids = [r.id for r in rlzs]

        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection smlt-%d-rlz-%s' % (
                ordinal, ','.join(map(str, rlz_ids))),
            output_type='ses')

        ses_coll = models.SESCollection.objects.create(
            output=output, lt_realization_ids=rlz_ids, ordinal=ordinal)

        for rlz in rlzs:
            if self.job.hazard_calculation.ground_motion_fields:
                output = models.Output.objects.create(
                    oq_job=self.job,
                    display_name='GMF rlz-%s' % rlz.id,
                    output_type='gmf')
                models.Gmf.objects.create(output=output, lt_realization=rlz)

        all_ses = []
        for i in xrange(1, self.hc.ses_per_logic_tree_path + 1):
            all_ses.append(
                models.SES.objects.create(
                    ses_collection=ses_coll,
                    investigation_time=self.hc.investigation_time,
                    ordinal=i))
        return all_ses

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        super(EventBasedHazardCalculator, self).pre_execute()
        for i, rlzs in enumerate(self.rlzs_per_ltpath.itervalues()):
            self.initialize_ses_db_records(i, rlzs)

    def post_process(self):
        """
        If requested, perform additional processing of GMFs to produce hazard
        curves.
        """
        if self.hc.hazard_curves_from_gmfs:
            with EnginePerformanceMonitor('generating hazard curves',
                                          self.job.id):
                self.parallelize(
                    post_processing.gmf_to_hazard_curve_task,
                    post_processing.gmf_to_hazard_curve_arg_gen(self.job),
                    self.log_percent)

            # If `mean_hazard_curves` is True and/or `quantile_hazard_curves`
            # has some value (not an empty list), do this additional
            # post-processing.
            if self.hc.mean_hazard_curves or self.hc.quantile_hazard_curves:
                with EnginePerformanceMonitor(
                        'generating mean/quantile curves', self.job.id):
                    self.do_aggregate_post_proc()

            if self.hc.hazard_maps:
                with EnginePerformanceMonitor(
                        'generating hazard maps', self.job.id):
                    self.parallelize(
                        cls_post_proc.hazard_curves_to_hazard_map_task,
                        cls_post_proc.hazard_curves_to_hazard_map_task_arg_gen(
                            self.job),
                        self.log_percent)
<commit_msg>Fixed s_sites<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core calculator functionality for computing stochastic event sets and ground
motion fields using the 'event-based' method.

Stochastic events sets (which can be thought of as collections of ruptures) are
computed iven a set of seismic sources and investigation time span (in years).

For more information on computing stochastic event sets, see
:mod:`openquake.hazardlib.calc.stochastic`.

One can optionally compute a ground motion field (GMF) given a rupture, a site
collection (which is a collection of geographical points with associated soil
parameters), and a ground shaking intensity model (GSIM).

For more information on computing ground motion fields, see
:mod:`openquake.hazardlib.calc.gmf`.
"""

import random
import collections

import numpy.random

from django.db import transaction
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc import gmf
from openquake.hazardlib.imt import from_string

from openquake.engine import writer
from openquake.engine.calculators.hazard import general
from openquake.engine.calculators.hazard.classical import (
    post_processing as cls_post_proc)
from openquake.engine.calculators.hazard.event_based import post_processing
from openquake.engine.db import models
from openquake.engine.utils import tasks
from openquake.engine.performance import EnginePerformanceMonitor, LightMonitor

#: Always 1 for the computation of ground motion fields in the event-based
#: hazard calculator.
DEFAULT_GMF_REALIZATIONS = 1

# NB: beware of large caches
inserter = writer.CacheInserter(models.GmfData, 1000)

@tasks.oqtask
def compute_ses_and_gmfs(job_id, src_seeds, gsims_by_rlz, task_no):
    """
    Celery task for the stochastic event set calculator.

    Samples logic trees and calls the stochastic event set calculator.

    Once stochastic event sets are calculated, results will be saved to the
    database. See :class:`openquake.engine.db.models.SESCollection`.

    Optionally (specified in the job configuration using the
    `ground_motion_fields` parameter), GMFs can be computed from each rupture
    in each stochastic event set. GMFs are also saved to the database.

    :param int job_id:
        ID of the currently running job.
    :param src_seeds:
        List of pairs (source, seed)
    :params gsims_by_rlz:
        dictionary of GSIM
    :param task_no:
        an ordinal so that GMV can be collected in a reproducible order
    """
    rlz_ids = [r.id for r in gsims_by_rlz]
    ses_coll = models.SESCollection.objects.get(lt_realization_ids=rlz_ids)

    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    all_ses = models.SES.objects.filter(ses_collection=ses_coll)
    imts = map(from_string, hc.intensity_measure_types)
    params = dict(
        correl_model=general.get_correl_model(hc),
        truncation_level=hc.truncation_level,
        maximum_distance=hc.maximum_distance)

    gmvs_per_site = collections.defaultdict(list)
    ruptures_per_site = collections.defaultdict(list)

    mon1 = LightMonitor('filtering sites', job_id, compute_ses_and_gmfs)
    mon2 = LightMonitor('generating ruptures', job_id, compute_ses_and_gmfs)
    mon3 = LightMonitor('sampling ruptures', job_id, compute_ses_and_gmfs)
    mon4 = LightMonitor('saving ruptures', job_id, compute_ses_and_gmfs)
    mon5 = LightMonitor('generating gmfs', job_id, compute_ses_and_gmfs)

    # Compute and save stochastic event sets
    rnd = random.Random()
    for src, seed in src_seeds:
        rnd.seed(seed)

        ruptures = []
        ses_ruptures = []
        rupture_seeds = []

        with mon1:
            s_sites = src.filter_sites_by_distance_to_source(
                hc.maximum_distance, hc.site_collection
            ) if hc.maximum_distance else hc.site_collection
            if s_sites is None:
                continue

        with mon2:
            rupts = []
            for r in src.iter_ruptures():
                r_sites = r.source_typology.\
                    filter_sites_by_distance_to_rupture(
                        r, hc.maximum_distance, s_sites
                    ) if hc.maximum_distance else s_sites
                if r_sites is not None:
                    rupts.append(r)
            if not rupts:
                continue

        for ses in all_ses:
            numpy.random.seed(rnd.randint(0, models.MAX_SINT_32))
            for i, r in enumerate(rupts):
                with mon3:
                    for j in xrange(r.sample_number_of_occurrences()):
                        rup = models.SESRupture(
                            ses=ses,
                            rupture=r,
                            tag='smlt=%02d|ses=%04d|src=%s|i=%04d-%02d' % (
                                ses_coll.ordinal,
                                ses.ordinal, src.source_id, i, j),
                            hypocenter=r.hypocenter.wkt2d,
                            magnitude=r.mag,
                        )
                        rup_seed = rnd.randint(0, models.MAX_SINT_32)
                        ruptures.append(r)
                        ses_ruptures.append(rup)
                        rupture_seeds.append(rup_seed)

        if not ruptures:
            continue

        with mon4:
            rupture_ids = _save_ses_ruptures(ses_ruptures)

        if not hc.ground_motion_fields:
            continue

        for rlz, gsims in gsims_by_rlz.items():
            with mon5:
                for imt, site_id, gmv, rup_id in _compute_gmf(
                        params, imts, gsims, hc.site_collection,
                        zip(ruptures, rupture_ids, rupture_seeds)):
                    gmvs_per_site[rlz, imt, site_id].append(gmv)
                    ruptures_per_site[rlz, imt, site_id].append(rup_id)

    mon1.flush()
    mon2.flush()
    mon3.flush()
    mon4.flush()
    mon5.flush()

    if not hc.ground_motion_fields:
        return

    for rlz in gsims_by_rlz:
        with EnginePerformanceMonitor(
                'saving gmfs', job_id, compute_ses_and_gmfs):
            _save_gmfs(gmvs_per_site, ruptures_per_site,
                       hc.site_collection, task_no)

def _save_ses_ruptures(ruptures):
    """
    Helper function for saving stochastic event set ruptures to the database.
    :param ruptures:
        A list of :class:`openquake.engine.db.models.SESRupture` instances.
    """
    # TODO: Possible future optimization:
    # Refactor this to do bulk insertion of ruptures
    rupture_ids = []
    with transaction.commit_on_success(using='job_init'):
        for r in ruptures:
            r.save()
            rupture_ids.append(r.id)
    return rupture_ids

# NB: I tried to return a single dictionary {site_id: [(gmv, rupt_id),...]}
# but it takes a lot more memory (MS)
def _compute_gmf(params, imts, gsims, site_coll, rupture_id_seed_triples):
    """
    Compute a ground motion field value for each rupture, for all the
    points affected by that rupture, for the given IMT. Returns a
    dictionary with the nonzero contributions to each site id, and a dictionary
    with the ids of the contributing ruptures for each site id.
    assert len(ruptures) == len(rupture_seeds)

    :param params:
        a dictionary containing the keys
        correl_model, truncation_level, maximum_distance
    :param imts:
        a list of hazardlib intensity measure types
    :param gsims:
        a dictionary {tectonic region type -> GSIM instance}
    :param site_coll:
        a SiteCollection instance
    :param rupture_id_seed_triple:
        a list of triples with types
        (:class:`openquake.hazardlib.source.rupture.Rupture`, int, int)
    """
    # Compute and save ground motion fields
    for i, (rupture, rup_id, rup_seed) in enumerate(rupture_id_seed_triples):
        gmf_calc_kwargs = {
            'rupture': rupture,
            'sites': site_coll,
            'imts': imts,
            'gsim': gsims[rupture.tectonic_region_type],
            'truncation_level': params['truncation_level'],
            'realizations': DEFAULT_GMF_REALIZATIONS,
            'correlation_model': params['correl_model'],
        }
        numpy.random.seed(rup_seed)
        gmf_dict = gmf.ground_motion_fields(**gmf_calc_kwargs)
        for imt, gmf_1_realiz in gmf_dict.iteritems():
            # since DEFAULT_GMF_REALIZATIONS is 1, gmf_1_realiz is a matrix
            # with n_sites rows and 1 column
            for site, gmv in zip(site_coll, gmf_1_realiz):
                gmv = float(gmv)  # convert a 1x1 matrix into a float
                if gmv:  # nonzero contribution to site
                    yield imt, site.id, gmv, rup_id

@transaction.commit_on_success(using='job_init')
def _save_gmfs(gmvs_per_site, ruptures_per_site, sites, task_no):
    """
    Helper method to save computed GMF data to the database.
    :param gmf_per_site:
        The GMFs per rupture
    :param rupture_per_site:
        The associated rupture ids
    :param sites:
        An :class:`openquake.hazardlib.site.SiteCollection` object,
        representing the sites of interest for a calculation.
    :param task_no:
        The ordinal of the task which generated the current GMFs to save
    """
    for rlz, imt, site_id in gmvs_per_site:
        gmf = models.Gmf.objects.get(lt_realization=rlz)
        imt_name, sa_period, sa_damping = imt
        inserter.add(models.GmfData(
            gmf=gmf,
            task_no=task_no,
            imt=imt_name,
            sa_period=sa_period,
            sa_damping=sa_damping,
            site_id=site_id,
            gmvs=gmvs_per_site[rlz, imt, site_id],
            rupture_ids=ruptures_per_site[rlz, imt, site_id]))
    inserter.flush()

class EventBasedHazardCalculator(general.BaseHazardCalculator):
    """
    Probabilistic Event-Based hazard calculator. Computes stochastic event sets
    and (optionally) ground motion fields.
    """
    core_calc_task = compute_ses_and_gmfs

    def task_arg_gen(self, _block_size=None):
        """
        Loop through realizations and sources to generate a sequence of
        task arg tuples. Each tuple of args applies to a single task.
        Yielded results are tuples of the form job_id, sources, ses, seeds
        (seeds will be used to seed numpy for temporal occurence sampling).
        """
        hc = self.hc
        rnd = random.Random()
        rnd.seed(hc.random_seed)
        task_no = 0
        for job_id, block, gsims_by_rlz in super(
                EventBasedHazardCalculator, self).task_arg_gen():
            ss = [(src, rnd.randint(0, models.MAX_SINT_32))
                  for src in block]  # source, seed pairs
            yield job_id, ss, gsims_by_rlz, task_no
            task_no += 1

        # now the source_blocks_per_ltpath dictionary can be cleared
        self.source_blocks_per_ltpath.clear()

    def initialize_ses_db_records(self, ordinal, rlzs):
        """
        Create :class:`~openquake.engine.db.models.Output`,
        :class:`~openquake.engine.db.models.SESCollection` and
        :class:`~openquake.engine.db.models.SES` "container" records for
        a single realization.

        Stochastic event set ruptures computed for this realization will be
        associated to these containers.

        NOTE: Many tasks can contribute ruptures to the same SES.
        """
        rlz_ids = [r.id for r in rlzs]

        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection smlt-%d-rlz-%s' % (
                ordinal, ','.join(map(str, rlz_ids))),
            output_type='ses')

        ses_coll = models.SESCollection.objects.create(
            output=output, lt_realization_ids=rlz_ids, ordinal=ordinal)

        for rlz in rlzs:
            if self.job.hazard_calculation.ground_motion_fields:
                output = models.Output.objects.create(
                    oq_job=self.job,
                    display_name='GMF rlz-%s' % rlz.id,
                    output_type='gmf')
                models.Gmf.objects.create(output=output, lt_realization=rlz)

        all_ses = []
        for i in xrange(1, self.hc.ses_per_logic_tree_path + 1):
            all_ses.append(
                models.SES.objects.create(
                    ses_collection=ses_coll,
                    investigation_time=self.hc.investigation_time,
                    ordinal=i))
        return all_ses

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        super(EventBasedHazardCalculator, self).pre_execute()
        for i, rlzs in enumerate(self.rlzs_per_ltpath.itervalues()):
            self.initialize_ses_db_records(i, rlzs)

    def post_process(self):
        """
        If requested, perform additional processing of GMFs to produce hazard
        curves.
        """
        if self.hc.hazard_curves_from_gmfs:
            with EnginePerformanceMonitor('generating hazard curves',
                                          self.job.id):
                self.parallelize(
                    post_processing.gmf_to_hazard_curve_task,
                    post_processing.gmf_to_hazard_curve_arg_gen(self.job),
                    self.log_percent)

            # If `mean_hazard_curves` is True and/or `quantile_hazard_curves`
            # has some value (not an empty list), do this additional
            # post-processing.
            if self.hc.mean_hazard_curves or self.hc.quantile_hazard_curves:
                with EnginePerformanceMonitor(
                        'generating mean/quantile curves', self.job.id):
                    self.do_aggregate_post_proc()

            if self.hc.hazard_maps:
                with EnginePerformanceMonitor(
                        'generating hazard maps', self.job.id):
                    self.parallelize(
                        cls_post_proc.hazard_curves_to_hazard_map_task,
                        cls_post_proc.hazard_curves_to_hazard_map_task_arg_gen(
                            self.job),
                        self.log_percent)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from decimal import Decimal
from django.db import migrations
from moneyed import Money, RUB, USD, EUR

from accountant.models import Account

def create_accounts(apps, schema_editor):
    account_model = apps.get_model('accountant', 'Account')
    sheaf_model = apps.get_model('accountant', 'Sheaf')
    wallet = account_model.objects.create(title='Кошелёк', type=Account.ACCOUNT)
    for currency in (RUB, USD, EUR):
        sheaf_model.objects.create(amount=1, currency=currency, account=wallet)

class Migration(migrations.Migration):

    dependencies = [
        ('accountant', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(
            create_accounts
        )
    ]
<commit_msg>more items in test data<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from moneyed import RUB, USD, EUR

from accountant.models import Account

def create_accounts(apps, schema_editor):
    account_model = apps.get_model('accountant', 'Account')
    sheaf_model = apps.get_model('accountant', 'Sheaf')
    wallet = account_model.objects.create(title='Кошелёк', type=Account.ACCOUNT)
    for currency in (RUB, USD, EUR):
        sheaf_model.objects.create(amount=10, currency=currency, account=wallet)

    salary = account_model.objects.create(title='Зарплата в Exante', type=Account.INCOME)

    for expence in ('Бензин', 'Хлеб', 'Колбаса'):
        account_model.objects.create(title=expence, type=Account.EXPENSE)

class Migration(migrations.Migration):

    dependencies = [
        ('accountant', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(
            create_accounts
        )
    ]
<|endoftext|>


<commit_before>import sys
import getopt
import requests
import pprint

import mapping
import settings
import utils

dryrun = False
verbose = False
quiet = False

OVH_API_URL = 'https://ws.ovh.ca/dedicated/r2/ws.dispatcher/getAvailability2'

def query_api():
    return requests.get(OVH_API_URL, timeout=30).json()

def parse_data(data):
    offers = data.get('answer').get('availability')
    servers = utils.recursive_dict()

    for offer in offers:
        ref = offer.get('reference')
        zones = offer.get('zones')
        server = mapping.OFFER_TO_SERVER_MAPPING.get(ref)

        if ref not in mapping.OFFER_TO_SERVER_MAPPING:
            continue
        if server not in settings.WATCHED_SERVER_LIST:
            continue

        for zone in zones:
            avail = zone.get('availability')
            dc = zone.get('zone')

            if dc not in settings.WATCHED_DC_LIST:
                continue

            if avail not in ['unknown', 'unavailable']:
                servers[server][dc] = avail
            else:
                servers[server][dc] = False

    return servers

def fetch_avails(servers, previous_state):
    avails = []

    for server, dcs in servers.items():
        for dc, stock in dcs.items():
            if stock is not False:
                already_avail = False

                try:
                    already_avail = previous_state.get(server).get(dc)
                except AttributeError:
                    pass

                if not already_avail:
                        avails.append({'server': server, 'stock': stock, 'dc': dc.upper()})

    return avails

def update_state(servers):
    state = utils.recursive_dict()

    for server, dcs in servers.items():
        for dc, time in dcs.items():
            state[server][dc] = (time is not False)

    return state

def check():
    if not quiet:
        print('Running check.py in dry run (not sending any SMS)')

    data = query_api()
    # data = load_state(filename='data.pickle')

    servers = parse_data(data)
    if verbose:
        pprint.pprint(servers)

    previous_state = utils.load_state()
    if verbose:
        pprint.pprint(previous_state)

    avails = fetch_avails(servers, previous_state)
    if verbose:
        pprint.pprint(avails)

    for avail in avails:
        message = '{} is now available in {} at {}'.format(avail['server'], avail['stock'], avail['dc'])
        if not quiet:
            print(message)
        if not dryrun:
            utils.send_sms('OVH-Availability: {}'.format(message))

    state = update_state(servers)
    if verbose:
        pprint.pprint(state)

    utils.save_state(state)
    # utils.save_state(data, filename='data.pickle')

def main():
    opts, args = getopt.getopt(sys.argv[1:], 'n:v:q', ['dry-run', 'verbose', 'quiet'])

    global dryrun, verbose, quiet

    for opt in opts:
        if opt[0] in ('-n', '--dry-run'):
            dryrun = True
        elif opt[0] in ('-v', '--verbose'):
            verbose = True
        elif opt[0] in ('-q', '--quiet'):
            quiet = True

    check()

if __name__ == '__main__':
    main()
<commit_msg>Added option to display sold-out offers.<commit_after>import sys
import getopt
import requests
import pprint

import mapping
import settings
import utils

sold_out = False
dry_run = False
verbose = False
quiet = False

OVH_API_URL = 'https://ws.ovh.ca/dedicated/r2/ws.dispatcher/getAvailability2'

def query_api():
    return requests.get(OVH_API_URL, timeout=30).json()

def parse_data(data):
    offers = data.get('answer').get('availability')
    servers = utils.recursive_dict()

    for offer in offers:
        ref = offer.get('reference')
        zones = offer.get('zones')
        server = mapping.OFFER_TO_SERVER_MAPPING.get(ref)

        if ref not in mapping.OFFER_TO_SERVER_MAPPING:
            continue
        if server not in settings.WATCHED_SERVER_LIST:
            continue

        for zone in zones:
            avail = zone.get('availability')
            dc = zone.get('zone')

            if dc not in settings.WATCHED_DC_LIST:
                continue

            if avail not in ['unknown', 'unavailable']:
                servers[server][dc] = avail
            else:
                servers[server][dc] = False

    return servers

def fetch_available(servers, previous_state):
    offers = []

    for server, dcs in servers.items():
        for dc, stock in dcs.items():
            if stock is not False:
                already_avail = False

                try:
                    already_avail = previous_state.get(server).get(dc)
                except AttributeError:
                    pass

                if not already_avail:
                    offers.append({'server': server, 'stock': stock, 'dc': dc.upper()})

    return offers

def fetch_sold_out(servers, previous_state):
    offers = []

    for server, dcs in servers.items():
        for dc, stock in dcs.items():
            if stock is False:
                already_avail = False

                try:
                    already_avail = previous_state.get(server).get(dc)
                except AttributeError:
                    pass

                if already_avail:
                    offers.append({'server': server, 'stock': stock, 'dc': dc.upper()})

    return offers

def update_state(servers):
    state = utils.recursive_dict()

    for server, dcs in servers.items():
        for dc, time in dcs.items():
            state[server][dc] = (time is not False)

    return state

def check():
    if not quiet:
        print('Running check.py in dry run (not sending any SMS)')

    data = query_api()
    # data = load_state(filename='data.pickle')

    servers = parse_data(data)
    if verbose:
        pprint.pprint(servers)

    previous_state = utils.load_state()
    if verbose:
        pprint.pprint(previous_state)

    offers = fetch_available(servers, previous_state)
    if verbose:
        pprint.pprint(offers)

    for offer in offers:
        message = '{} is now available in {} at {}'.format(offer['server'], offer['stock'], offer['dc'])
        if not quiet:
            print(message)
        if not dry_run:
            utils.send_sms('OVH-Availability: {}'.format(message))

    if sold_out:
        offers = fetch_sold_out(servers, previous_state)
        if verbose:
            pprint.pprint(offers)

        for offer in offers:
            message = '{} is no longer available at {}'.format(offer['server'], offer['dc'])
            if not quiet:
                print(message)
            if not dry_run:
                utils.send_sms('OVH-Availability: {}'.format(message))

    state = update_state(servers)
    if verbose:
        pprint.pprint(state)

    utils.save_state(state)
    # utils.save_state(data, filename='data.pickle')

def main():
    opts, args = getopt.getopt(sys.argv[1:], 's:n:v:q', ['sold-out', 'dry-run', 'verbose', 'quiet'])

    global sold_out, dry_run, verbose, quiet

    for opt in opts:
        if opt[0] in ('-s', '--sold-out'):
            sold_out = True
        elif opt[0] in ('-n', '--dry-run'):
            dry_run = True
        elif opt[0] in ('-v', '--verbose'):
            verbose = True
        elif opt[0] in ('-q', '--quiet'):
            quiet = True

    check()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (C) 2009  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Brazilian Localization Sale",
    "category": "Localisation",
    "license": "AGPL-3",
    "author":
        'Akretion, '
        'Odoo Community Association (OCA)',
    "website": "http://github.com/OCA/l10n-brazil",
    "version": "12.0.5.3.0",
    "depends": ["sale_management", "l10n_br_account"],
    "data": [
        # Data
        "data/company.xml",

        # Security
        "security/ir.model.access.csv",
        "security/l10n_br_sale_security.xml",

        # View
        "views/res_config_settings_view.xml",
        "views/res_company_view.xml",
        "views/sale_view.xml",

        # Report
        "report/sale_report_view.xml",
        "report/sale_report_templates.xml",
    ],
    "demo": [
        # Demo
        "demo/product.xml",
        "demo/company.xml",
        "demo/l10n_br_sale.xml",
    ],
    "installable": True,
    "auto_install": True,
    "post_init_hook": "post_init_hook",
    "development_status": "Production/Stable",
    "maintainers": ["renatonlima"],
}
<commit_msg>l10n_br_sale 12.0.5.4.0<commit_after># Copyright (C) 2009  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Brazilian Localization Sale",
    "category": "Localisation",
    "license": "AGPL-3",
    "author":
        'Akretion, '
        'Odoo Community Association (OCA)',
    "website": "http://github.com/OCA/l10n-brazil",
    "version": "12.0.5.4.0",
    "depends": ["sale_management", "l10n_br_account"],
    "data": [
        # Data
        "data/company.xml",

        # Security
        "security/ir.model.access.csv",
        "security/l10n_br_sale_security.xml",

        # View
        "views/res_config_settings_view.xml",
        "views/res_company_view.xml",
        "views/sale_view.xml",

        # Report
        "report/sale_report_view.xml",
        "report/sale_report_templates.xml",
    ],
    "demo": [
        # Demo
        "demo/product.xml",
        "demo/company.xml",
        "demo/l10n_br_sale.xml",
    ],
    "installable": True,
    "auto_install": True,
    "post_init_hook": "post_init_hook",
    "development_status": "Production/Stable",
    "maintainers": ["renatonlima"],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# ----------------------------------------------------------------------
#
# Brad T. Aagaard, U.S. Geological Survey
# Charles A. Williams, GNS Science
# Matthew G. Knepley, University of Chicago
#
# This code was developed as part of the Computational Infrastructure
# for Geodynamics (http://geodynamics.org).
#
# Copyright (c) 2010-2016 University of California, Davis
#
# See COPYING for license information.
#
# ----------------------------------------------------------------------
#
# Create PyLith binary package.
#
# Run from the top-level PyLith build dir.
#
# Prerequisite: Copy setup_PLATFORM.sh file to PREFIX.
#
# Usage: make_package.py

import os, sys, platform
from popen2 import Popen4
from glob import glob
from os.path import basename, dirname, isabs, isdir, isfile, join
import shutil
from distutils.sysconfig import parse_makefile
from ConfigParser import ConfigParser

class Packaging(object):
    _name = 'packaging'
    _attrs = ['bin_dirs',
              'lib_dirs',
              'misc_dirs',
              'files',
              'strip_list',
              'exclude',
              'scripts',
              'urls',
              ]

class Config(object):

    sections = [Packaging]

    #@classmethod
    def readPackingList(cls):

        parser = ConfigParser()
        curdir = dirname(__file__)
        parser.read(join(curdir, "packinglist.cfg"))

        config = Config()

        for sectionCls in cls.sections:
            sectionDct = parser._sections.get(sectionCls._name, None)
            section = None
            if sectionDct is not None:
                section = sectionCls()
                for attr in section._attrs:
                    value = sectionDct.get(attr, "")
                    value = value.split()
                    setattr(section, attr, value)
            setattr(config, sectionCls._name, section)

        return config
    readPackingList = classmethod(readPackingList)

def getMakeInfo():
    # NYI: This is Autotools-specific.
    makefile = parse_makefile("Makefile")
    info = {'package_name': makefile['PACKAGE_NAME'],
            'package': makefile['PACKAGE'],
            'version': makefile['VERSION'],
            'srcdir': makefile['top_srcdir'],
            'prefix': makefile['prefix'],
            'python_version': makefile['PYTHON_VERSION'],
            }
    return info

def walkDirTree(dirname):
    """
    Get list of files in tree under dirname.
    """
    fullNames = []
    for dirpath, dirs, files in os.walk(dirname):
        for f in files:
            fullNames.append(os.path.join(dirpath, f))
    return fullNames

def filterList(l, excludes):
    try:
        import re, functools, operator

        pats = [re.compile(ex) for ex in excludes]
        return [i for i in l if not functools.reduce(operator.or_, [bool(pat.match(i)) for pat in pats])]
    except (ImportError, AttributeError):
        import re, operator

        pats = [re.compile(ex) for ex in excludes]
        return [i for i in l if not reduce(operator.or_, [bool(pat.match(i)) for pat in pats])]

class PackingList(object):
    def __init__(self, config, opSys, python):

        binDirs = config.packaging.bin_dirs
        libDirs = config.packaging.lib_dirs
        miscDirs = config.packaging.misc_dirs
        if platform.machine() == "x86_64":
            libDirs.append("lib64")
        self.directories = binDirs + libDirs + miscDirs

        self.extraFiles = config.packaging.files
        self.exclude = config.packaging.exclude

        self.programs = []
        for d in binDirs:
            self.programs.extend(walkDirTree(d))
        self.programs = filterList(self.programs, self.exclude)

        self.libraries = []
        for d in libDirs:
            self.libraries.extend(walkDirTree(d))
        self.libraries = filterList(self.libraries, self.exclude)

        self.misc = []
        for d in miscDirs:
            self.misc.extend(walkDirTree(d))
        self.misc = filterList(self.misc, self.exclude)

        # KLUDGE Explicitly add include/pythonX.X/pyconfig.h if it exists.
        pythonConfigFile = "include/" + python + "/pyconfig.h"
        if isfile(pythonConfigFile):
            self.misc.append(pythonConfigFile)

        # Scripts are application specific.
        self.scripts = []
        for s in config.packaging.scripts:
            s = "bin/" + s
            self.scripts.append(s)

        # Suffix for libraries that will be stripped of symbols.
        if opSys == "linux":
            libSuffix = ".so"
        elif opSys == "darwin":
            libSuffix = ".dylib"
        elif opSys == "win":
            libSuffix = ".dll"
        else:
            sys.exit("Unknown OS: " + opSys)

        self.stripList = []
        if opSys == "win":
            for f in config.packaging.strip_list:
                fdll = f.replace("lib/lib", "bin/cyg")+"-0"+libSuffix
                if fdll in self.libraries:
                    self.stripList.append(fdll)
        else:
            for f in config.packaging.strip_list:
                flib = f+libSuffix
                if flib in self.libraries:
                    self.stripList.append(flib)

        cig = [("CIG", "cig", "http://www.geodynamics.org/")]
        self.urls = cig + tupleUp(config.packaging.urls, 3)

        return

    def addFile(self, f):
        self.misc.append(f)

    def addDirectory(self, d):
        self.directories.append(d)
        newFiles = walkDirTree(d)
        newFiles = filterList(newFiles, self.exclude)
        self.misc.extend(newFiles)

    def files(self):
        for f in self.programs:
            yield f
        for f in self.libraries:
            yield f
        for f in self.misc:
            yield f
        for f in self.extraFiles:
            yield f
        return

    def all(self):
        for d in self.directories:
            yield d
        for f in self.files():
            yield f
        return

def tupleUp(l, n):
    tuples = []
    i = iter(l)
    try:
        while True:
            t = []
            for count in xrange(n):
                t.append(i.next())
            tuples.append(tuple(t))
    except StopIteration:
        pass
    return tuples

def spawn(*argv):
    print ' '.join(argv)
    status = os.spawnvp(os.P_WAIT, argv[0], argv)
    if status != 0:
        statusMsg = "%s: %s: exit %d" % (sys.argv[0], argv[0], status)
        sys.exit(statusMsg)
    return

def ospawn(*argv):
    print ' '.join(argv)
    child = Popen4(argv)

    child.tochild.close()

    output = child.fromchild.readlines()
    status = child.wait()

    exitStatus = None
    if (os.WIFSIGNALED(status)):
        statusStr = "signal %d" % os.WTERMSIG(status)
    elif (os.WIFEXITED(status)):
        exitStatus = os.WEXITSTATUS(status)
        statusStr = "exit %d" % exitStatus
    else:
        statusStr = "status %d" % status
    if exitStatus != 0:
        sys.exit("%s: %s: %s" % (sys.argv[0], argv[0], statusStr))

    return output

def getGitInfo(srcdir):
    workdir = os.getcwd()
    os.chdir(srcdir)
    output = ospawn("git", "branch", "-v")
    os.chdir(workdir)
    revision = "unknown"
    for line in output:
        if line[0] == "*":
            values = line.split()
            revision = values[2]
    return revision

def stripBinaries(pl, opSys):
    strip = "strip"
    if opSys == "darwin":
        # Just plain "strip" renders our Python interpreter unusable
        # by extension modules.
        strip = "strip -S"
    if len(pl.stripList) > 0:
        status = os.system(strip + " " + " ".join(pl.stripList))
        if status != 0:
            sys.exit("strip: exit %d" % status)
    return

def rewriteScripts(pl, prefix, opSys):
    # Tweak the shebang line of scripts so they are relative instead
    # of absolute.

    if opSys == "win":
        # chrooted environment
        relative = "#!/bin/%s"
    else:
        relative = "#!/usr/bin/env %s"
    absolute = "#!" + prefix + "/bin/"
    absoluteAlt = absolute.replace("PyLith-binary-packaging","PyLith-binary")

    for script in pl.scripts:
        s = open(script, "r")
        lines = s.readlines()
        s.close()
        shebang = lines[0]
        if shebang.startswith(absolute) or shebang.startswith(absoluteAlt):
            if shebang.startswith(absolute):
                interpreter = shebang[len(absolute):]
            elif shebang.startswith(absoluteAlt):
                interpreter = shebang[len(absoluteAlt):]
            shebang = relative % interpreter
            s = open(script, "w")
            s.write(shebang)
            for line in lines[1:]:
                s.write(line)
            s.close()

    return

def installSource(pl, taggedArchive):
    src = "src"
    shutil.rmtree(src, ignore_errors=True)
    if not isdir(src):
        os.makedirs(src)
    spawn("tar", "-C", src, "-xf", taggedArchive)
    pl.addDirectory(src)
    return

def mkpkg():
    # WithProperties simply doesn't work in this case.
    if False:
        revision = sys.argv[2]
        if revision:
            revision = "r" + revision
        else:
            revision = "forced"

    makeinfo = getMakeInfo()
    python = "python"+makeinfo['python_version']
    distdir = makeinfo['package'] + "-" + makeinfo['version']
    prefix = makeinfo['prefix']
    package = makeinfo['package']
    print "version:", makeinfo['version']
    print "python:", python
    print "prefix:", prefix
    print "distdir:", distdir

    revision = getGitInfo(makeinfo['srcdir'])
    opSys = platform.system().lower()
    if opSys.startswith("cygwin"):
        opSys = "win"
    if opSys=="darwin":
        arch = opSys + "-" + platform.mac_ver()[0]
    else:
        arch = opSys + "-" + (platform.processor() or platform.machine()) # Why does every last detail have to be painful?

    config = Config.readPackingList()
    if config.packaging is None:
        sys.exit("ERROR: No packaging configuation.")

    # Make source distribution (also extracted into binary package)
    archive = distdir + ".tar.gz"
    taggedArchive = revision + "-" + archive
    status = os.system("make dist")
    if status != 0:
        sys.exit("make: exit %d" % status)
    os.system("mv " + archive + " " + taggedArchive)

    workdir = os.getcwd()
    os.chdir(prefix)

    pl = PackingList(config, opSys, python)

    stripBinaries(pl, opSys)
    rewriteScripts(pl, prefix, opSys)
    installSource(pl, join(workdir, taggedArchive))

    distdir_arch = distdir + "-" + arch
    if opSys == "darwin" or opSys == "linux":
        # No .dmg on Mac for now.
        os.chdir(workdir)
        os.system("rm " + distdir_arch)
        os.system("ln -s " + prefix + " " + distdir_arch)

        # Write packing list to file
        plfile = open("tar_list", "w")
        plfile.write('\n'.join([os.path.join(distdir_arch, m) for m in pl.files()]))
        plfile.close()

        archive = distdir_arch + ".tar.gz"
        taggedArchive = revision + "-" + archive
        status = os.system("tar cvzf " + archive + " -T tar_list")
        if status != 0:
            sys.exit("tar: exit %d" % status)

        os.system("mv " + archive + " " + taggedArchive)
        os.system("rm tar_list " + distdir_arch)
        shutil.rmtree("src", ignore_errors=True)

    elif opSys == "win":
        import cygwin

        name = "pylith"
        version = makeinfo['version']
        stage = cygwin.stageInstallation(prefix, workdir, python, pl)
        cygwin.createInstaller(python, stage, workdir, name, package, version, pl,
                               revision + "-" + distdir_arch)
    else:
        sys.exit("unknown OS: " + opSys)

    return

if __name__ == "__main__":
    mkpkg()
<commit_msg>Allow packaging using source tarballs (not repositories).<commit_after>#!/usr/bin/env python
#
# ----------------------------------------------------------------------
#
# Brad T. Aagaard, U.S. Geological Survey
# Charles A. Williams, GNS Science
# Matthew G. Knepley, University of Chicago
#
# This code was developed as part of the Computational Infrastructure
# for Geodynamics (http://geodynamics.org).
#
# Copyright (c) 2010-2016 University of California, Davis
#
# See COPYING for license information.
#
# ----------------------------------------------------------------------
#
# Create PyLith binary package.
#
# Run from the top-level PyLith build dir.
#
# Prerequisite: Copy setup_PLATFORM.sh file to PREFIX.
#
# Usage: make_package.py

import os, sys, platform
from popen2 import Popen4
from glob import glob
from os.path import basename, dirname, isabs, isdir, isfile, join
import shutil
from distutils.sysconfig import parse_makefile
from ConfigParser import ConfigParser

class Packaging(object):
    _name = 'packaging'
    _attrs = ['bin_dirs',
              'lib_dirs',
              'misc_dirs',
              'files',
              'strip_list',
              'exclude',
              'scripts',
              'urls',
              ]

class Config(object):

    sections = [Packaging]

    #@classmethod
    def readPackingList(cls):

        parser = ConfigParser()
        curdir = dirname(__file__)
        parser.read(join(curdir, "packinglist.cfg"))

        config = Config()

        for sectionCls in cls.sections:
            sectionDct = parser._sections.get(sectionCls._name, None)
            section = None
            if sectionDct is not None:
                section = sectionCls()
                for attr in section._attrs:
                    value = sectionDct.get(attr, "")
                    value = value.split()
                    setattr(section, attr, value)
            setattr(config, sectionCls._name, section)

        return config
    readPackingList = classmethod(readPackingList)

def getMakeInfo():
    # NYI: This is Autotools-specific.
    makefile = parse_makefile("Makefile")
    info = {'package_name': makefile['PACKAGE_NAME'],
            'package': makefile['PACKAGE'],
            'version': makefile['VERSION'],
            'srcdir': makefile['top_srcdir'],
            'prefix': makefile['prefix'],
            'python_version': makefile['PYTHON_VERSION'],
            }
    return info

def walkDirTree(dirname):
    """
    Get list of files in tree under dirname.
    """
    fullNames = []
    for dirpath, dirs, files in os.walk(dirname):
        for f in files:
            fullNames.append(os.path.join(dirpath, f))
    return fullNames

def filterList(l, excludes):
    try:
        import re, functools, operator

        pats = [re.compile(ex) for ex in excludes]
        return [i for i in l if not functools.reduce(operator.or_, [bool(pat.match(i)) for pat in pats])]
    except (ImportError, AttributeError):
        import re, operator

        pats = [re.compile(ex) for ex in excludes]
        return [i for i in l if not reduce(operator.or_, [bool(pat.match(i)) for pat in pats])]

class PackingList(object):
    def __init__(self, config, opSys, python):

        binDirs = config.packaging.bin_dirs
        libDirs = config.packaging.lib_dirs
        miscDirs = config.packaging.misc_dirs
        if platform.machine() == "x86_64":
            libDirs.append("lib64")
        self.directories = binDirs + libDirs + miscDirs

        self.extraFiles = config.packaging.files
        self.exclude = config.packaging.exclude

        self.programs = []
        for d in binDirs:
            self.programs.extend(walkDirTree(d))
        self.programs = filterList(self.programs, self.exclude)

        self.libraries = []
        for d in libDirs:
            self.libraries.extend(walkDirTree(d))
        self.libraries = filterList(self.libraries, self.exclude)

        self.misc = []
        for d in miscDirs:
            self.misc.extend(walkDirTree(d))
        self.misc = filterList(self.misc, self.exclude)

        # KLUDGE Explicitly add include/pythonX.X/pyconfig.h if it exists.
        pythonConfigFile = "include/" + python + "/pyconfig.h"
        if isfile(pythonConfigFile):
            self.misc.append(pythonConfigFile)

        # Scripts are application specific.
        self.scripts = []
        for s in config.packaging.scripts:
            s = "bin/" + s
            self.scripts.append(s)

        # Suffix for libraries that will be stripped of symbols.
        if opSys == "linux":
            libSuffix = ".so"
        elif opSys == "darwin":
            libSuffix = ".dylib"
        elif opSys == "win":
            libSuffix = ".dll"
        else:
            sys.exit("Unknown OS: " + opSys)

        self.stripList = []
        if opSys == "win":
            for f in config.packaging.strip_list:
                fdll = f.replace("lib/lib", "bin/cyg")+"-0"+libSuffix
                if fdll in self.libraries:
                    self.stripList.append(fdll)
        else:
            for f in config.packaging.strip_list:
                flib = f+libSuffix
                if flib in self.libraries:
                    self.stripList.append(flib)

        cig = [("CIG", "cig", "http://www.geodynamics.org/")]
        self.urls = cig + tupleUp(config.packaging.urls, 3)

        return

    def addFile(self, f):
        self.misc.append(f)

    def addDirectory(self, d):
        self.directories.append(d)
        newFiles = walkDirTree(d)
        newFiles = filterList(newFiles, self.exclude)
        self.misc.extend(newFiles)

    def files(self):
        for f in self.programs:
            yield f
        for f in self.libraries:
            yield f
        for f in self.misc:
            yield f
        for f in self.extraFiles:
            yield f
        return

    def all(self):
        for d in self.directories:
            yield d
        for f in self.files():
            yield f
        return

def tupleUp(l, n):
    tuples = []
    i = iter(l)
    try:
        while True:
            t = []
            for count in xrange(n):
                t.append(i.next())
            tuples.append(tuple(t))
    except StopIteration:
        pass
    return tuples

def spawn(*argv):
    print ' '.join(argv)
    status = os.spawnvp(os.P_WAIT, argv[0], argv)
    if status != 0:
        statusMsg = "%s: %s: exit %d" % (sys.argv[0], argv[0], status)
        sys.exit(statusMsg)
    return

def ospawn(*argv):
    print ' '.join(argv)
    child = Popen4(argv)

    child.tochild.close()

    output = child.fromchild.readlines()
    status = child.wait()

    exitStatus = None
    if (os.WIFSIGNALED(status)):
        statusStr = "signal %d" % os.WTERMSIG(status)
    elif (os.WIFEXITED(status)):
        exitStatus = os.WEXITSTATUS(status)
        statusStr = "exit %d" % exitStatus
    else:
        statusStr = "status %d" % status
    if exitStatus != 0:
        sys.exit("%s: %s: %s" % (sys.argv[0], argv[0], statusStr))

    return output

def getGitInfo(srcdir):
    revision = "unknown"

    workdir = os.getcwd()
    os.chdir(srcdir)
    try:
        output = ospawn("git", "branch", "-v")
        for line in output:
            if line[0] == "*":
                values = line.split()
                revision = values[2]
    except:
        revision = "unknown"
    os.chdir(workdir)
    return revision

def stripBinaries(pl, opSys):
    strip = "strip"
    if opSys == "darwin":
        # Just plain "strip" renders our Python interpreter unusable
        # by extension modules.
        strip = "strip -S"
    if len(pl.stripList) > 0:
        status = os.system(strip + " " + " ".join(pl.stripList))
        if status != 0:
            sys.exit("strip: exit %d" % status)
    return

def rewriteScripts(pl, prefix, opSys):
    # Tweak the shebang line of scripts so they are relative instead
    # of absolute.

    if opSys == "win":
        # chrooted environment
        relative = "#!/bin/%s"
    else:
        relative = "#!/usr/bin/env %s"
    absolute = "#!" + prefix + "/bin/"
    absoluteAlt = absolute.replace("PyLith-binary-packaging","PyLith-binary")

    for script in pl.scripts:
        s = open(script, "r")
        lines = s.readlines()
        s.close()
        shebang = lines[0]
        if shebang.startswith(absolute) or shebang.startswith(absoluteAlt):
            if shebang.startswith(absolute):
                interpreter = shebang[len(absolute):]
            elif shebang.startswith(absoluteAlt):
                interpreter = shebang[len(absoluteAlt):]
            shebang = relative % interpreter
            s = open(script, "w")
            s.write(shebang)
            for line in lines[1:]:
                s.write(line)
            s.close()

    return

def installSource(pl, taggedArchive):
    src = "src"
    shutil.rmtree(src, ignore_errors=True)
    if not isdir(src):
        os.makedirs(src)
    spawn("tar", "-C", src, "-xf", taggedArchive)
    pl.addDirectory(src)
    return

def mkpkg():
    # WithProperties simply doesn't work in this case.
    if False:
        revision = sys.argv[2]
        if revision:
            revision = "r" + revision
        else:
            revision = "forced"

    makeinfo = getMakeInfo()
    python = "python"+makeinfo['python_version']
    distdir = makeinfo['package'] + "-" + makeinfo['version']
    prefix = makeinfo['prefix']
    package = makeinfo['package']
    print "version:", makeinfo['version']
    print "python:", python
    print "prefix:", prefix
    print "distdir:", distdir

    revision = getGitInfo(makeinfo['srcdir'])
    opSys = platform.system().lower()
    if opSys.startswith("cygwin"):
        opSys = "win"
    if opSys=="darwin":
        arch = opSys + "-" + platform.mac_ver()[0]
    else:
        arch = opSys + "-" + (platform.processor() or platform.machine()) # Why does every last detail have to be painful?

    config = Config.readPackingList()
    if config.packaging is None:
        sys.exit("ERROR: No packaging configuation.")

    # Make source distribution (also extracted into binary package)
    archive = distdir + ".tar.gz"
    taggedArchive = revision + "-" + archive
    status = os.system("make dist")
    if status != 0:
        sys.exit("make: exit %d" % status)
    os.system("mv " + archive + " " + taggedArchive)

    workdir = os.getcwd()
    os.chdir(prefix)

    pl = PackingList(config, opSys, python)

    stripBinaries(pl, opSys)
    rewriteScripts(pl, prefix, opSys)
    installSource(pl, join(workdir, taggedArchive))

    distdir_arch = distdir + "-" + arch
    if opSys == "darwin" or opSys == "linux":
        # No .dmg on Mac for now.
        os.chdir(workdir)
        os.system("rm " + distdir_arch)
        os.system("ln -s " + prefix + " " + distdir_arch)

        # Write packing list to file
        plfile = open("tar_list", "w")
        plfile.write('\n'.join([os.path.join(distdir_arch, m) for m in pl.files()]))
        plfile.close()

        archive = distdir_arch + ".tar.gz"
        taggedArchive = revision + "-" + archive
        status = os.system("tar cvzf " + archive + " -T tar_list")
        if status != 0:
            sys.exit("tar: exit %d" % status)

        os.system("mv " + archive + " " + taggedArchive)
        os.system("rm tar_list " + distdir_arch)
        shutil.rmtree("src", ignore_errors=True)

    elif opSys == "win":
        import cygwin

        name = "pylith"
        version = makeinfo['version']
        stage = cygwin.stageInstallation(prefix, workdir, python, pl)
        cygwin.createInstaller(python, stage, workdir, name, package, version, pl,
                               revision + "-" + distdir_arch)
    else:
        sys.exit("unknown OS: " + opSys)

    return

if __name__ == "__main__":
    mkpkg()
<|endoftext|>