<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import base64
import sys
import uuid

from tempest.api.compute import base
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr

class ServersNegativeTestJSON(base.BaseComputeTest):
    _interface = 'json'

    def setUp(self):
        super(ServersNegativeTestJSON, self).setUp()
        try:
            self.client.wait_for_server_status(self.server_id, 'ACTIVE')
        except Exception:
            self.rebuild_server()

    @classmethod
    def setUpClass(cls):
        super(ServersNegativeTestJSON, cls).setUpClass()
        cls.client = cls.servers_client
        cls.img_client = cls.images_client
        cls.alt_os = clients.AltManager()
        cls.alt_client = cls.alt_os.servers_client
        resp, server = cls.create_server(wait_until='ACTIVE')
        cls.server_id = server['id']

    @attr(type=['negative', 'gate'])
    def test_server_name_blank(self):
        # Create a server with name parameter empty

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name='')

    @attr(type=['negative', 'gate'])
    def test_personality_file_contents_not_encoded(self):
        # Use an unencoded file when creating a server with personality

        file_contents = 'This is a test file.'
        person = [{'path': '/etc/testfile.txt',
                   'contents': file_contents}]

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          personality=person)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_image(self):
        # Create a server with an unknown image

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          image_id=-1)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_flavor(self):
        # Create a server with an unknown flavor

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          flavor=-1,)

    @attr(type=['negative', 'gate'])
    def test_invalid_access_ip_v4_address(self):
        # An access IPv4 address must match a valid address pattern

        IPv4 = '1.1.1.1.1.1'
        self.assertRaises(exceptions.BadRequest,
                          self.create_server, accessIPv4=IPv4)

    @attr(type=['negative', 'gate'])
    def test_invalid_ip_v6_address(self):
        # An access IPv6 address must match a valid address pattern

        IPv6 = 'notvalid'

        self.assertRaises(exceptions.BadRequest,
                          self.create_server, accessIPv6=IPv6)

    @attr(type=['negative', 'gate'])
    def test_resize_server_with_non_existent_flavor(self):
        # Resize a server with non-existent flavor
        nonexistent_flavor = str(uuid.uuid4())
        self.assertRaises(exceptions.BadRequest, self.client.resize,
                          self.server_id, flavor_ref=nonexistent_flavor)

    @attr(type=['negative', 'gate'])
    def test_resize_server_with_null_flavor(self):
        # Resize a server with null flavor
        self.assertRaises(exceptions.BadRequest, self.client.resize,
                          self.server_id, flavor_ref="")

    @attr(type=['negative', 'gate'])
    def test_reboot_non_existent_server(self):
        # Reboot a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.reboot,
                          nonexistent_server, 'SOFT')

    @attr(type=['negative', 'gate'])
    def test_reboot_deleted_server(self):
        # Reboot a deleted server
        self.client.delete_server(self.server_id)
        self.client.wait_for_server_termination(self.server_id)
        self.assertRaises(exceptions.NotFound, self.client.reboot,
                          self.server_id, 'SOFT')

    @attr(type=['negative', 'gate'])
    def test_pause_paused_server(self):
        # Pause a paused server.
        self.client.pause_server(self.server_id)
        self.client.wait_for_server_status(self.server_id, 'PAUSED')
        self.assertRaises(exceptions.Duplicate,
                          self.client.pause_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_rebuild_deleted_server(self):
        # Rebuild a deleted server
        self.client.delete_server(self.server_id)
        self.client.wait_for_server_termination(self.server_id)

        self.assertRaises(exceptions.NotFound,
                          self.client.rebuild,
                          self.server_id, self.image_ref_alt)

    @attr(type=['negative', 'gate'])
    def test_rebuild_non_existent_server(self):
        # Rebuild a non existent server
        nonexistent_server = str(uuid.uuid4())
        meta = {'rebuild': 'server'}
        new_name = rand_name('server')
        file_contents = 'Test server rebuild.'
        personality = [{'path': '/etc/rebuild.txt',
                        'contents': base64.b64encode(file_contents)}]
        self.assertRaises(exceptions.NotFound,
                          self.client.rebuild,
                          nonexistent_server,
                          self.image_ref_alt,
                          name=new_name, meta=meta,
                          personality=personality,
                          adminPass='rebuild')

    @attr(type=['negative', 'gate'])
    def test_create_numeric_server_name(self):
        # Create a server with a numeric name
        if self.__class__._interface == "xml":
            raise self.skipException("Not testable in XML")

        server_name = 12345
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name=server_name)

    @attr(type=['negative', 'gate'])
    def test_create_server_name_length_exceeds_256(self):
        # Create a server with name length exceeding 256 characters

        server_name = 'a' * 256
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name=server_name)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_network_uuid(self):
        # Pass invalid network uuid while creating a server

        networks = [{'fixed_ip': '10.0.1.1', 'uuid': 'a-b-c-d-e-f-g-h-i-j'}]

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          networks=networks)

    @attr(type=['negative', 'gate'])
    def test_create_with_non_existant_keypair(self):
        # Pass a non-existent keypair while creating a server

        key_name = rand_name('key')
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          key_name=key_name)

    @attr(type=['negative', 'gate'])
    def test_create_server_metadata_exceeds_length_limit(self):
        # Pass really long metadata while creating a server

        metadata = {'a': 'b' * 260}
        self.assertRaises(exceptions.OverLimit,
                          self.create_server,
                          meta=metadata)

    @attr(type=['negative', 'gate'])
    def test_update_name_of_non_existent_server(self):
        # Update name of a non-existent server

        server_name = rand_name('server')
        new_name = rand_name('server') + '_updated'

        self.assertRaises(exceptions.NotFound, self.client.update_server,
                          server_name, name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_set_empty_name(self):
        # Update name of the server to an empty string

        server_name = rand_name('server')
        new_name = ''

        self.assertRaises(exceptions.BadRequest, self.client.update_server,
                          server_name, name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_of_another_tenant(self):
        # Update name of a server that belongs to another tenant

        new_name = self.server_id + '_new'
        self.assertRaises(exceptions.NotFound,
                          self.alt_client.update_server, self.server_id,
                          name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_name_length_exceeds_256(self):
        # Update name of server exceed the name length limit

        new_name = 'a' * 256
        self.assertRaises(exceptions.BadRequest,
                          self.client.update_server,
                          self.server_id,
                          name=new_name)

    @attr(type=['negative', 'gate'])
    def test_delete_non_existent_server(self):
        # Delete a non existent server

        self.assertRaises(exceptions.NotFound, self.client.delete_server,
                          '999erra43')

    @attr(type=['negative', 'gate'])
    def test_delete_a_server_of_another_tenant(self):
        # Delete a server that belongs to another tenant
        self.assertRaises(exceptions.NotFound,
                          self.alt_client.delete_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_delete_server_pass_negative_id(self):
        # Pass an invalid string parameter to delete server

        self.assertRaises(exceptions.NotFound, self.client.delete_server, -1)

    @attr(type=['negative', 'gate'])
    def test_delete_server_pass_id_exceeding_length_limit(self):
        # Pass a server ID that exceeds length limit to delete server

        self.assertRaises(exceptions.NotFound, self.client.delete_server,
                          sys.maxint + 1)

    @attr(type=['negative', 'gate'])
    def test_create_with_nonexistent_security_group(self):
        # Create a server with a nonexistent security group

        security_groups = [{'name': 'does_not_exist'}]
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          security_groups=security_groups)

    @attr(type=['negative', 'gate'])
    def test_get_non_existent_server(self):
        # Get a non existent server details

        self.assertRaises(exceptions.NotFound, self.client.get_server,
                          '999erra43')

    @attr(type=['negative', 'gate'])
    def test_stop_non_existent_server(self):
        # Stop a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.servers_client.stop,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_pause_non_existent_server(self):
        # pause a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.pause_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_unpause_non_existent_server(self):
        # unpause a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.unpause_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_unpause_server_invalid_state(self):
        # unpause an active server.
        self.assertRaises(exceptions.Duplicate,
                          self.client.unpause_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_suspend_non_existent_server(self):
        # suspend a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.suspend_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_suspend_server_invalid_state(self):
        # suspend a suspended server.
        resp, _ = self.client.suspend_server(self.server_id)
        self.assertEqual(202, resp.status)
        self.client.wait_for_server_status(self.server_id, 'SUSPENDED')
        self.assertRaises(exceptions.Duplicate,
                          self.client.suspend_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_resume_non_existent_server(self):
        # resume a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.resume_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_resume_server_invalid_state(self):
        # resume an active server.
        self.assertRaises(exceptions.Duplicate,
                          self.client.resume_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_get_console_output_of_non_existent_server(self):
        # get the console output for a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound,
                          self.client.get_console_output,
                          nonexistent_server, 10)

class ServersNegativeTestXML(ServersNegativeTestJSON):
    _interface = 'xml'
<commit_msg>server status remained in unexpected state<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import base64
import sys
import uuid

from tempest.api.compute import base
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr

class ServersNegativeTestJSON(base.BaseComputeTest):
    _interface = 'json'

    def setUp(self):
        super(ServersNegativeTestJSON, self).setUp()
        try:
            self.client.wait_for_server_status(self.server_id, 'ACTIVE')
        except Exception:
            self.rebuild_server()

    @classmethod
    def setUpClass(cls):
        super(ServersNegativeTestJSON, cls).setUpClass()
        cls.client = cls.servers_client
        cls.img_client = cls.images_client
        cls.alt_os = clients.AltManager()
        cls.alt_client = cls.alt_os.servers_client
        resp, server = cls.create_server(wait_until='ACTIVE')
        cls.server_id = server['id']

    @attr(type=['negative', 'gate'])
    def test_server_name_blank(self):
        # Create a server with name parameter empty

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name='')

    @attr(type=['negative', 'gate'])
    def test_personality_file_contents_not_encoded(self):
        # Use an unencoded file when creating a server with personality

        file_contents = 'This is a test file.'
        person = [{'path': '/etc/testfile.txt',
                   'contents': file_contents}]

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          personality=person)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_image(self):
        # Create a server with an unknown image

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          image_id=-1)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_flavor(self):
        # Create a server with an unknown flavor

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          flavor=-1,)

    @attr(type=['negative', 'gate'])
    def test_invalid_access_ip_v4_address(self):
        # An access IPv4 address must match a valid address pattern

        IPv4 = '1.1.1.1.1.1'
        self.assertRaises(exceptions.BadRequest,
                          self.create_server, accessIPv4=IPv4)

    @attr(type=['negative', 'gate'])
    def test_invalid_ip_v6_address(self):
        # An access IPv6 address must match a valid address pattern

        IPv6 = 'notvalid'

        self.assertRaises(exceptions.BadRequest,
                          self.create_server, accessIPv6=IPv6)

    @attr(type=['negative', 'gate'])
    def test_resize_server_with_non_existent_flavor(self):
        # Resize a server with non-existent flavor
        nonexistent_flavor = str(uuid.uuid4())
        self.assertRaises(exceptions.BadRequest, self.client.resize,
                          self.server_id, flavor_ref=nonexistent_flavor)

    @attr(type=['negative', 'gate'])
    def test_resize_server_with_null_flavor(self):
        # Resize a server with null flavor
        self.assertRaises(exceptions.BadRequest, self.client.resize,
                          self.server_id, flavor_ref="")

    @attr(type=['negative', 'gate'])
    def test_reboot_non_existent_server(self):
        # Reboot a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.reboot,
                          nonexistent_server, 'SOFT')

    @attr(type=['negative', 'gate'])
    def test_pause_paused_server(self):
        # Pause a paused server.
        self.client.pause_server(self.server_id)
        self.addCleanup(self.client.unpause_server,
                        self.server_id)
        self.client.wait_for_server_status(self.server_id, 'PAUSED')
        self.assertRaises(exceptions.Duplicate,
                          self.client.pause_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_rebuild_reboot_deleted_server(self):
        # Rebuild and Reboot a deleted server
        _, server = self.create_server()
        self.client.delete_server(server['id'])
        self.client.wait_for_server_termination(server['id'])

        self.assertRaises(exceptions.NotFound,
                          self.client.rebuild,
                          server['id'], self.image_ref_alt)
        self.assertRaises(exceptions.NotFound, self.client.reboot,
                          server['id'], 'SOFT')

    @attr(type=['negative', 'gate'])
    def test_rebuild_non_existent_server(self):
        # Rebuild a non existent server
        nonexistent_server = str(uuid.uuid4())
        meta = {'rebuild': 'server'}
        new_name = rand_name('server')
        file_contents = 'Test server rebuild.'
        personality = [{'path': '/etc/rebuild.txt',
                        'contents': base64.b64encode(file_contents)}]
        self.assertRaises(exceptions.NotFound,
                          self.client.rebuild,
                          nonexistent_server,
                          self.image_ref_alt,
                          name=new_name, meta=meta,
                          personality=personality,
                          adminPass='rebuild')

    @attr(type=['negative', 'gate'])
    def test_create_numeric_server_name(self):
        # Create a server with a numeric name
        if self.__class__._interface == "xml":
            raise self.skipException("Not testable in XML")

        server_name = 12345
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name=server_name)

    @attr(type=['negative', 'gate'])
    def test_create_server_name_length_exceeds_256(self):
        # Create a server with name length exceeding 256 characters

        server_name = 'a' * 256
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          name=server_name)

    @attr(type=['negative', 'gate'])
    def test_create_with_invalid_network_uuid(self):
        # Pass invalid network uuid while creating a server

        networks = [{'fixed_ip': '10.0.1.1', 'uuid': 'a-b-c-d-e-f-g-h-i-j'}]

        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          networks=networks)

    @attr(type=['negative', 'gate'])
    def test_create_with_non_existant_keypair(self):
        # Pass a non-existent keypair while creating a server

        key_name = rand_name('key')
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          key_name=key_name)

    @attr(type=['negative', 'gate'])
    def test_create_server_metadata_exceeds_length_limit(self):
        # Pass really long metadata while creating a server

        metadata = {'a': 'b' * 260}
        self.assertRaises(exceptions.OverLimit,
                          self.create_server,
                          meta=metadata)

    @attr(type=['negative', 'gate'])
    def test_update_name_of_non_existent_server(self):
        # Update name of a non-existent server

        server_name = rand_name('server')
        new_name = rand_name('server') + '_updated'

        self.assertRaises(exceptions.NotFound, self.client.update_server,
                          server_name, name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_set_empty_name(self):
        # Update name of the server to an empty string

        server_name = rand_name('server')
        new_name = ''

        self.assertRaises(exceptions.BadRequest, self.client.update_server,
                          server_name, name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_of_another_tenant(self):
        # Update name of a server that belongs to another tenant

        new_name = self.server_id + '_new'
        self.assertRaises(exceptions.NotFound,
                          self.alt_client.update_server, self.server_id,
                          name=new_name)

    @attr(type=['negative', 'gate'])
    def test_update_server_name_length_exceeds_256(self):
        # Update name of server exceed the name length limit

        new_name = 'a' * 256
        self.assertRaises(exceptions.BadRequest,
                          self.client.update_server,
                          self.server_id,
                          name=new_name)

    @attr(type=['negative', 'gate'])
    def test_delete_non_existent_server(self):
        # Delete a non existent server

        self.assertRaises(exceptions.NotFound, self.client.delete_server,
                          '999erra43')

    @attr(type=['negative', 'gate'])
    def test_delete_a_server_of_another_tenant(self):
        # Delete a server that belongs to another tenant
        self.assertRaises(exceptions.NotFound,
                          self.alt_client.delete_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_delete_server_pass_negative_id(self):
        # Pass an invalid string parameter to delete server

        self.assertRaises(exceptions.NotFound, self.client.delete_server, -1)

    @attr(type=['negative', 'gate'])
    def test_delete_server_pass_id_exceeding_length_limit(self):
        # Pass a server ID that exceeds length limit to delete server

        self.assertRaises(exceptions.NotFound, self.client.delete_server,
                          sys.maxint + 1)

    @attr(type=['negative', 'gate'])
    def test_create_with_nonexistent_security_group(self):
        # Create a server with a nonexistent security group

        security_groups = [{'name': 'does_not_exist'}]
        self.assertRaises(exceptions.BadRequest,
                          self.create_server,
                          security_groups=security_groups)

    @attr(type=['negative', 'gate'])
    def test_get_non_existent_server(self):
        # Get a non existent server details

        self.assertRaises(exceptions.NotFound, self.client.get_server,
                          '999erra43')

    @attr(type=['negative', 'gate'])
    def test_stop_non_existent_server(self):
        # Stop a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.servers_client.stop,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_pause_non_existent_server(self):
        # pause a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.pause_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_unpause_non_existent_server(self):
        # unpause a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.unpause_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_unpause_server_invalid_state(self):
        # unpause an active server.
        self.assertRaises(exceptions.Duplicate,
                          self.client.unpause_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_suspend_non_existent_server(self):
        # suspend a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.suspend_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_suspend_server_invalid_state(self):
        # suspend a suspended server.
        resp, _ = self.client.suspend_server(self.server_id)
        self.addCleanup(self.client.resume_server,
                        self.server_id)
        self.assertEqual(202, resp.status)
        self.client.wait_for_server_status(self.server_id, 'SUSPENDED')
        self.assertRaises(exceptions.Duplicate,
                          self.client.suspend_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_resume_non_existent_server(self):
        # resume a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound, self.client.resume_server,
                          nonexistent_server)

    @attr(type=['negative', 'gate'])
    def test_resume_server_invalid_state(self):
        # resume an active server.
        self.assertRaises(exceptions.Duplicate,
                          self.client.resume_server,
                          self.server_id)

    @attr(type=['negative', 'gate'])
    def test_get_console_output_of_non_existent_server(self):
        # get the console output for a non existent server
        nonexistent_server = str(uuid.uuid4())
        self.assertRaises(exceptions.NotFound,
                          self.client.get_console_output,
                          nonexistent_server, 10)

class ServersNegativeTestXML(ServersNegativeTestJSON):
    _interface = 'xml'
<|endoftext|>


<commit_before>from numpy import pi as _pi

def cylinder(target, throat_diameter='throat.diameter'):
    r"""
    Calculate throat cross-sectional area for a cylindrical throat

    Parameters
    ----------
    geometry : OpenPNM Geometry Object
        The Geometry object which this model is associated with. This controls
        the length of the calculated array, and also provides access to other
        necessary geometric properties.

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    diams = target[throat_diameter]
    value = _pi/4*(diams)**2
    return value

def cuboid(target, throat_diameter='throat.diameter'):
    r"""
    Calculate throat cross-sectional area for a cuboid throat

    Parameters
    ----------
    geometry : OpenPNM Geometry Object
        The Geometry object which this model is associated with. This controls
        the length of the calculated array, and also provides access to other
        necessary geometric properties.

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    diams = target[throat_diameter]
    value = (diams)**2
    return value
<commit_msg>Fixed docstrings<commit_after>from numpy import pi as _pi

def cylinder(target, throat_diameter='throat.diameter'):
    r"""
    Calculate throat cross-sectional area for a cylindrical throat

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    diams = target[throat_diameter]
    value = _pi/4*(diams)**2
    return value

def cuboid(target, throat_diameter='throat.diameter'):
    r"""
    Calculate throat cross-sectional area for a cuboid throat

    Parameters
    ----------
    target : OpenPNM Object
        The object which this model is associated with. This controls the
        length of the calculated array, and also provides access to other
        necessary properties.

    throat_diameter : string
        Dictionary key of the throat diameter values

    """
    diams = target[throat_diameter]
    value = (diams)**2
    return value
<|endoftext|>


<commit_before>import logging
import glob
from multiprocessing import Process

from CPTCorpus import CPTCorpus
from CPT_Gibbs import GibbsSampler

def run_sampler(corpus, nTopics, nIter, beta, out_dir):
    sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
                           alpha=(50.0/n), beta=beta, beta_o=beta,
                           out_dir=out_dir.format(nTopics))
    sampler._initialize()
    sampler.run()

logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

logger.setLevel(logging.DEBUG)
#logger.setLevel(logging.INFO)

files = glob.glob('/home/jvdzwaan/data/tmp/test/*')

out_dir = '/home/jvdzwaan/data/tmp/generated/test_exp/{}'

corpus = CPTCorpus(files, testSplit=20)
corpus.filter_dictionaries(minFreq=5, removeTopTF=100, removeTopDF=100)
corpus.save_dictionaries(directory=out_dir.format(''))
corpus.save(out_dir.format('corpus.json'))

#corpus = CPTCorpus.CPTCorpus.load('{}corpus.json'.format(out_dir),
#                                  topicDict='{}/topicDict.dict'.format(out_dir),
#                                  opinionDict='{}/opinionDict.dict'.format(out_dir))

nIter = 200
beta = 0.02

nTopics = range(20, 201, 20)
logger.info('running Gibbs sampler for {} configurations'.format(len(nTopics)))

processes = [Process(target=run_sampler,
                     args=(corpus, n, nIter, beta, out_dir))
             for n in nTopics]

# Run processes
for p in processes:
    p.start()

# Exit the completed processes
for p in processes:
    p.join()
<commit_msg>Fix bug in experiment_number_of_topics script<commit_after>import logging
import glob
from multiprocessing import Process

from CPTCorpus import CPTCorpus
from CPT_Gibbs import GibbsSampler

def run_sampler(corpus, nTopics, nIter, beta, out_dir):
    sampler = GibbsSampler(corpus, nTopics=nTopics, nIter=nIter,
                           alpha=(50.0/nTopics), beta=beta, beta_o=beta,
                           out_dir=out_dir.format(nTopics))
    sampler._initialize()
    sampler.run()

logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

logger.setLevel(logging.DEBUG)
#logger.setLevel(logging.INFO)

files = glob.glob('/home/jvdzwaan/data/tmp/test/*')

out_dir = '/home/jvdzwaan/data/tmp/generated/test_exp/{}'

corpus = CPTCorpus(files, testSplit=20)
corpus.filter_dictionaries(minFreq=5, removeTopTF=100, removeTopDF=100)
corpus.save_dictionaries(directory=out_dir.format(''))
corpus.save(out_dir.format('corpus.json'))

#corpus = CPTCorpus.CPTCorpus.load('{}corpus.json'.format(out_dir),
#                                  topicDict='{}/topicDict.dict'.format(out_dir),
#                                  opinionDict='{}/opinionDict.dict'.format(out_dir))

nIter = 200
beta = 0.02

nTopics = range(20, 201, 20)
logger.info('running Gibbs sampler for {} configurations'.format(len(nTopics)))

processes = [Process(target=run_sampler,
                     args=(corpus, n, nIter, beta, out_dir))
             for n in nTopics]

# Run processes
for p in processes:
    p.start()

# Exit the completed processes
for p in processes:
    p.join()
<|endoftext|>


<commit_before>import os
import tempfile
import unittest
import numpy
from openquake.baselib import hdf5
from openquake.commonlib import nrml_examples, calc, source
from openquake.commonlib.sourceconverter import SourceConverter

NRML_DIR = os.path.dirname(nrml_examples.__file__)
MIXED_SRC_MODEL = os.path.join(NRML_DIR, 'source_model/mixed.xml')
parser = source.SourceModelParser(SourceConverter(
    investigation_time=50.,
    rupture_mesh_spacing=1,  # km
    complex_fault_mesh_spacing=1,  # km
    width_of_mfd_bin=1.,  # for Truncated GR MFDs
    area_source_discretization=1.,  # km
))

class SerializeRuptureTestCase(unittest.TestCase):
    def test(self):
        groups = parser.parse_groups(MIXED_SRC_MODEL)
        ([point], [cmplx], [area, simple],
         [char_simple, char_complex, char_multi]) = groups
        fh, self.path = tempfile.mkstemp(suffix='.hdf5')
        os.close(fh)
        print('Writing on %s' % self.path)
        self.i = 0
        self.sids = numpy.array([0], numpy.uint32)
        self.events = numpy.array([(0, 1, 1, 0)], calc.event_dt)
        self.write_read(point)
        self.write_read(char_simple)
        self.write_read(char_complex)
        self.write_read(char_multi)

    def write_read(self, src):
        with hdf5.File(self.path, 'r+') as f:
            for rup in src.iter_ruptures():
                ebr = calc.EBRupture(
                    rup, self.sids, self.events, src.source_id, 0, self.i)
                f[str(self.i)] = ebr
                self.i += 1
        with hdf5.File(self.path, 'r') as f:
            for key in f:
                f[key]
<commit_msg>Fixed SerializeRuptureTestCase<commit_after>import os
import tempfile
import unittest
import numpy
from openquake.baselib import hdf5
from openquake.commonlib import nrml_examples, calc, source
from openquake.commonlib.sourceconverter import SourceConverter

NRML_DIR = os.path.dirname(nrml_examples.__file__)
MIXED_SRC_MODEL = os.path.join(NRML_DIR, 'source_model/mixed.xml')
parser = source.SourceModelParser(SourceConverter(
    investigation_time=50.,
    rupture_mesh_spacing=1,  # km
    complex_fault_mesh_spacing=1,  # km
    width_of_mfd_bin=1.,  # for Truncated GR MFDs
    area_source_discretization=1.,  # km
))

class SerializeRuptureTestCase(unittest.TestCase):
    def test(self):
        groups = parser.parse_groups(MIXED_SRC_MODEL)
        ([point], [cmplx], [area, simple],
         [char_simple, char_complex, char_multi]) = groups
        fh, self.path = tempfile.mkstemp(suffix='.hdf5')
        os.close(fh)
        print('Writing on %s' % self.path)
        self.i = 0
        self.sids = numpy.array([0], numpy.uint32)
        self.events = numpy.array([(0, 1, 1, 0)], calc.event_dt)
        self.write_read(point)
        self.write_read(char_simple)
        self.write_read(char_complex)
        self.write_read(char_multi)

    def write_read(self, src):
        with hdf5.File(self.path, 'r+') as f:
            for rup in src.iter_ruptures():
                rup.seed = 0
                ebr = calc.EBRupture(
                    rup, self.sids, self.events, src.source_id, 0, self.i)
                f[str(self.i)] = ebr
                self.i += 1
        with hdf5.File(self.path, 'r') as f:
            for key in f:
                f[key]
<|endoftext|>


<commit_before>import os
import re
import time
import urllib
import importlib
from openquake.engine import logs

class DuplicatedVersion(RuntimeError):
    pass

class VersionTooSmall(RuntimeError):
    pass

class VersioningNotInstalled(RuntimeError):
    pass

CREATE_VERSIONING = '''\
CREATE TABLE %s(
version TEXT PRIMARY KEY,
scriptname TEXT NOT NULL,
executed TIMESTAMP NOT NULL DEFAULT now()
);
'''

class WrappedConnection(object):
    """
    This is an utility class that wraps a DB API-2 connection
    providing a couple of convenient features.

    1) it is possible to set a debug flag to print on stdout
       the executed queries;
    2) there is a .run method to run a query with a dedicated
       cursor; it returns the cursor, which can be iterated over

    :param conn: a DB API2-compatible connection
    """
    def __init__(self, conn, debug=False):
        self._conn = conn
        self.debug = debug

    def __getattr__(self, name):
        return getattr(self._conn, name)

    def run(self, templ, *args):
        """
        A simple utility to run SQL queries.

        :param templ: a query or query template
        :param args: the arguments (or the empty tuple)
        :returns: the DB API 2 cursor used to run the query
        """
        curs = self._conn.cursor()
        query = curs.mogrify(templ, args)
        if self.debug:
            print query
        curs.execute(query)
        return curs

def check_script(upgrade, dry_run=True, debug=True):
    """
    An utility to debug upgrade scripts written in Python

    :param upgrade: upgrade procedure
    :param dry_run: if True, do not change the database
    :param debug: if True, print the queries which are executed
    """
    from openquake.engine.db.models import getcursor
    conn = WrappedConnection(getcursor('admin').connection, debug=debug)
    try:
        upgrade(conn)
    except:
        conn.rollback()
        raise
    else:
        if dry_run:
            conn.rollback()
        else:
            conn.commit()

def apply_sql_script(conn, fname):
    """
    Apply the given SQL script to the database

    :param conn: a DB API 2 connection
    :param fname: full path to the creation script
    """
    sql = open(fname).read()
    try:
        conn.cursor().execute(sql)
    except:
        logs.LOG.error('Error executing %s' % fname)
        raise

# errors are not trapped on purpose, since transactions should be managed
# in client code
class UpgradeManager(object):
    """
    The package containing the upgrade scripts should contain an instance
    of the UpgradeManager called `upgrader` in the __init__.py file. It
    should also specify the initializations parameters

    :param upgrade_dir:
        the directory were the upgrade script reside
    :param version_table:
        the name of the versioning table (default public.revision_info)
    :param version_pattern:
        a regulation expression for the script version number (\d\d\d\d)
    """
    ENGINE_URL = 'https://github.com/gem/oq-engine/tree/master/'
    UPGRADES = 'openquake/engine/db/schema/upgrades/'

    def __init__(self, upgrade_dir, version_table='public.revision_info',
                 version_pattern='\d\d\d\d', flag_pattern='(-slow|-danger)?'):
        self.upgrade_dir = upgrade_dir
        self.version_table = version_table
        self.version_pattern = version_pattern
        self.flag_pattern = flag_pattern
        self.pattern = r'^(%s)%s-([\w\-_]+)\.(sql|py)$' % (
            version_pattern, flag_pattern)
        self.upgrades_url = self.ENGINE_URL + self.UPGRADES
        if '.' in version_table:  # contains the schema name
            self.version_schema_name, self.version_table_name = \
                version_table.split('.')
        else:  # assume the schema name is 'public'
            self.version_schema_name, self.version_table_name = \
                ('public', self.version_table)
        if re.match('[\w_\.]+', version_table) is None:
            raise ValueError(version_table)
        self.starting_version = None  # will be updated after the run

    def _insert_script(self, script, conn):
        conn.cursor().execute(
            'INSERT INTO {} VALUES (%s, %s)'.format(self.version_table),
            (script['version'], script['name']))

    def install_versioning(self, conn):
        """
        Create the version table into an already populated database
        and insert the base script.

        :param conn: a DB API 2 connection
        """
        logs.LOG.info('Creating the versioning table %s', self.version_table)
        conn.cursor().execute(CREATE_VERSIONING % self.version_table)
        self._insert_script(self.read_scripts()[0], conn)

    def init(self, conn):
        """
        Create the version table and run the base script on an empty database.

        :param conn: a DB API 2 connection
        """
        base = self.read_scripts()[0]['fname']
        logs.LOG.info('Creating the initial schema from %s',  base)
        apply_sql_script(conn, os.path.join(self.upgrade_dir, base))
        self.install_versioning(conn)

    def upgrade(self, conn, skip_versions=()):
        '''
        Upgrade the database from the current version to the maximum
        version in the upgrade scripts.

        :param conn: a DBAPI 2 connection
        :param skip_versions: the versions to skip
        '''
        db_versions = self.get_db_versions(conn)
        self.starting_version = max(db_versions)
        to_skip = sorted(db_versions | set(skip_versions))
        scripts = self.read_scripts(None, None, to_skip)
        if not scripts:  # no new scripts to apply
            return []
        self.ending_version = max(s['version'] for s in scripts)
        return self._upgrade(conn, scripts)

    def _upgrade(self, conn, scripts):
        conn = WrappedConnection(conn)
        versions_applied = []
        for script in scripts:  # script is a dictionary
            fullname = os.path.join(self.upgrade_dir, script['fname'])
            logs.LOG.info('Executing %s', fullname)
            if script['ext'] == 'py':  # Python script with a upgrade(conn)
                globs = {}
                execfile(fullname, globs)
                globs['upgrade'](conn)
                self._insert_script(script, conn)
            else:  # SQL script
                # notice that this prints the file name in case of error
                apply_sql_script(conn, fullname)
                self._insert_script(script, conn)
            versions_applied.append(script['version'])
        return versions_applied

    def check_versions(self, conn):
        """
        :param conn: a DB API 2 connection
        :returns: a list with the versions that will be applied
        """
        scripts = self.read_scripts(skip_versions=self.get_db_versions(conn))
        versions = [s['version'] for s in scripts]
        if scripts:
            raise SystemExit(
                'Your database is not updated. You can update it by running '
                'openquake --upgrade-db which will process the '
                'following new versions: %s' % versions)
        return versions

    def get_db_versions(self, conn):
        """
        Get all the versions stored in the database as a set.

        :param conn: a DB API 2 connection
        """
        curs = conn.cursor()
        query = 'select version from {}'.format(self.version_table)
        try:
            curs.execute(query)
            return set(version for version, in curs.fetchall())
        except:
            raise VersioningNotInstalled('Run openquake --upgrade-db')

    def parse_script_name(self, script_name):
        '''
        Parse a script name and return a dictionary with fields
        fname, name, version and ext (or None if the name does not match).

        :param name: name of the script
        '''
        match = re.match(self.pattern, script_name)
        if not match:
            return
        version, flag, name, ext = match.groups()
        return dict(fname=script_name, version=version, name=name,
                    flag=flag, ext=ext, url=self.upgrades_url + script_name)

    def read_scripts(self, minversion=None, maxversion=None, skip_versions=()):
        """
        Extract the upgrade scripts from a directory as a list of
        dictionaries, ordered by version.

        :param minversion: the minimum version to consider
        :param maxversion: the maximum version to consider
        :param skipversions: the versions to skip
        """
        scripts = []
        versions = {}  # a script is unique per version
        for scriptname in sorted(os.listdir(self.upgrade_dir)):
            match = self.parse_script_name(scriptname)
            if match:
                version = match['version']
                if version in skip_versions:
                    continue  # do not collect scripts already applied
                elif minversion and version <= minversion:
                    continue  # do not collect versions too old
                elif maxversion and version > maxversion:
                    continue  # do not collect versions too new
                try:
                    previousname = versions[version]
                except KeyError:  # no previous script with the same version
                    scripts.append(match)
                    versions[version] = scriptname
                else:
                    raise DuplicatedVersion(
                        'Duplicated versions {%s,%s}' %
                        (scriptname, previousname))
        return scripts

    def extract_upgrade_scripts(self):
        """
        Extract the OpenQuake upgrade scripts from the links in the GitHub page
        """
        link_pattern = '>\s*{0}\s*<'.format(self.pattern[1:-1])
        page = urllib.urlopen(self.upgrades_url).read()
        for mo in re.finditer(link_pattern, page):
            scriptname = mo.group(0)[1:-1].strip()
            yield self.parse_script_name(scriptname)

    @classmethod
    def instance(cls, conn, pkg_name='openquake.engine.db.schema.upgrades'):
        """
        Return an :class:`UpgradeManager` instance.

        :param conn: a DB API 2 connection
        :param str pkg_name: the name of the package with the upgrade scripts
        """
        try:
            # upgrader is an UpgradeManager instance defined in the __init__.py
            upgrader = importlib.import_module(pkg_name).upgrader
        except ImportError:
            raise SystemExit(
                'Could not import %s (not in the PYTHONPATH?)' % pkg_name)
        if not upgrader.read_scripts():
            raise SystemExit(
                'The upgrade_dir does not contain scripts matching '
                'the pattern %s' % upgrader.pattern)
        curs = conn.cursor()
        # check if there is already a versioning table
        curs.execute("SELECT tablename FROM pg_tables "
                     "WHERE schemaname=%s AND tablename=%s",
                     (upgrader.version_schema_name,
                      upgrader.version_table_name))
        versioning_table = curs.fetchall()
        # if not, run the base script and create the versioning table
        if not versioning_table:
            upgrader.init(conn)
            conn.commit()
        return upgrader

def upgrade_db(conn, pkg_name='openquake.engine.db.schema.upgrades',
               skip_versions=()):
    """
    Upgrade a database by running several scripts in a single transaction.

    :param conn: a DB API 2 connection
    :param str pkg_name: the name of the package with the upgrade scripts
    :param list skip_versions: the versions to skip
    :returns: the version numbers of the new scripts applied the database
    """
    upgrader = UpgradeManager.instance(conn, pkg_name)
    t0 = time.time()
    # run the upgrade scripts
    try:
        versions_applied = upgrader.upgrade(conn, skip_versions)
    except:
        conn.rollback()
        raise
    else:
        conn.commit()
    dt = time.time() - t0
    logs.LOG.info('Upgrade completed in %s seconds', dt)
    return versions_applied

def version_db(conn, pkg_name='openquake.engine.db.schema.upgrades'):
    """
    :param conn: a DB API 2 connection
    :param str pkg_name: the name of the package with the upgrade scripts
    :returns: the current version of the database
    """
    upgrader = UpgradeManager.instance(conn, pkg_name)
    return max(upgrader.get_db_versions(conn))

def what_if_I_upgrade(conn, pkg_name='openquake.engine.db.schema.upgrades',
                      extract_scripts='extract_upgrade_scripts'):
    """
    :param conn:
        a DB API 2 connection
    :param str pkg_name:
        the name of the package with the upgrade scripts
    :param extract_scripts:
        name of the method to extract the scripts
    """
    msg_safe_ = '\nThe following scripts can be applied safely:\n%s'
    msg_slow_ = '\nPlease note that the following scripts could be slow:\n%s'
    msg_danger_ = ('\nPlease note that the following scripts are potentially '
                   'dangerous and could destroy your data:\n%s')
    upgrader = UpgradeManager.instance(conn, pkg_name)
    applied_versions = upgrader.get_db_versions(conn)
    current_version = max(applied_versions)
    slow = []
    danger = []
    safe = []
    for script in getattr(upgrader, extract_scripts)():
        url = script['url']
        if script['version'] in applied_versions:
            continue
        elif script['version'] <= current_version:
            # you cannot apply a script with a version number lower than the
            # current db version: ensure that upgrades are strictly incremental
            raise VersionTooSmall(
                'Your database is at version %s but you want to apply %s??'
                % (current_version, script['fname']))
        elif script['flag'] == '-slow':
            slow.append(url)
        elif script['flag'] == '-danger':
            danger.append(url)
        else:  # safe script
            safe.append(url)

    if not safe and not slow and not danger:
        return 'Your database is already updated at version %s.' % \
            current_version

    header = 'Your database is at version %s.' % current_version
    msg_safe = msg_safe_ % '\n'.join(safe)
    msg_slow = msg_slow_ % '\n'.join(slow)
    msg_danger = msg_danger_ % '\n'.join(danger)
    msg = header + (msg_safe if safe else '') + (msg_slow if slow else '') \
        + (msg_danger if danger else '')
    msg += ('\nClick on the links if you want to know what exactly the '
            'scripts are doing.')
    if slow:
        msg += ('\nEven slow script can be fast if your database is small or'
                ' the upgrade affects tables that are empty.')
    if danger:
        msg += ('\nEven dangerous scripts are fine if they '
                'affect empty tables or data you are not interested in.')
    return msg
<commit_msg>Replace a reference to the 'openquake' command with 'oq-engine'<commit_after>import os
import re
import time
import urllib
import importlib
from openquake.engine import logs

class DuplicatedVersion(RuntimeError):
    pass

class VersionTooSmall(RuntimeError):
    pass

class VersioningNotInstalled(RuntimeError):
    pass

CREATE_VERSIONING = '''\
CREATE TABLE %s(
version TEXT PRIMARY KEY,
scriptname TEXT NOT NULL,
executed TIMESTAMP NOT NULL DEFAULT now()
);
'''

class WrappedConnection(object):
    """
    This is an utility class that wraps a DB API-2 connection
    providing a couple of convenient features.

    1) it is possible to set a debug flag to print on stdout
       the executed queries;
    2) there is a .run method to run a query with a dedicated
       cursor; it returns the cursor, which can be iterated over

    :param conn: a DB API2-compatible connection
    """
    def __init__(self, conn, debug=False):
        self._conn = conn
        self.debug = debug

    def __getattr__(self, name):
        return getattr(self._conn, name)

    def run(self, templ, *args):
        """
        A simple utility to run SQL queries.

        :param templ: a query or query template
        :param args: the arguments (or the empty tuple)
        :returns: the DB API 2 cursor used to run the query
        """
        curs = self._conn.cursor()
        query = curs.mogrify(templ, args)
        if self.debug:
            print query
        curs.execute(query)
        return curs

def check_script(upgrade, dry_run=True, debug=True):
    """
    An utility to debug upgrade scripts written in Python

    :param upgrade: upgrade procedure
    :param dry_run: if True, do not change the database
    :param debug: if True, print the queries which are executed
    """
    from openquake.engine.db.models import getcursor
    conn = WrappedConnection(getcursor('admin').connection, debug=debug)
    try:
        upgrade(conn)
    except:
        conn.rollback()
        raise
    else:
        if dry_run:
            conn.rollback()
        else:
            conn.commit()

def apply_sql_script(conn, fname):
    """
    Apply the given SQL script to the database

    :param conn: a DB API 2 connection
    :param fname: full path to the creation script
    """
    sql = open(fname).read()
    try:
        conn.cursor().execute(sql)
    except:
        logs.LOG.error('Error executing %s' % fname)
        raise

# errors are not trapped on purpose, since transactions should be managed
# in client code
class UpgradeManager(object):
    """
    The package containing the upgrade scripts should contain an instance
    of the UpgradeManager called `upgrader` in the __init__.py file. It
    should also specify the initializations parameters

    :param upgrade_dir:
        the directory were the upgrade script reside
    :param version_table:
        the name of the versioning table (default public.revision_info)
    :param version_pattern:
        a regulation expression for the script version number (\d\d\d\d)
    """
    ENGINE_URL = 'https://github.com/gem/oq-engine/tree/master/'
    UPGRADES = 'openquake/engine/db/schema/upgrades/'

    def __init__(self, upgrade_dir, version_table='public.revision_info',
                 version_pattern='\d\d\d\d', flag_pattern='(-slow|-danger)?'):
        self.upgrade_dir = upgrade_dir
        self.version_table = version_table
        self.version_pattern = version_pattern
        self.flag_pattern = flag_pattern
        self.pattern = r'^(%s)%s-([\w\-_]+)\.(sql|py)$' % (
            version_pattern, flag_pattern)
        self.upgrades_url = self.ENGINE_URL + self.UPGRADES
        if '.' in version_table:  # contains the schema name
            self.version_schema_name, self.version_table_name = \
                version_table.split('.')
        else:  # assume the schema name is 'public'
            self.version_schema_name, self.version_table_name = \
                ('public', self.version_table)
        if re.match('[\w_\.]+', version_table) is None:
            raise ValueError(version_table)
        self.starting_version = None  # will be updated after the run

    def _insert_script(self, script, conn):
        conn.cursor().execute(
            'INSERT INTO {} VALUES (%s, %s)'.format(self.version_table),
            (script['version'], script['name']))

    def install_versioning(self, conn):
        """
        Create the version table into an already populated database
        and insert the base script.

        :param conn: a DB API 2 connection
        """
        logs.LOG.info('Creating the versioning table %s', self.version_table)
        conn.cursor().execute(CREATE_VERSIONING % self.version_table)
        self._insert_script(self.read_scripts()[0], conn)

    def init(self, conn):
        """
        Create the version table and run the base script on an empty database.

        :param conn: a DB API 2 connection
        """
        base = self.read_scripts()[0]['fname']
        logs.LOG.info('Creating the initial schema from %s',  base)
        apply_sql_script(conn, os.path.join(self.upgrade_dir, base))
        self.install_versioning(conn)

    def upgrade(self, conn, skip_versions=()):
        '''
        Upgrade the database from the current version to the maximum
        version in the upgrade scripts.

        :param conn: a DBAPI 2 connection
        :param skip_versions: the versions to skip
        '''
        db_versions = self.get_db_versions(conn)
        self.starting_version = max(db_versions)
        to_skip = sorted(db_versions | set(skip_versions))
        scripts = self.read_scripts(None, None, to_skip)
        if not scripts:  # no new scripts to apply
            return []
        self.ending_version = max(s['version'] for s in scripts)
        return self._upgrade(conn, scripts)

    def _upgrade(self, conn, scripts):
        conn = WrappedConnection(conn)
        versions_applied = []
        for script in scripts:  # script is a dictionary
            fullname = os.path.join(self.upgrade_dir, script['fname'])
            logs.LOG.info('Executing %s', fullname)
            if script['ext'] == 'py':  # Python script with a upgrade(conn)
                globs = {}
                execfile(fullname, globs)
                globs['upgrade'](conn)
                self._insert_script(script, conn)
            else:  # SQL script
                # notice that this prints the file name in case of error
                apply_sql_script(conn, fullname)
                self._insert_script(script, conn)
            versions_applied.append(script['version'])
        return versions_applied

    def check_versions(self, conn):
        """
        :param conn: a DB API 2 connection
        :returns: a list with the versions that will be applied
        """
        scripts = self.read_scripts(skip_versions=self.get_db_versions(conn))
        versions = [s['version'] for s in scripts]
        if scripts:
            raise SystemExit(
                'Your database is not updated. You can update it by running '
                'oq-engine --upgrade-db which will process the '
                'following new versions: %s' % versions)
        return versions

    def get_db_versions(self, conn):
        """
        Get all the versions stored in the database as a set.

        :param conn: a DB API 2 connection
        """
        curs = conn.cursor()
        query = 'select version from {}'.format(self.version_table)
        try:
            curs.execute(query)
            return set(version for version, in curs.fetchall())
        except:
            raise VersioningNotInstalled('Run openquake --upgrade-db')

    def parse_script_name(self, script_name):
        '''
        Parse a script name and return a dictionary with fields
        fname, name, version and ext (or None if the name does not match).

        :param name: name of the script
        '''
        match = re.match(self.pattern, script_name)
        if not match:
            return
        version, flag, name, ext = match.groups()
        return dict(fname=script_name, version=version, name=name,
                    flag=flag, ext=ext, url=self.upgrades_url + script_name)

    def read_scripts(self, minversion=None, maxversion=None, skip_versions=()):
        """
        Extract the upgrade scripts from a directory as a list of
        dictionaries, ordered by version.

        :param minversion: the minimum version to consider
        :param maxversion: the maximum version to consider
        :param skipversions: the versions to skip
        """
        scripts = []
        versions = {}  # a script is unique per version
        for scriptname in sorted(os.listdir(self.upgrade_dir)):
            match = self.parse_script_name(scriptname)
            if match:
                version = match['version']
                if version in skip_versions:
                    continue  # do not collect scripts already applied
                elif minversion and version <= minversion:
                    continue  # do not collect versions too old
                elif maxversion and version > maxversion:
                    continue  # do not collect versions too new
                try:
                    previousname = versions[version]
                except KeyError:  # no previous script with the same version
                    scripts.append(match)
                    versions[version] = scriptname
                else:
                    raise DuplicatedVersion(
                        'Duplicated versions {%s,%s}' %
                        (scriptname, previousname))
        return scripts

    def extract_upgrade_scripts(self):
        """
        Extract the OpenQuake upgrade scripts from the links in the GitHub page
        """
        link_pattern = '>\s*{0}\s*<'.format(self.pattern[1:-1])
        page = urllib.urlopen(self.upgrades_url).read()
        for mo in re.finditer(link_pattern, page):
            scriptname = mo.group(0)[1:-1].strip()
            yield self.parse_script_name(scriptname)

    @classmethod
    def instance(cls, conn, pkg_name='openquake.engine.db.schema.upgrades'):
        """
        Return an :class:`UpgradeManager` instance.

        :param conn: a DB API 2 connection
        :param str pkg_name: the name of the package with the upgrade scripts
        """
        try:
            # upgrader is an UpgradeManager instance defined in the __init__.py
            upgrader = importlib.import_module(pkg_name).upgrader
        except ImportError:
            raise SystemExit(
                'Could not import %s (not in the PYTHONPATH?)' % pkg_name)
        if not upgrader.read_scripts():
            raise SystemExit(
                'The upgrade_dir does not contain scripts matching '
                'the pattern %s' % upgrader.pattern)
        curs = conn.cursor()
        # check if there is already a versioning table
        curs.execute("SELECT tablename FROM pg_tables "
                     "WHERE schemaname=%s AND tablename=%s",
                     (upgrader.version_schema_name,
                      upgrader.version_table_name))
        versioning_table = curs.fetchall()
        # if not, run the base script and create the versioning table
        if not versioning_table:
            upgrader.init(conn)
            conn.commit()
        return upgrader

def upgrade_db(conn, pkg_name='openquake.engine.db.schema.upgrades',
               skip_versions=()):
    """
    Upgrade a database by running several scripts in a single transaction.

    :param conn: a DB API 2 connection
    :param str pkg_name: the name of the package with the upgrade scripts
    :param list skip_versions: the versions to skip
    :returns: the version numbers of the new scripts applied the database
    """
    upgrader = UpgradeManager.instance(conn, pkg_name)
    t0 = time.time()
    # run the upgrade scripts
    try:
        versions_applied = upgrader.upgrade(conn, skip_versions)
    except:
        conn.rollback()
        raise
    else:
        conn.commit()
    dt = time.time() - t0
    logs.LOG.info('Upgrade completed in %s seconds', dt)
    return versions_applied

def version_db(conn, pkg_name='openquake.engine.db.schema.upgrades'):
    """
    :param conn: a DB API 2 connection
    :param str pkg_name: the name of the package with the upgrade scripts
    :returns: the current version of the database
    """
    upgrader = UpgradeManager.instance(conn, pkg_name)
    return max(upgrader.get_db_versions(conn))

def what_if_I_upgrade(conn, pkg_name='openquake.engine.db.schema.upgrades',
                      extract_scripts='extract_upgrade_scripts'):
    """
    :param conn:
        a DB API 2 connection
    :param str pkg_name:
        the name of the package with the upgrade scripts
    :param extract_scripts:
        name of the method to extract the scripts
    """
    msg_safe_ = '\nThe following scripts can be applied safely:\n%s'
    msg_slow_ = '\nPlease note that the following scripts could be slow:\n%s'
    msg_danger_ = ('\nPlease note that the following scripts are potentially '
                   'dangerous and could destroy your data:\n%s')
    upgrader = UpgradeManager.instance(conn, pkg_name)
    applied_versions = upgrader.get_db_versions(conn)
    current_version = max(applied_versions)
    slow = []
    danger = []
    safe = []
    for script in getattr(upgrader, extract_scripts)():
        url = script['url']
        if script['version'] in applied_versions:
            continue
        elif script['version'] <= current_version:
            # you cannot apply a script with a version number lower than the
            # current db version: ensure that upgrades are strictly incremental
            raise VersionTooSmall(
                'Your database is at version %s but you want to apply %s??'
                % (current_version, script['fname']))
        elif script['flag'] == '-slow':
            slow.append(url)
        elif script['flag'] == '-danger':
            danger.append(url)
        else:  # safe script
            safe.append(url)

    if not safe and not slow and not danger:
        return 'Your database is already updated at version %s.' % \
            current_version

    header = 'Your database is at version %s.' % current_version
    msg_safe = msg_safe_ % '\n'.join(safe)
    msg_slow = msg_slow_ % '\n'.join(slow)
    msg_danger = msg_danger_ % '\n'.join(danger)
    msg = header + (msg_safe if safe else '') + (msg_slow if slow else '') \
        + (msg_danger if danger else '')
    msg += ('\nClick on the links if you want to know what exactly the '
            'scripts are doing.')
    if slow:
        msg += ('\nEven slow script can be fast if your database is small or'
                ' the upgrade affects tables that are empty.')
    if danger:
        msg += ('\nEven dangerous scripts are fine if they '
                'affect empty tables or data you are not interested in.')
    return msg
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
:mod:`openquake.hazardlib.calc.stochastic` contains
:func:`stochastic_event_set`.
"""
import sys
import time
import operator
import collections
import numpy
from openquake.baselib.general import AccumDict
from openquake.baselib.performance import Monitor
from openquake.baselib.python3compat import range, raise_
from openquake.hazardlib.calc.filters import FarAwayRupture
from openquake.hazardlib.source.rupture import EBRupture
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.calc import filters

TWO32 = 2 ** 32  # 4,294,967,296
F64 = numpy.float64
U64 = numpy.uint64
U32 = numpy.uint32
U16 = numpy.uint16
event_dt = numpy.dtype([('eid', U64), ('grp_id', U16), ('ses', U32),
                        ('sample', U32)])

def stochastic_event_set(
        sources,
        sites=None,
        source_site_filter=filters.source_site_noop_filter):
    """
    Generates a 'Stochastic Event Set' (that is a collection of earthquake
    ruptures) representing a possible *realization* of the seismicity as
    described by a source model.

    The calculator loops over sources. For each source, it loops over ruptures.
    For each rupture, the number of occurrence is randomly sampled by
    calling
    :meth:`openquake.hazardlib.source.rupture.BaseProbabilisticRupture.sample_number_of_occurrences`

    .. note::
        This calculator is using random numbers. In order to reproduce the
        same results numpy random numbers generator needs to be seeded, see
        http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.seed.html

    :param sources:
        An iterator of seismic sources objects (instances of subclasses
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param sites:
        A list of sites to consider (or None)
    :param source_site_filter:
        The source filter to use (only meaningful is sites is not None)
    :param source_site_filter:
        The rupture filter to use (only meaningful is sites is not None)
    :returns:
        Generator of :class:`~openquake.hazardlib.source.rupture.Rupture`
        objects that are contained in an event set. Some ruptures can be
        missing from it, others can appear one or more times in a row.
    """
    if sites is None:  # no filtering
        for source in sources:
            try:
                for rupture in source.iter_ruptures():
                    for i in range(rupture.sample_number_of_occurrences()):
                        yield rupture
            except Exception as err:
                etype, err, tb = sys.exc_info()
                msg = 'An error occurred with source id=%s. Error: %s'
                msg %= (source.source_id, str(err))
                raise_(etype, msg, tb)
        return
    # else apply filtering
    for source, s_sites in source_site_filter(sources, sites):
        try:
            for rupture in source.iter_ruptures():
                for i in range(rupture.sample_number_of_occurrences()):
                    yield rupture
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

# ######################## rupture calculator ############################ #

def set_eids(ebruptures):
    """
    Set event IDs on the given list of ebruptures.

    :param ebruptures: a non-empty list of ruptures with the same grp_id
    :returns: the total number of events set
    """
    if not ebruptures:
        return 0
    num_events = sum(ebr.multiplicity for ebr in ebruptures)
    for ebr in ebruptures:
        assert ebr.multiplicity < TWO32, ebr.multiplicity
        eids = U64(TWO32 * ebr.serial) + numpy.arange(
            ebr.multiplicity, dtype=U64)
        ebr.events['eid'] = eids
    return num_events

def sample_ruptures(group, src_filter, gsims, param, monitor=Monitor()):
    """
    :param group:
        a SourceGroup or a sequence of sources of the same group
    :param src_filter:
        a source site filter
    :param gsims:
        a list of GSIMs for the current tectonic region model
    :param param:
        a dictionary of additional parameters
    :param monitor:
        monitor instance
    :returns:
        a dictionary with eb_ruptures, num_events, num_ruptures, calc_times
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        weight = {src: w for src, w in zip(group, group.srcs_weights)}
    else:
        weight = {src: 1 for src in group}
    eb_ruptures = []
    calc_times = []
    rup_mon = monitor('making contexts', measuremem=False)
    # Compute and save stochastic event sets
    num_ruptures = 0
    cmaker = ContextMaker(gsims, src_filter.integration_distance)
    for src, s_sites in src_filter(group):
        t0 = time.time()
        num_ruptures += src.num_ruptures
        num_occ_by_rup = _sample_ruptures(
            src, weight[src], param['ses_per_logic_tree_path'], group.samples,
            param['seed'])
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in _build_eb_ruptures(
                src, num_occ_by_rup, cmaker, s_sites, param['seed'], rup_mon):
            eb_ruptures.append(ebr)
        dt = time.time() - t0
        calc_times.append((src.id, dt))
    dic = dict(eb_ruptures=eb_ruptures, num_events=set_eids(eb_ruptures),
               calc_times=calc_times, num_ruptures=num_ruptures)
    return dic

def _sample_ruptures(src, weight, num_ses, num_samples, seed):
    """
    Sample the ruptures contained in the given source.

    :param src: a hazardlib source object
    :param weight: source weight (1 for indep sources, < 1 for mutex sources)
    :param num_ses: the number of Stochastic Event Sets to generate
    :param num_samples: how many samples for the given source
    :param seed: master seed from the job.ini file
    :returns: a dictionary of dictionaries rupture -> {ses_id: num_occurrences}
    """
    # the dictionary `num_occ_by_rup` contains a dictionary
    # ses_id -> num_occurrences for each occurring rupture
    num_occ_by_rup = collections.defaultdict(AccumDict)
    # generating ruptures for the given source
    for rup_no, rup in enumerate(src.iter_ruptures()):
        rup.seed = src.serial[rup_no] + seed
        numpy.random.seed(rup.seed)
        for sam_idx in range(num_samples):
            for ses_idx in range(1, num_ses + 1):
                num_occ = rup.sample_number_of_occurrences()
                ok_occ = numpy.random.random() < weight if weight < 1 else True
                if num_occ and ok_occ:
                    num_occ_by_rup[rup] += {(sam_idx, ses_idx): num_occ}
        rup.rup_no = rup_no + 1
    return num_occ_by_rup

def _build_eb_ruptures(
        src, num_occ_by_rup, cmaker, s_sites, random_seed, rup_mon):
    """
    Filter the ruptures stored in the dictionary num_occ_by_rup and
    yield pairs (rupture, <list of associated EBRuptures>)
    """
    for rup in sorted(num_occ_by_rup, key=operator.attrgetter('rup_no')):
        with rup_mon:
            try:
                rup.ctx = cmaker.make_contexts(s_sites, rup)
                indices = rup.ctx[0].sids
            except FarAwayRupture:
                # ignore ruptures which are far away
                del num_occ_by_rup[rup]  # save memory
                continue

        # creating EBRuptures
        serial = rup.seed - random_seed + 1
        events = []
        for (sam_idx, ses_idx), num_occ in sorted(
                num_occ_by_rup[rup].items()):
            for _ in range(num_occ):
                # NB: the 0 below is a placeholder; the right eid will be
                # set a bit later, in set_eids
                events.append((0, src.src_group_id, ses_idx, sam_idx))
        if events:
            yield EBRupture(rup, indices, numpy.array(events, event_dt),
                            serial)
<commit_msg>Small renaming<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
:mod:`openquake.hazardlib.calc.stochastic` contains
:func:`stochastic_event_set`.
"""
import sys
import time
import operator
import collections
import numpy
from openquake.baselib.general import AccumDict
from openquake.baselib.performance import Monitor
from openquake.baselib.python3compat import range, raise_
from openquake.hazardlib.calc.filters import FarAwayRupture
from openquake.hazardlib.source.rupture import EBRupture
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.calc import filters

TWO32 = 2 ** 32  # 4,294,967,296
F64 = numpy.float64
U64 = numpy.uint64
U32 = numpy.uint32
U16 = numpy.uint16
event_dt = numpy.dtype([('eid', U64), ('grp_id', U16), ('ses', U32),
                        ('sample', U32)])

def stochastic_event_set(
        sources,
        sites=None,
        source_site_filter=filters.source_site_noop_filter):
    """
    Generates a 'Stochastic Event Set' (that is a collection of earthquake
    ruptures) representing a possible *realization* of the seismicity as
    described by a source model.

    The calculator loops over sources. For each source, it loops over ruptures.
    For each rupture, the number of occurrence is randomly sampled by
    calling
    :meth:`openquake.hazardlib.source.rupture.BaseProbabilisticRupture.sample_number_of_occurrences`

    .. note::
        This calculator is using random numbers. In order to reproduce the
        same results numpy random numbers generator needs to be seeded, see
        http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.seed.html

    :param sources:
        An iterator of seismic sources objects (instances of subclasses
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param sites:
        A list of sites to consider (or None)
    :param source_site_filter:
        The source filter to use (only meaningful is sites is not None)
    :param source_site_filter:
        The rupture filter to use (only meaningful is sites is not None)
    :returns:
        Generator of :class:`~openquake.hazardlib.source.rupture.Rupture`
        objects that are contained in an event set. Some ruptures can be
        missing from it, others can appear one or more times in a row.
    """
    if sites is None:  # no filtering
        for source in sources:
            try:
                for rupture in source.iter_ruptures():
                    for i in range(rupture.sample_number_of_occurrences()):
                        yield rupture
            except Exception as err:
                etype, err, tb = sys.exc_info()
                msg = 'An error occurred with source id=%s. Error: %s'
                msg %= (source.source_id, str(err))
                raise_(etype, msg, tb)
        return
    # else apply filtering
    for source, s_sites in source_site_filter(sources, sites):
        try:
            for rupture in source.iter_ruptures():
                for i in range(rupture.sample_number_of_occurrences()):
                    yield rupture
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, str(err))
            raise_(etype, msg, tb)

# ######################## rupture calculator ############################ #

def set_eids(ebruptures):
    """
    Set event IDs on the given list of ebruptures.

    :param ebruptures: a non-empty list of ruptures with the same grp_id
    :returns: the total number of events set
    """
    if not ebruptures:
        return 0
    num_events = sum(ebr.multiplicity for ebr in ebruptures)
    for ebr in ebruptures:
        assert ebr.multiplicity < TWO32, ebr.multiplicity
        eids = U64(TWO32 * ebr.serial) + numpy.arange(
            ebr.multiplicity, dtype=U64)
        ebr.events['eid'] = eids
    return num_events

def sample_ruptures(group, src_filter, gsims, param, monitor=Monitor()):
    """
    :param group:
        a SourceGroup or a sequence of sources of the same group
    :param src_filter:
        a source site filter
    :param gsims:
        a list of GSIMs for the current tectonic region model
    :param param:
        a dictionary of additional parameters
    :param monitor:
        monitor instance
    :returns:
        a dictionary with eb_ruptures, num_events, num_ruptures, calc_times
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        prob = {src: sw for src, sw in zip(group, group.srcs_weights)}
    else:
        prob = {src: 1 for src in group}
    eb_ruptures = []
    calc_times = []
    rup_mon = monitor('making contexts', measuremem=False)
    # Compute and save stochastic event sets
    num_ruptures = 0
    cmaker = ContextMaker(gsims, src_filter.integration_distance)
    for src, s_sites in src_filter(group):
        t0 = time.time()
        num_ruptures += src.num_ruptures
        num_occ_by_rup = _sample_ruptures(
            src, prob[src], param['ses_per_logic_tree_path'], group.samples,
            param['seed'])
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in _build_eb_ruptures(
                src, num_occ_by_rup, cmaker, s_sites, param['seed'], rup_mon):
            eb_ruptures.append(ebr)
        dt = time.time() - t0
        calc_times.append((src.id, dt))
    dic = dict(eb_ruptures=eb_ruptures, num_events=set_eids(eb_ruptures),
               calc_times=calc_times, num_ruptures=num_ruptures)
    return dic

def _sample_ruptures(src, prob, num_ses, num_samples, seed):
    """
    Sample the ruptures contained in the given source.

    :param src: a hazardlib source object
    :param prob: a probability (1 for indep sources, < 1 for mutex sources)
    :param num_ses: the number of Stochastic Event Sets to generate
    :param num_samples: how many samples for the given source
    :param seed: master seed from the job.ini file
    :returns: a dictionary of dictionaries rupture -> {ses_id: num_occurrences}
    """
    # the dictionary `num_occ_by_rup` contains a dictionary
    # ses_id -> num_occurrences for each occurring rupture
    num_occ_by_rup = collections.defaultdict(AccumDict)
    # generating ruptures for the given source
    for rup_no, rup in enumerate(src.iter_ruptures()):
        rup.seed = src.serial[rup_no] + seed
        numpy.random.seed(rup.seed)
        for sam_idx in range(num_samples):
            for ses_idx in range(1, num_ses + 1):
                num_occ = rup.sample_number_of_occurrences()
                ok_occ = numpy.random.random() < prob if prob < 1 else True
                if num_occ and ok_occ:
                    num_occ_by_rup[rup] += {(sam_idx, ses_idx): num_occ}
        rup.rup_no = rup_no + 1
    return num_occ_by_rup

def _build_eb_ruptures(
        src, num_occ_by_rup, cmaker, s_sites, random_seed, rup_mon):
    """
    Filter the ruptures stored in the dictionary num_occ_by_rup and
    yield pairs (rupture, <list of associated EBRuptures>)
    """
    for rup in sorted(num_occ_by_rup, key=operator.attrgetter('rup_no')):
        with rup_mon:
            try:
                rup.ctx = cmaker.make_contexts(s_sites, rup)
                indices = rup.ctx[0].sids
            except FarAwayRupture:
                # ignore ruptures which are far away
                del num_occ_by_rup[rup]  # save memory
                continue

        # creating EBRuptures
        serial = rup.seed - random_seed + 1
        events = []
        for (sam_idx, ses_idx), num_occ in sorted(
                num_occ_by_rup[rup].items()):
            for _ in range(num_occ):
                # NB: the 0 below is a placeholder; the right eid will be
                # set a bit later, in set_eids
                events.append((0, src.src_group_id, ses_idx, sam_idx))
        if events:
            yield EBRupture(rup, indices, numpy.array(events, event_dt),
                            serial)
<|endoftext|>