<commit_before>"""
Django settings for ittc project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '9u$pbamv*a1s09(5grvnko2)n)isa50=uui@lm3syhp6)jyrhg'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

# Note that Django automatically includes the "templates" dir in all the
# INSTALLED_APPS, se there is no need to add maps/templates or admin/templates
TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, "ittc/templates"),
)

ALLOWED_HOSTS = []

# Application definition

ITTC_APPS = (
    'ittc.capabilities',
    'ittc.cache',
    'ittc.proxy',
    'ittc.source',
)

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Theme
    #"pinax_theme_bootstrap_account",
    "pinax_theme_bootstrap",
    #'django_forms_bootstrap',

    'corsheaders',
    'leaflet',
    'jquery',

) + ITTC_APPS

MIDDLEWARE_CLASSES = (
    'corsheaders.middleware.CorsMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'ittc.urls'

WSGI_APPLICATION = 'ittc.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/
#STATIC_ROOT = os.path.join(BASE_DIR, "static_root")
STATIC_ROOT = '/var/www/ittc/static/'
STATIC_URL = '/static/'
#STATICFILES_DIRS = [
#    os.path.join(BASE_DIR, "static"),
#]
#STATICFILES_FINDERS = (
#    'django.contrib.staticfiles.finders.FileSystemFinder',
#    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#)

LEAFLET_CONFIG = {
    'TILES': [
        # Find tiles at:
        # http://leaflet-extras.github.io/leaflet-providers/preview/

        ('OpenStreetMap',
         'http://{s}.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png',
         '© OpenStreetMap, Tiles courtesy of Humanitarian OpenStreetMap Team'),
    ]
}

CACHES = {
    'default': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11211',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    },
    'tiles': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11212',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    },
    'celery_results': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11213',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    }
}

CELERY_CACHE_BACKEND = 'default'

###Settings ITTC Capabilities
ITTC_SERVER = {
    'name': 'HIU Imagery Services',
    'cache': {
        'memory': {
            'enabled': True,
            'size': 1000,
            'minZoom': 0,
            'maxZoom': 16,
            'expiration': 'origin'
        }
    },
    'heuristic': {
        'down': {
            'enabled': True,
            'depth': 1,
            'minZoom': 0,
            'maxZoom': 18
        },
        'up': {
            'enabled': True
        },
        'nearby': {
            'enabled': True,
            'radius': 2
        }
    }
}

SITEURL = "http://localhost:8000/"

CORS_ORIGIN_ALLOW_ALL = True

PROXY_ALLOWED_HOSTS = ( 'tile.openstreetmap.org', 'tile.openstreetmap.fr', 'tiles.virtualearth.net', 'tiles.mapbox.com', 'hiu-maps.net' )

PROXY_URL = '/proxy/?url='

CELERY_RESULT_BACKEND = 'cache+memcached://127.0.0.1:11213/'
BROKER_URL = 'amqp://guest:guest@localhost:5672//'

LOG_ROOT = BASE_DIR+'/logs'

LOG_FORMAT = {
    'tile_request': '{status}	{tilesource}	{z}	{x}	{y}	{ip}	{datetime}'
}

LOG_COLLECTION = 'logs'

CUSTOM_STATS = [
    {'name': 'total', 'collection': 'stats_total', 'attributes': []},
    {'name': 'by_ip', 'collection': 'stats_by_ip', 'attributes': ['ip']},
    {'name': 'by_source', 'collection': 'stats_by_source', 'attributes': ['source']},
    {'name': 'by_location', 'collection': 'stats_by_location', 'attributes': ['location']},
    {'name': 'by_zoom', 'collection': 'stats_by_zoom', 'attributes': ['z']},
    {'name': 'by_status', 'collection': 'stats_by_status', 'attributes': ['status']},
    {'name': 'by_year', 'collection': 'stats_by_year', 'attributes': ['year']},
    {'name': 'by_month', 'collection': 'stats_by_month', 'attributes': ['month']},
    {'name': 'by_date', 'collection': 'stats_by_date', 'attributes': ['date']},

    {'name': 'by_year_source', 'collection': 'stats_by_year_source', 'attributes': ['year', 'source']},
    {'name': 'by_date_source', 'collection': 'stats_by_date_source', 'attributes': ['date', 'source']},
    {'name': 'by_ip_source', 'collection': 'stats_by_ip_source', 'attributes': ['ip', 'source']},
    {'name': 'by_source_status', 'collection': 'stats_by_source_status', 'attributes': ['source', 'status']},
    {'name': 'by_month_source', 'collection': 'stats_by_month_source', 'attributes': ['month', 'source']},
    {'name': 'by_zoom_status', 'collection': 'stats_by_zoom_status', 'attributes': ['z', 'status']},
    {'name': 'by_date_location', 'collection': 'stats_by_date_location', 'attributes': ['date', 'location']},
    {'name': 'by_source_location', 'collection': 'stats_by_source_location', 'attributes': ['source', 'location']},
    {'name': 'by_source_date_location', 'collection': 'stats_by_source_date_location', 'attributes': ['source', 'date', 'location']},
]
<commit_msg>Update settings.py<commit_after>"""
Django settings for ittc project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '9u$pbamv*a1s09(5grvnko2)n)isa50=uui@lm3syhp6)jyrhg'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

# Note that Django automatically includes the "templates" dir in all the
# INSTALLED_APPS, se there is no need to add maps/templates or admin/templates
TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, "ittc/templates"),
)

ALLOWED_HOSTS = []

# Application definition

ITTC_APPS = (
    'ittc.capabilities',
    'ittc.cache',
    'ittc.proxy',
    'ittc.source',
)

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Theme
    #"pinax_theme_bootstrap_account",
    "pinax_theme_bootstrap",
    #'django_forms_bootstrap',

    'corsheaders',
    #'leaflet',
    'jquery',

) + ITTC_APPS

MIDDLEWARE_CLASSES = (
    'corsheaders.middleware.CorsMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'ittc.urls'

WSGI_APPLICATION = 'ittc.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/
#STATIC_ROOT = os.path.join(BASE_DIR, "static_root")
STATIC_ROOT = '/var/www/ittc/static/'
STATIC_URL = '/static/'
#STATICFILES_DIRS = [
#    os.path.join(BASE_DIR, "static"),
#]
#STATICFILES_FINDERS = (
#    'django.contrib.staticfiles.finders.FileSystemFinder',
#    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#)

#LEAFLET_CONFIG = {
#    'TILES': [
#        # Find tiles at:
#        # http://leaflet-extras.github.io/leaflet-providers/preview/
#
#        ('OpenStreetMap',
#         'http://{s}.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png',
#         '© OpenStreetMap, Tiles courtesy of Humanitarian OpenStreetMap Team'),
#    ]
#}

CACHES = {
    'default': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11211',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    },
    'tiles': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11212',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    },
    'celery_results': {
        'BACKEND': 'memcachepool.cache.UMemcacheCache',
        'LOCATION': '127.0.0.1:11213',
        'OPTIONS': {
            'MAX_POOL_SIZE': 40,
            'BLACKLIST_TIME': 60,
            'SOCKET_TIMEOUT': 5,
            'MAX_ITEM_SIZE': 1000*1000*1000
        }
    }
}

CELERY_CACHE_BACKEND = 'default'

###Settings ITTC Capabilities
ITTC_SERVER = {
    'name': 'HIU Imagery Services',
    'cache': {
        'memory': {
            'enabled': True,
            'size': 1000,
            'minZoom': 0,
            'maxZoom': 16,
            'expiration': 'origin'
        }
    },
    'heuristic': {
        'down': {
            'enabled': True,
            'depth': 1,
            'minZoom': 0,
            'maxZoom': 18
        },
        'up': {
            'enabled': True
        },
        'nearby': {
            'enabled': True,
            'radius': 2
        }
    }
}

SITEURL = "http://localhost:8000/"

CORS_ORIGIN_ALLOW_ALL = True

PROXY_ALLOWED_HOSTS = ( 'tile.openstreetmap.org', 'tile.openstreetmap.fr', 'tiles.virtualearth.net', 'tiles.mapbox.com', 'hiu-maps.net' )

PROXY_URL = '/proxy/?url='

CELERY_RESULT_BACKEND = 'cache+memcached://127.0.0.1:11213/'
BROKER_URL = 'amqp://guest:guest@localhost:5672//'

LOG_ROOT = BASE_DIR+'/logs'

LOG_FORMAT = {
    'tile_request': '{status}	{tilesource}	{z}	{x}	{y}	{ip}	{datetime}'
}

LOG_COLLECTION = 'logs'

CUSTOM_STATS = [
    {'name': 'total', 'collection': 'stats_total', 'attributes': []},
    {'name': 'by_ip', 'collection': 'stats_by_ip', 'attributes': ['ip']},
    {'name': 'by_source', 'collection': 'stats_by_source', 'attributes': ['source']},
    {'name': 'by_location', 'collection': 'stats_by_location', 'attributes': ['location']},
    {'name': 'by_zoom', 'collection': 'stats_by_zoom', 'attributes': ['z']},
    {'name': 'by_status', 'collection': 'stats_by_status', 'attributes': ['status']},
    {'name': 'by_year', 'collection': 'stats_by_year', 'attributes': ['year']},
    {'name': 'by_month', 'collection': 'stats_by_month', 'attributes': ['month']},
    {'name': 'by_date', 'collection': 'stats_by_date', 'attributes': ['date']},

    {'name': 'by_year_source', 'collection': 'stats_by_year_source', 'attributes': ['year', 'source']},
    {'name': 'by_date_source', 'collection': 'stats_by_date_source', 'attributes': ['date', 'source']},
    {'name': 'by_ip_source', 'collection': 'stats_by_ip_source', 'attributes': ['ip', 'source']},
    {'name': 'by_source_status', 'collection': 'stats_by_source_status', 'attributes': ['source', 'status']},
    {'name': 'by_month_source', 'collection': 'stats_by_month_source', 'attributes': ['month', 'source']},
    {'name': 'by_zoom_status', 'collection': 'stats_by_zoom_status', 'attributes': ['z', 'status']},
    {'name': 'by_date_location', 'collection': 'stats_by_date_location', 'attributes': ['date', 'location']},
    {'name': 'by_source_location', 'collection': 'stats_by_source_location', 'attributes': ['source', 'location']},
    {'name': 'by_source_date_location', 'collection': 'stats_by_source_date_location', 'attributes': ['source', 'date', 'location']},
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '1.3.1'
<commit_msg>Bump version to 1.4.0<commit_after># -*- coding: utf-8 -*-

__version__ = '1.4.0'
<|endoftext|>


<commit_before>"""
Library for Spunky Bot
http://www.spunkybot.de
Author: Alexander Kress

This program is released under the MIT License.
"""

__version__ = '1.0.3'

### IMPORTS
import time
from threading import Thread
from threading import RLock

### CLASS Rules ###
class Rules(object):
    """
    Display the rules
    """

    def __init__(self, rules_file, rules_frequency, rcon_handle):
        """
        create a new instance of Rules

        @param rules_file: The full path of the rules.conf file
        @type  rules_file: String
        @param rules_frequency: The frequency of rules iteration in seconds
        @type  rules_frequency: Integer
        @param rcon_handle: RCON handler instance
        @type  rcon_handle: Instance
        """
        self.rules_file = rules_file
        if rules_frequency < 5:
            # avoid flooding with too less delay
            self.rules_frequency = 5
        else:
            self.rules_frequency = rules_frequency
        self.rcon_handle = rcon_handle
        self.rcon_lock = RLock()
        # start Thread
        self.processor = Thread(target=self.process)
        self.processor.setDaemon(True)
        self.processor.start()

    def process(self):
        """
        Thread process
        """
        # initial wait
        time.sleep(30)
        while 1:
            filehandle = open(self.rules_file, 'r+')
            rotation_msg = filehandle.readlines()
            if not rotation_msg:
                filehandle.close()
                break
            for line in rotation_msg:
                # display rule
                with self.rcon_lock:
                    self.rcon_handle.push("say ^2%s" % line.strip())
                # wait for given delay in the config file
                time.sleep(self.rules_frequency)
            filehandle.close()
<commit_msg>Code improvement<commit_after>"""
Library for Spunky Bot
http://www.spunkybot.de
Author: Alexander Kress

This program is released under the MIT License.
"""

__version__ = '1.0.4'

### IMPORTS
import time
from threading import Thread
from threading import RLock

### CLASS Rules ###
class Rules(object):
    """
    Display the rules
    """

    def __init__(self, rules_file, rules_frequency, rcon_handle):
        """
        create a new instance of Rules

        @param rules_file: The full path of the rules.conf file
        @type  rules_file: String
        @param rules_frequency: The frequency of rules iteration in seconds
        @type  rules_frequency: Integer
        @param rcon_handle: RCON handler instance
        @type  rcon_handle: Instance
        """
        self.rules_file = rules_file
        if rules_frequency < 5:
            # avoid flooding with too less delay
            self.rules_frequency = 5
        else:
            self.rules_frequency = rules_frequency
        self.rcon_handle = rcon_handle
        self.rcon_lock = RLock()
        # start Thread
        self.processor = Thread(target=self.process)
        self.processor.setDaemon(True)
        self.processor.start()

    def process(self):
        """
        Thread process
        """
        # initial wait
        time.sleep(30)
        while 1:
            with open(self.rules_file, 'r') as filehandle:
                rotation_msg = filehandle.readlines()
            if not rotation_msg:
                break
            for line in rotation_msg:
                # display rule
                with self.rcon_lock:
                    self.rcon_handle.push("say ^2%s" % line.strip())
                # wait for given delay in the config file
                time.sleep(self.rules_frequency)
<|endoftext|>


<commit_before>"""Command-line utilities for storage subsystem."""

import pprint

from jacquard.commands import BaseCommand, CommandError

from .utils import retrying, copy_data, open_engine

class StorageDump(BaseCommand):
    """
    Dump everything in the store to stdout.

    This is a debugging command, mainly for developer use.
    """

    plumbing = True
    help = "dump all objects in storage"

    def handle(self, config, options):
        """Run command."""
        with config.storage.transaction(read_only=True) as store:
            for key, value in store.items():
                print(key)
                print('=' * len(key))
                pprint.pprint(value)
                print()

class StorageImport(BaseCommand):
    """
    Import all data from a given engine.

    This is a mechanism to make migrations between storage engines easier,
    when the time comes to upgrade. Also useful for restoring backups.

    Note that this does not flush the current store before importing, it is
    wise to do so manually.
    """

    help = "load stored data from another storage engine"

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument('engine', help="storage engine to load from")
        parser.add_argument('url', help="storage URL to load from")

    @retrying
    def handle(self, config, option):
        """Run command."""
        src = open_engine(config, option.engine, option.url)
        copy_data(src, config.storage)

class StorageExport(BaseCommand):
    """
    Export all data to a given engine.

    This is a mechanism mainly for backups, although it may also be useful for
    migrations.
    """

    help = "export stored data to another storage engine"

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument('engine', help="storage engine to save to")
        parser.add_argument('url', help="storage URL to save to")
        parser.add_argument(
            '--flush',
            action='store_true',
            help="erase storage destination before writing",
        )

    @retrying
    def handle(self, config, option):
        """Run command."""
        dst = open_engine(config, option.engine, option.url)
        copy_data(config.storage, dst, flush=option.flush)

class StorageFlush(BaseCommand):
    """
    Delete every key from the current store.

    A command which should be used with care for obvious reasons! Mainly useful
    before imports.

    There is an open question as to whether it would be wise to make this the
    default behaviour of importing.
    """

    help = "clear everything in storage"
    plumbing = True

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument(
            '--force',
            action='store_true',
            help=(
                "force Jacquard to actually do this despite its being "
                "obviously a bad idea"
            ),
        )

    @retrying
    def handle(self, config, option):
        """Run command."""
        if not option.force:
            raise CommandError(
                "This command would erase the database. If you are really "
                "sure this is something you want to do, add --force.",
            )

        with config.storage.transaction() as store:
            store.clear()
<commit_msg>Add flush argument to `storage-import`<commit_after>"""Command-line utilities for storage subsystem."""

import pprint

from jacquard.commands import BaseCommand, CommandError

from .utils import retrying, copy_data, open_engine

class StorageDump(BaseCommand):
    """
    Dump everything in the store to stdout.

    This is a debugging command, mainly for developer use.
    """

    plumbing = True
    help = "dump all objects in storage"

    def handle(self, config, options):
        """Run command."""
        with config.storage.transaction(read_only=True) as store:
            for key, value in store.items():
                print(key)
                print('=' * len(key))
                pprint.pprint(value)
                print()

class StorageImport(BaseCommand):
    """
    Import all data from a given engine.

    This is a mechanism to make migrations between storage engines easier,
    when the time comes to upgrade. Also useful for restoring backups.
    """

    help = "load stored data from another storage engine"

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument('engine', help="storage engine to load from")
        parser.add_argument('url', help="storage URL to load from")
        parser.add_argument(
            '--flush',
            action='store_true',
            help="flush out the previous data in storage",
        )

    @retrying
    def handle(self, config, option):
        """Run command."""
        src = open_engine(config, option.engine, option.url)
        copy_data(src, config.storage, flush=option.flush)

class StorageExport(BaseCommand):
    """
    Export all data to a given engine.

    This is a mechanism mainly for backups, although it may also be useful for
    migrations.
    """

    help = "export stored data to another storage engine"

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument('engine', help="storage engine to save to")
        parser.add_argument('url', help="storage URL to save to")
        parser.add_argument(
            '--flush',
            action='store_true',
            help="erase storage destination before writing",
        )

    @retrying
    def handle(self, config, option):
        """Run command."""
        dst = open_engine(config, option.engine, option.url)
        copy_data(config.storage, dst, flush=option.flush)

class StorageFlush(BaseCommand):
    """
    Delete every key from the current store.

    A command which should be used with care for obvious reasons! Mainly useful
    before imports.

    There is an open question as to whether it would be wise to make this the
    default behaviour of importing.
    """

    help = "clear everything in storage"
    plumbing = True

    def add_arguments(self, parser):
        """Add argparse arguments."""
        parser.add_argument(
            '--force',
            action='store_true',
            help=(
                "force Jacquard to actually do this despite its being "
                "obviously a bad idea"
            ),
        )

    @retrying
    def handle(self, config, option):
        """Run command."""
        if not option.force:
            raise CommandError(
                "This command would erase the database. If you are really "
                "sure this is something you want to do, add --force.",
            )

        with config.storage.transaction() as store:
            store.clear()
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core calculator functionality for computing stochastic event sets and ground
motion fields using the 'event-based' method.

Stochastic events sets (which can be thought of as collections of ruptures) are
computed iven a set of seismic sources and investigation time span (in years).

For more information on computing stochastic event sets, see
:mod:`openquake.hazardlib.calc.stochastic`.

One can optionally compute a ground motion field (GMF) given a rupture, a site
collection (which is a collection of geographical points with associated soil
parameters), and a ground shaking intensity model (GSIM).

For more information on computing ground motion fields, see
:mod:`openquake.hazardlib.calc.gmf`.
"""

import random
import collections

import numpy.random

from django.db import transaction
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc import gmf
from openquake.hazardlib.imt import from_string

from openquake.engine import writer
from openquake.engine.calculators.hazard import general
from openquake.engine.calculators.hazard.classical import (
    post_processing as cls_post_proc)
from openquake.engine.calculators.hazard.event_based import post_processing
from openquake.engine.db import models
from openquake.engine.utils import tasks
from openquake.engine.performance import EnginePerformanceMonitor, LightMonitor

#: Always 1 for the computation of ground motion fields in the event-based
#: hazard calculator.
DEFAULT_GMF_REALIZATIONS = 1

# NB: beware of large caches
inserter = writer.CacheInserter(models.GmfData, 1000)

@tasks.oqtask
def compute_ses_and_gmfs(job_id, src_seeds, gsims_by_rlz, task_no):
    """
    Celery task for the stochastic event set calculator.

    Samples logic trees and calls the stochastic event set calculator.

    Once stochastic event sets are calculated, results will be saved to the
    database. See :class:`openquake.engine.db.models.SESCollection`.

    Optionally (specified in the job configuration using the
    `ground_motion_fields` parameter), GMFs can be computed from each rupture
    in each stochastic event set. GMFs are also saved to the database.

    :param int job_id:
        ID of the currently running job.
    :param src_seeds:
        List of pairs (source, seed)
    :params gsims_by_rlz:
        dictionary of GSIM
    :param task_no:
        an ordinal so that GMV can be collected in a reproducible order
    """
    rlz_ids = [r.id for r in gsims_by_rlz]
    ses_coll = models.SESCollection.objects.get(lt_realization_ids=rlz_ids)

    rnd = random.Random()
    all_ses = models.SES.objects.filter(ses_collection=ses_coll)
    ruptures = []
    ses_ruptures = []
    rupture_seeds = []

    mon1 = LightMonitor('generating ruptures', job_id, compute_ses_and_gmfs)
    mon2 = LightMonitor('sampling ruptures', job_id, compute_ses_and_gmfs)

    # Compute and save stochastic event sets
    for src, seed in src_seeds:
        rnd.seed(seed)
        with mon1:
            rupts = list(src.iter_ruptures())

        for ses in all_ses:
            numpy.random.seed(rnd.randint(0, models.MAX_SINT_32))
            for i, r in enumerate(rupts):
                with mon2:
                    for j in xrange(r.sample_number_of_occurrences()):
                        rup = models.SESRupture(
                            ses=ses,
                            rupture=r,
                            tag='smlt=%02d|ses=%04d|src=%s|i=%04d-%02d' % (
                                ses_coll.ordinal,
                                ses.ordinal, src.source_id, i, j),
                            hypocenter=r.hypocenter.wkt2d,
                            magnitude=r.mag,
                        )
                        rup_seed = rnd.randint(0, models.MAX_SINT_32)
                        ruptures.append(r)
                        ses_ruptures.append(rup)
                        rupture_seeds.append(rup_seed)

    mon1.flush()
    mon2.flush()
    if not ruptures:
        return

    with EnginePerformanceMonitor('saving ses', job_id, compute_ses_and_gmfs):
        rupture_ids = _save_ses_ruptures(ses_ruptures)

    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    if not hc.ground_motion_fields:
        return

    # compute GMFs
    imts = map(from_string, hc.intensity_measure_types)
    params = dict(
        correl_model=general.get_correl_model(hc),
        truncation_level=hc.truncation_level,
        maximum_distance=hc.maximum_distance)

    for rlz, gsims in gsims_by_rlz.items():
        with EnginePerformanceMonitor(
                'computing gmfs', job_id, compute_ses_and_gmfs):
            gmvs_per_site, ruptures_per_site = _compute_gmf(
                params, imts, gsims, hc.site_collection,
                zip(ruptures, rupture_ids, rupture_seeds))

        with EnginePerformanceMonitor(
                'saving gmfs', job_id, compute_ses_and_gmfs):
            gmf_coll = models.Gmf.objects.get(lt_realization=rlz)
            _save_gmfs(gmf_coll, gmvs_per_site, ruptures_per_site,
                       hc.site_collection, task_no)

def _save_ses_ruptures(ruptures):
    """
    Helper function for saving stochastic event set ruptures to the database.
    :param ruptures:
        A list of :class:`openquake.engine.db.models.SESRupture` instances.
    """
    # TODO: Possible future optimization:
    # Refactor this to do bulk insertion of ruptures
    rupture_ids = []
    with transaction.commit_on_success(using='job_init'):
        for r in ruptures:
            r.save()
            rupture_ids.append(r.id)
    return rupture_ids

# NB: I tried to return a single dictionary {site_id: [(gmv, rupt_id),...]}
# but it takes a lot more memory (MS)
def _compute_gmf(params, imts, gsims, site_coll, rupture_id_seed_triples):
    """
    Compute a ground motion field value for each rupture, for all the
    points affected by that rupture, for the given IMT. Returns a
    dictionary with the nonzero contributions to each site id, and a dictionary
    with the ids of the contributing ruptures for each site id.
    assert len(ruptures) == len(rupture_seeds)

    :param params:
        a dictionary containing the keys
        correl_model, truncation_level, maximum_distance
    :param imts:
        a list of hazardlib intensity measure types
    :param gsims:
        a dictionary {tectonic region type -> GSIM instance}
    :param site_coll:
        a SiteCollection instance
    :param rupture_id_seed_triple:
        a list of triples with types
        (:class:`openquake.hazardlib.source.rupture.Rupture`, int, int)
    """
    gmvs_per_site = collections.defaultdict(list)
    ruptures_per_site = collections.defaultdict(list)

    # Compute and save ground motion fields
    for i, (rupture, rup_id, rup_seed) in enumerate(rupture_id_seed_triples):
        gmf_calc_kwargs = {
            'rupture': rupture,
            'sites': site_coll,
            'imts': imts,
            'gsim': gsims[rupture.tectonic_region_type],
            'truncation_level': params['truncation_level'],
            'realizations': DEFAULT_GMF_REALIZATIONS,
            'correlation_model': params['correl_model'],
            'rupture_site_filter': filters.rupture_site_distance_filter(
                params['maximum_distance']),
        }
        numpy.random.seed(rup_seed)
        gmf_dict = gmf.ground_motion_fields(**gmf_calc_kwargs)
        for imt, gmf_1_realiz in gmf_dict.iteritems():
            # since DEFAULT_GMF_REALIZATIONS is 1, gmf_1_realiz is a matrix
            # with n_sites rows and 1 column
            for site, gmv in zip(site_coll, gmf_1_realiz):
                gmv = float(gmv)  # convert a 1x1 matrix into a float
                if gmv:  # nonzero contribution to site
                    gmvs_per_site[imt, site.id].append(gmv)
                    ruptures_per_site[imt, site.id].append(rup_id)
    return gmvs_per_site, ruptures_per_site

@transaction.commit_on_success(using='job_init')
def _save_gmfs(gmf, gmvs_per_site, ruptures_per_site, sites, task_no):
    """
    Helper method to save computed GMF data to the database.

    :param gmf:
        The Gmf instance where to save
    :param gmf_per_site:
        The GMFs per rupture
    :param rupture_per_site:
        The associated rupture ids
    :param sites:
        An :class:`openquake.hazardlib.site.SiteCollection` object,
        representing the sites of interest for a calculation.
    :param task_no:
        The ordinal of the task which generated the current GMFs to save
    """
    for imt, site_id in gmvs_per_site:
        imt_name, sa_period, sa_damping = imt
        inserter.add(models.GmfData(
            gmf=gmf,
            task_no=task_no,
            imt=imt_name,
            sa_period=sa_period,
            sa_damping=sa_damping,
            site_id=site_id,
            gmvs=gmvs_per_site[imt, site_id],
            rupture_ids=ruptures_per_site[imt, site_id]))
    inserter.flush()

class EventBasedHazardCalculator(general.BaseHazardCalculator):
    """
    Probabilistic Event-Based hazard calculator. Computes stochastic event sets
    and (optionally) ground motion fields.
    """
    core_calc_task = compute_ses_and_gmfs

    def task_arg_gen(self, _block_size=None):
        """
        Loop through realizations and sources to generate a sequence of
        task arg tuples. Each tuple of args applies to a single task.
        Yielded results are tuples of the form job_id, sources, ses, seeds
        (seeds will be used to seed numpy for temporal occurence sampling).
        """
        hc = self.hc
        rnd = random.Random()
        rnd.seed(hc.random_seed)
        task_no = 0
        for job_id, block, gsims_by_rlz in super(
                EventBasedHazardCalculator, self).task_arg_gen():
            ss = [(src, rnd.randint(0, models.MAX_SINT_32))
                  for src in block]  # source, seed pairs
            yield job_id, ss, gsims_by_rlz, task_no
            task_no += 1

        # now the source_blocks_per_ltpath dictionary can be cleared
        self.source_blocks_per_ltpath.clear()

    def initialize_ses_db_records(self, ordinal, rlzs):
        """
        Create :class:`~openquake.engine.db.models.Output`,
        :class:`~openquake.engine.db.models.SESCollection` and
        :class:`~openquake.engine.db.models.SES` "container" records for
        a single realization.

        Stochastic event set ruptures computed for this realization will be
        associated to these containers.

        NOTE: Many tasks can contribute ruptures to the same SES.
        """
        rlz_ids = [r.id for r in rlzs]

        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection smlt-%d-rlz-%s' % (
                ordinal, ','.join(map(str, rlz_ids))),
            output_type='ses')

        ses_coll = models.SESCollection.objects.create(
            output=output, lt_realization_ids=rlz_ids, ordinal=ordinal)

        for rlz in rlzs:
            if self.job.hazard_calculation.ground_motion_fields:
                output = models.Output.objects.create(
                    oq_job=self.job,
                    display_name='GMF rlz-%s' % rlz.id,
                    output_type='gmf')
                models.Gmf.objects.create(output=output, lt_realization=rlz)

        all_ses = []
        for i in xrange(1, self.hc.ses_per_logic_tree_path + 1):
            all_ses.append(
                models.SES.objects.create(
                    ses_collection=ses_coll,
                    investigation_time=self.hc.investigation_time,
                    ordinal=i))
        return all_ses

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        super(EventBasedHazardCalculator, self).pre_execute()
        for i, rlzs in enumerate(self.rlzs_per_ltpath.itervalues()):
            self.initialize_ses_db_records(i, rlzs)

    def post_process(self):
        """
        If requested, perform additional processing of GMFs to produce hazard
        curves.
        """
        if self.hc.hazard_curves_from_gmfs:
            with EnginePerformanceMonitor('generating hazard curves',
                                          self.job.id):
                self.parallelize(
                    post_processing.gmf_to_hazard_curve_task,
                    post_processing.gmf_to_hazard_curve_arg_gen(self.job),
                    self.log_percent)

            # If `mean_hazard_curves` is True and/or `quantile_hazard_curves`
            # has some value (not an empty list), do this additional
            # post-processing.
            if self.hc.mean_hazard_curves or self.hc.quantile_hazard_curves:
                with EnginePerformanceMonitor(
                        'generating mean/quantile curves', self.job.id):
                    self.do_aggregate_post_proc()

            if self.hc.hazard_maps:
                with EnginePerformanceMonitor(
                        'generating hazard maps', self.job.id):
                    self.parallelize(
                        cls_post_proc.hazard_curves_to_hazard_map_task,
                        cls_post_proc.hazard_curves_to_hazard_map_task_arg_gen(
                            self.job),
                        self.log_percent)
<commit_msg>Added source and rupture filtering<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core calculator functionality for computing stochastic event sets and ground
motion fields using the 'event-based' method.

Stochastic events sets (which can be thought of as collections of ruptures) are
computed iven a set of seismic sources and investigation time span (in years).

For more information on computing stochastic event sets, see
:mod:`openquake.hazardlib.calc.stochastic`.

One can optionally compute a ground motion field (GMF) given a rupture, a site
collection (which is a collection of geographical points with associated soil
parameters), and a ground shaking intensity model (GSIM).

For more information on computing ground motion fields, see
:mod:`openquake.hazardlib.calc.gmf`.
"""

import random
import collections

import numpy.random

from django.db import transaction
from openquake.hazardlib.calc import filters
from openquake.hazardlib.calc import gmf
from openquake.hazardlib.imt import from_string

from openquake.engine import writer
from openquake.engine.calculators.hazard import general
from openquake.engine.calculators.hazard.classical import (
    post_processing as cls_post_proc)
from openquake.engine.calculators.hazard.event_based import post_processing
from openquake.engine.db import models
from openquake.engine.utils import tasks
from openquake.engine.performance import EnginePerformanceMonitor, LightMonitor

#: Always 1 for the computation of ground motion fields in the event-based
#: hazard calculator.
DEFAULT_GMF_REALIZATIONS = 1

# NB: beware of large caches
inserter = writer.CacheInserter(models.GmfData, 1000)

@tasks.oqtask
def compute_ses_and_gmfs(job_id, src_seeds, gsims_by_rlz, task_no):
    """
    Celery task for the stochastic event set calculator.

    Samples logic trees and calls the stochastic event set calculator.

    Once stochastic event sets are calculated, results will be saved to the
    database. See :class:`openquake.engine.db.models.SESCollection`.

    Optionally (specified in the job configuration using the
    `ground_motion_fields` parameter), GMFs can be computed from each rupture
    in each stochastic event set. GMFs are also saved to the database.

    :param int job_id:
        ID of the currently running job.
    :param src_seeds:
        List of pairs (source, seed)
    :params gsims_by_rlz:
        dictionary of GSIM
    :param task_no:
        an ordinal so that GMV can be collected in a reproducible order
    """
    rlz_ids = [r.id for r in gsims_by_rlz]
    ses_coll = models.SESCollection.objects.get(lt_realization_ids=rlz_ids)

    hc = models.HazardCalculation.objects.get(oqjob=job_id)
    all_ses = models.SES.objects.filter(ses_collection=ses_coll)
    ruptures = []
    ses_ruptures = []
    rupture_seeds = []

    mon1 = LightMonitor('filtering sites', job_id, compute_ses_and_gmfs)
    mon2 = LightMonitor('generating ruptures', job_id, compute_ses_and_gmfs)
    mon3 = LightMonitor('sampling ruptures', job_id, compute_ses_and_gmfs)

    # Compute and save stochastic event sets
    rnd = random.Random()
    for src, seed in src_seeds:
        rnd.seed(seed)

        with mon1:
            s_sites = src.filter_sites_by_distance_to_source(
                hc.maximum_distance, hc.site_collection
            ) if hc.maximum_distance else hc.site_collection
            if s_sites is None:
                continue

        with mon2:
            rupts = []
            for r in src.iter_ruptures():
                r_sites = r.source_typology.\
                    filter_sites_by_distance_to_rupture(
                        r, hc.maximum_distance, s_sites
                    ) if hc.maximum_distance else s_sites
                if r_sites is not None:
                    rupts.append(r)

        for ses in all_ses:
            numpy.random.seed(rnd.randint(0, models.MAX_SINT_32))
            for i, r in enumerate(rupts):
                with mon3:
                    for j in xrange(r.sample_number_of_occurrences()):
                        rup = models.SESRupture(
                            ses=ses,
                            rupture=r,
                            tag='smlt=%02d|ses=%04d|src=%s|i=%04d-%02d' % (
                                ses_coll.ordinal,
                                ses.ordinal, src.source_id, i, j),
                            hypocenter=r.hypocenter.wkt2d,
                            magnitude=r.mag,
                        )
                        rup_seed = rnd.randint(0, models.MAX_SINT_32)
                        ruptures.append(r)
                        ses_ruptures.append(rup)
                        rupture_seeds.append(rup_seed)

    mon1.flush()
    mon2.flush()
    mon3.flush()
    if not ruptures:
        return

    with EnginePerformanceMonitor('saving ses', job_id, compute_ses_and_gmfs):
        rupture_ids = _save_ses_ruptures(ses_ruptures)

    if not hc.ground_motion_fields:
        return

    # compute GMFs
    imts = map(from_string, hc.intensity_measure_types)
    params = dict(
        correl_model=general.get_correl_model(hc),
        truncation_level=hc.truncation_level,
        maximum_distance=hc.maximum_distance)

    for rlz, gsims in gsims_by_rlz.items():
        with EnginePerformanceMonitor(
                'computing gmfs', job_id, compute_ses_and_gmfs):
            gmvs_per_site, ruptures_per_site = _compute_gmf(
                params, imts, gsims, hc.site_collection,
                zip(ruptures, rupture_ids, rupture_seeds))

        with EnginePerformanceMonitor(
                'saving gmfs', job_id, compute_ses_and_gmfs):
            gmf_coll = models.Gmf.objects.get(lt_realization=rlz)
            _save_gmfs(gmf_coll, gmvs_per_site, ruptures_per_site,
                       hc.site_collection, task_no)

def _save_ses_ruptures(ruptures):
    """
    Helper function for saving stochastic event set ruptures to the database.
    :param ruptures:
        A list of :class:`openquake.engine.db.models.SESRupture` instances.
    """
    # TODO: Possible future optimization:
    # Refactor this to do bulk insertion of ruptures
    rupture_ids = []
    with transaction.commit_on_success(using='job_init'):
        for r in ruptures:
            r.save()
            rupture_ids.append(r.id)
    return rupture_ids

# NB: I tried to return a single dictionary {site_id: [(gmv, rupt_id),...]}
# but it takes a lot more memory (MS)
def _compute_gmf(params, imts, gsims, site_coll, rupture_id_seed_triples):
    """
    Compute a ground motion field value for each rupture, for all the
    points affected by that rupture, for the given IMT. Returns a
    dictionary with the nonzero contributions to each site id, and a dictionary
    with the ids of the contributing ruptures for each site id.
    assert len(ruptures) == len(rupture_seeds)

    :param params:
        a dictionary containing the keys
        correl_model, truncation_level, maximum_distance
    :param imts:
        a list of hazardlib intensity measure types
    :param gsims:
        a dictionary {tectonic region type -> GSIM instance}
    :param site_coll:
        a SiteCollection instance
    :param rupture_id_seed_triple:
        a list of triples with types
        (:class:`openquake.hazardlib.source.rupture.Rupture`, int, int)
    """
    gmvs_per_site = collections.defaultdict(list)
    ruptures_per_site = collections.defaultdict(list)

    # Compute and save ground motion fields
    for i, (rupture, rup_id, rup_seed) in enumerate(rupture_id_seed_triples):
        gmf_calc_kwargs = {
            'rupture': rupture,
            'sites': site_coll,
            'imts': imts,
            'gsim': gsims[rupture.tectonic_region_type],
            'truncation_level': params['truncation_level'],
            'realizations': DEFAULT_GMF_REALIZATIONS,
            'correlation_model': params['correl_model'],
        }
        numpy.random.seed(rup_seed)
        gmf_dict = gmf.ground_motion_fields(**gmf_calc_kwargs)
        for imt, gmf_1_realiz in gmf_dict.iteritems():
            # since DEFAULT_GMF_REALIZATIONS is 1, gmf_1_realiz is a matrix
            # with n_sites rows and 1 column
            for site, gmv in zip(site_coll, gmf_1_realiz):
                gmv = float(gmv)  # convert a 1x1 matrix into a float
                if gmv:  # nonzero contribution to site
                    gmvs_per_site[imt, site.id].append(gmv)
                    ruptures_per_site[imt, site.id].append(rup_id)
    return gmvs_per_site, ruptures_per_site

@transaction.commit_on_success(using='job_init')
def _save_gmfs(gmf, gmvs_per_site, ruptures_per_site, sites, task_no):
    """
    Helper method to save computed GMF data to the database.

    :param gmf:
        The Gmf instance where to save
    :param gmf_per_site:
        The GMFs per rupture
    :param rupture_per_site:
        The associated rupture ids
    :param sites:
        An :class:`openquake.hazardlib.site.SiteCollection` object,
        representing the sites of interest for a calculation.
    :param task_no:
        The ordinal of the task which generated the current GMFs to save
    """
    for imt, site_id in gmvs_per_site:
        imt_name, sa_period, sa_damping = imt
        inserter.add(models.GmfData(
            gmf=gmf,
            task_no=task_no,
            imt=imt_name,
            sa_period=sa_period,
            sa_damping=sa_damping,
            site_id=site_id,
            gmvs=gmvs_per_site[imt, site_id],
            rupture_ids=ruptures_per_site[imt, site_id]))
    inserter.flush()

class EventBasedHazardCalculator(general.BaseHazardCalculator):
    """
    Probabilistic Event-Based hazard calculator. Computes stochastic event sets
    and (optionally) ground motion fields.
    """
    core_calc_task = compute_ses_and_gmfs

    def task_arg_gen(self, _block_size=None):
        """
        Loop through realizations and sources to generate a sequence of
        task arg tuples. Each tuple of args applies to a single task.
        Yielded results are tuples of the form job_id, sources, ses, seeds
        (seeds will be used to seed numpy for temporal occurence sampling).
        """
        hc = self.hc
        rnd = random.Random()
        rnd.seed(hc.random_seed)
        task_no = 0
        for job_id, block, gsims_by_rlz in super(
                EventBasedHazardCalculator, self).task_arg_gen():
            ss = [(src, rnd.randint(0, models.MAX_SINT_32))
                  for src in block]  # source, seed pairs
            yield job_id, ss, gsims_by_rlz, task_no
            task_no += 1

        # now the source_blocks_per_ltpath dictionary can be cleared
        self.source_blocks_per_ltpath.clear()

    def initialize_ses_db_records(self, ordinal, rlzs):
        """
        Create :class:`~openquake.engine.db.models.Output`,
        :class:`~openquake.engine.db.models.SESCollection` and
        :class:`~openquake.engine.db.models.SES` "container" records for
        a single realization.

        Stochastic event set ruptures computed for this realization will be
        associated to these containers.

        NOTE: Many tasks can contribute ruptures to the same SES.
        """
        rlz_ids = [r.id for r in rlzs]

        output = models.Output.objects.create(
            oq_job=self.job,
            display_name='SES Collection smlt-%d-rlz-%s' % (
                ordinal, ','.join(map(str, rlz_ids))),
            output_type='ses')

        ses_coll = models.SESCollection.objects.create(
            output=output, lt_realization_ids=rlz_ids, ordinal=ordinal)

        for rlz in rlzs:
            if self.job.hazard_calculation.ground_motion_fields:
                output = models.Output.objects.create(
                    oq_job=self.job,
                    display_name='GMF rlz-%s' % rlz.id,
                    output_type='gmf')
                models.Gmf.objects.create(output=output, lt_realization=rlz)

        all_ses = []
        for i in xrange(1, self.hc.ses_per_logic_tree_path + 1):
            all_ses.append(
                models.SES.objects.create(
                    ses_collection=ses_coll,
                    investigation_time=self.hc.investigation_time,
                    ordinal=i))
        return all_ses

    def pre_execute(self):
        """
        Do pre-execution work. At the moment, this work entails:
        parsing and initializing sources, parsing and initializing the
        site model (if there is one), parsing vulnerability and
        exposure files, and generating logic tree realizations. (The
        latter piece basically defines the work to be done in the
        `execute` phase.)
        """
        super(EventBasedHazardCalculator, self).pre_execute()
        for i, rlzs in enumerate(self.rlzs_per_ltpath.itervalues()):
            self.initialize_ses_db_records(i, rlzs)

    def post_process(self):
        """
        If requested, perform additional processing of GMFs to produce hazard
        curves.
        """
        if self.hc.hazard_curves_from_gmfs:
            with EnginePerformanceMonitor('generating hazard curves',
                                          self.job.id):
                self.parallelize(
                    post_processing.gmf_to_hazard_curve_task,
                    post_processing.gmf_to_hazard_curve_arg_gen(self.job),
                    self.log_percent)

            # If `mean_hazard_curves` is True and/or `quantile_hazard_curves`
            # has some value (not an empty list), do this additional
            # post-processing.
            if self.hc.mean_hazard_curves or self.hc.quantile_hazard_curves:
                with EnginePerformanceMonitor(
                        'generating mean/quantile curves', self.job.id):
                    self.do_aggregate_post_proc()

            if self.hc.hazard_maps:
                with EnginePerformanceMonitor(
                        'generating hazard maps', self.job.id):
                    self.parallelize(
                        cls_post_proc.hazard_curves_to_hazard_map_task,
                        cls_post_proc.hazard_curves_to_hazard_map_task_arg_gen(
                            self.job),
                        self.log_percent)
<|endoftext|>


<commit_before>import sys

if sys.version_info[:2] >= (3, 2):
    from ConfigParser import ConfigParser as SafeConfigParser
    from ConfigParser import Error
else:
    from ConfigParser import SafeConfigParser
    from ConfigParser import Error

class MultipleValueError(Error):
    pass

class ConfigParser(SafeConfigParser, object):

    def __init__(self, warn_func=None):
        super(ConfigParser, self).__init__()
        self.warnings = []
        self.warn_func = warn_func
        # Python < 3.2
        if hasattr(self, '_boolean_states'):
            self.BOOLEAN_STATES = self._boolean_states

    def warn(self, msg):
        self.warnings.append(msg)
        if self.warn_func is not None:
            self.warn_func(msg)

    def read(self, filenames):
        self.warnings = []
        try:
            super(ConfigParser, self).read(filenames)
        except Error, e:
            self.warn(str(e))

    def get(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            return value
        return default

    def getlist(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            return self.to_list(value)
        return default

    def getstring(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            try:
                return self.to_string(value)
            except MultipleValueError, e:
                self.warn("Multiple values not allowed: %s = %r" % (option, self._value_from_exc(e)))
        return default

    def getboolean(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            try:
                return self.to_boolean(value)
            except MultipleValueError, e:
                self.warn("Multiple values not allowed: %s = %r" % (option, self._value_from_exc(e)))
            except ValueError, e:
                self.warn('Not a boolean: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def getint(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            try:
                return self.to_int(value)
            except MultipleValueError, e:
                self.warn('Multiple values not allowed: %s = %r' % (option, self._value_from_exc(e)))
            except ValueError, e:
                self.warn('Not an integer: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def getfloat(self, section, option, default=None):
        if self.has_option(section, option):
            value = super(ConfigParser, self).get(section, option)
            try:
                return self.to_float(value)
            except MultipleValueError, e:
                self.warn('Multiple values not allowed: %s = %r' % (option, self._value_from_exc(e)))
            except ValueError, e:
                self.warn('Not a float: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def to_list(self, value):
        return value.split()

    def to_string(self, value):
        v = self._single_value(value)
        return v

    def to_boolean(self, value):
        v = self._single_value(value).lower()
        if v not in self.BOOLEAN_STATES:
            raise ValueError('Not a boolean: %s' % v)
        return self.BOOLEAN_STATES[v]

    def to_int(self, value):
        v = self._single_value(value)
        return int(v)

    def to_float(self, value):
        v = self._single_value(value)
        return float(v)

    def _single_value(self, value):
        v = value.strip()
        if len(v.split()) > 1:
            raise MultipleValueError('Multiple values not allowed: %s' % v)
        return v

    def _value_from_exc(self, exc):
        # e.g.: invalid literal for int() with base 10: 'a'
        msg = str(exc)
        colon = msg.find(':')
        if colon >= 0:
            value = msg[colon+1:].lstrip()
            if (value.startswith("'") and value.endswith("'")) or \
               (value.startswith('"') and value.endswith('"')):
                value = value[1:-1]
            return value
        return ''

del SafeConfigParser
<commit_msg>Use context manager to turn errors into warnings; also switch off interpolation by default.<commit_after>import sys

if sys.version_info[:2] >= (3, 2):
    from ConfigParser import ConfigParser as SafeConfigParser
    from ConfigParser import Error
else:
    from ConfigParser import SafeConfigParser
    from ConfigParser import Error

class MultipleValueError(Error):
    pass

class errors2warnings(object):
    """Turn ConfigParser.Errors into warnings."""

    def __init__(self, context):
        self.context = context

    def __enter__(self):
        pass

    def __exit__(self, type, value, tb):
        if isinstance(value, Error):
            self.context.warn(str(value))
            return True

class ConfigParser(SafeConfigParser, object):

    def __init__(self, warn_func=None, raw=True):
        super(ConfigParser, self).__init__()
        self.warnings = []
        self.warn_func = warn_func
        self.raw = raw
        # Python < 3.2
        if hasattr(self, '_boolean_states'):
            self.BOOLEAN_STATES = self._boolean_states

    def warn(self, msg):
        self.warnings.append(msg)
        if self.warn_func is not None:
            self.warn_func(msg)

    def read(self, filenames):
        self.warnings = []
        with errors2warnings(self):
            super(ConfigParser, self).read(filenames)

    def items(self, section, default=None):
        if self.has_section(section):
            with errors2warnings(self):
                value = super(ConfigParser, self).items(section, raw=self.raw)
                return value
        return default

    def get(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                return value
        return default

    def getlist(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                return self.to_list(value)
        return default

    def getstring(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                try:
                    return self.to_string(value)
                except MultipleValueError, e:
                    self.warn("Multiple values not allowed: %s = %r" % (option, self._value_from_exc(e)))
        return default

    def getboolean(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                try:
                    return self.to_boolean(value)
                except MultipleValueError, e:
                    self.warn("Multiple values not allowed: %s = %r" % (option, self._value_from_exc(e)))
                except ValueError, e:
                    self.warn('Not a boolean: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def getint(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                try:
                    return self.to_int(value)
                except MultipleValueError, e:
                    self.warn('Multiple values not allowed: %s = %r' % (option, self._value_from_exc(e)))
                except ValueError, e:
                    self.warn('Not an integer: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def getfloat(self, section, option, default=None):
        if self.has_option(section, option):
            with errors2warnings(self):
                value = super(ConfigParser, self).get(section, option, raw=self.raw)
                try:
                    return self.to_float(value)
                except MultipleValueError, e:
                    self.warn('Multiple values not allowed: %s = %r' % (option, self._value_from_exc(e)))
                except ValueError, e:
                    self.warn('Not a float: %s = %r' % (option, self._value_from_exc(e)))
        return default

    def to_list(self, value):
        return value.split()

    def to_string(self, value):
        v = self._single_value(value)
        return v

    def to_boolean(self, value):
        v = self._single_value(value).lower()
        if v not in self.BOOLEAN_STATES:
            raise ValueError('Not a boolean: %s' % v)
        return self.BOOLEAN_STATES[v]

    def to_int(self, value):
        v = self._single_value(value)
        return int(v)

    def to_float(self, value):
        v = self._single_value(value)
        return float(v)

    def _single_value(self, value):
        v = value.strip()
        if len(v.split()) > 1:
            raise MultipleValueError('Multiple values not allowed: %s' % v)
        return v

    def _value_from_exc(self, exc):
        # e.g.: invalid literal for int() with base 10: 'a'
        msg = str(exc)
        colon = msg.find(':')
        if colon >= 0:
            value = msg[colon+1:].lstrip()
            if (value.startswith("'") and value.endswith("'")) or \
               (value.startswith('"') and value.endswith('"')):
                value = value[1:-1]
            return value
        return ''

del SafeConfigParser
<|endoftext|>


<commit_before># Copyright 2013 Kylin, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.utils.translation import ugettext_lazy as _

from horizon import tables

class QuotaFilterAction(tables.FilterAction):
    def filter(self, table, tenants, filter_string):
        q = filter_string.lower()

        def comp(tenant):
            if q in tenant.name.lower():
                return True
            return False

        return filter(comp, tenants)

class UpdateDefaultQuotas(tables.LinkAction):
    name = "update_defaults"
    verbose_name = _("Update Defaults")
    url = "horizon:admin:defaults:update_defaults"
    classes = ("ajax-modal",)
    icon = "pencil"

def get_quota_name(quota):
    QUOTA_NAMES = {
        'injected_file_content_bytes': _('Injected File Content Bytes'),
        'injected_file_path_bytes': _('Length of Injected File Path'),
        'metadata_items': _('Metadata Items'),
        'cores': _('VCPUs'),
        'instances': _('Instances'),
        'injected_files': _('Injected Files'),
        'volumes': _('Volumes'),
        'snapshots': _('Volume Snapshots'),
        'gigabytes': _('Total Size of Volumes and Snapshots (GB)'),
        'ram': _('RAM (MB)'),
        'floating_ips': _('Floating IPs'),
        'security_groups': _('Security Groups'),
        'security_group_rules': _('Security Group Rules'),
        'key_pairs': _('Key Pairs'),
        'fixed_ips': _('Fixed IPs'),
        'volumes_volume_luks': _('LUKS Volumes'),
        'snapshots_volume_luks': _('LUKS Volume Snapshots'),
        'gigabytes_volume_luks':
        _('Total Size of LUKS Volumes and Snapshots (GB)'),
        'dm-crypt': _('dm-crypt'),
    }
    return QUOTA_NAMES.get(quota.name, quota.name.replace("_", " ").title())

class QuotasTable(tables.DataTable):
    name = tables.Column(get_quota_name, verbose_name=_('Quota Name'))
    limit = tables.Column("limit", verbose_name=_('Limit'))

    def get_object_id(self, obj):
        return obj.name

    class Meta(object):
        name = "quotas"
        verbose_name = _("Quotas")
        table_actions = (QuotaFilterAction, UpdateDefaultQuotas)
        multi_select = False
<commit_msg>Make all quota names in Defaults panel translatable<commit_after># Copyright 2013 Kylin, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.utils.translation import ugettext_lazy as _

from horizon import tables

class QuotaFilterAction(tables.FilterAction):
    def filter(self, table, tenants, filter_string):
        q = filter_string.lower()

        def comp(tenant):
            if q in tenant.name.lower():
                return True
            return False

        return filter(comp, tenants)

class UpdateDefaultQuotas(tables.LinkAction):
    name = "update_defaults"
    verbose_name = _("Update Defaults")
    url = "horizon:admin:defaults:update_defaults"
    classes = ("ajax-modal",)
    icon = "pencil"

def get_quota_name(quota):
    QUOTA_NAMES = {
        'injected_file_content_bytes': _('Injected File Content Bytes'),
        'injected_file_path_bytes': _('Length of Injected File Path'),
        'metadata_items': _('Metadata Items'),
        'cores': _('VCPUs'),
        'instances': _('Instances'),
        'injected_files': _('Injected Files'),
        'volumes': _('Volumes'),
        'snapshots': _('Volume Snapshots'),
        'gigabytes': _('Total Size of Volumes and Snapshots (GB)'),
        'ram': _('RAM (MB)'),
        'floating_ips': _('Floating IPs'),
        'security_groups': _('Security Groups'),
        'security_group_rules': _('Security Group Rules'),
        'key_pairs': _('Key Pairs'),
        'fixed_ips': _('Fixed IPs'),
        'volumes_volume_luks': _('LUKS Volumes'),
        'snapshots_volume_luks': _('LUKS Volume Snapshots'),
        'gigabytes_volume_luks':
        _('Total Size of LUKS Volumes and Snapshots (GB)'),
        'dm-crypt': _('dm-crypt'),
        'server_group_members': _('Server Group Members'),
        'server_groups': _('Server Groups'),
        'backup_gigabytes': _('Backup Gigabytes'),
        'backups': _('Backups')
    }
    return QUOTA_NAMES.get(quota.name, quota.name.replace("_", " ").title())

class QuotasTable(tables.DataTable):
    name = tables.Column(get_quota_name, verbose_name=_('Quota Name'))
    limit = tables.Column("limit", verbose_name=_('Limit'))

    def get_object_id(self, obj):
        return obj.name

    class Meta(object):
        name = "quotas"
        verbose_name = _("Quotas")
        table_actions = (QuotaFilterAction, UpdateDefaultQuotas)
        multi_select = False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os, tarfile, json, re, urllib
import numpy as np
from pandas import DataFrame
from mpcontribs.io.core.recdict import RecursiveDict
from mpcontribs.io.core.utils import nest_dict
from monty.json import MontyDecoder
#from mpcontribs.users.boltztrap.rest.rester import BoltztrapRester

def run(mpfile, nmax=10, dup_check_test_site=True):

    # book-keeping
    existing_mpids = {}
    #for b in [False, True]:
    #    with BoltztrapRester(test_site=b) as mpr:
    #        for doc in mpr.query_contributions(criteria=mpr.query):
    #            existing_mpids[doc['mp_cat_id']] = doc['_id']
    #    if not dup_check_test_site:
    #        break

    for typ in ['2d', '3d']:

        url = mpfile.hdata['_hdata']['input_url'].format(typ)
        dbfile = url.rsplit('/')[-1]

        if not os.path.exists(dbfile):
            print 'downloading', dbfile, '...'
            url_opener = urllib.URLopener()
            url_opener.retrieve(url, dbfile)

        else:
            print 'unpacking', dbfile, '...'
            with tarfile.open(dbfile, "r:gz") as tar:
                member = tar.getmembers()[0]
                d = json.load(tar.extractfile(member), cls=MontyDecoder)
                print len(d), 'materials loaded.'

                for idx,i in enumerate(d):
                    mpid = i['mpid']
                    print 'adding', mpid, '...'
                    data = RecursiveDict()
                    data['jid'] = i['jid']
                    data['formula'] = i['final_str'].composition.reduced_formula
                    data['spacegroup'] = i['final_str'].get_space_group_info()[0]
                    data['final_energy'] = '{} eV'.format(i["fin_en"])
                    data['optB88vDW_bandgap'] = '{} eV'.format(i["op_gap"])
                    data['mbj_bandgap'] = None if i["mbj_gap"] == 'na' else '{} eV'.format(i["mbj_gap"])
                    data['bulk_modulus'] = '{} GPa'.format(i["kv"])
                    data['shear_modulus'] = '{} GPa'.format(i["gv"])
                    mpfile.add_hierarchical_data(nest_dict(data, ['data', typ]), identifier=mpid)
                    mpfile.add_structure(i['final_str'], name=data['formula'], identifier=mpid)

                    if idx >= nmax-1:
                        break

            print 'DONE with', typ
<commit_msg>jarvis_dft: dupcheck and clean_value<commit_after># -*- coding: utf-8 -*-
import os, tarfile, json, re, urllib
import numpy as np
from pandas import DataFrame
from mpcontribs.io.core.recdict import RecursiveDict
from mpcontribs.io.core.utils import nest_dict
from monty.json import MontyDecoder
from mpcontribs.users.jarvis_dft.rest.rester import DefectGenomePcfcMaterialsRester

def clean_value(val, unit):
    return None if val == 'na' else '{} {}'.format(val, unit)

def run(mpfile, nmax=10, dup_check_test_site=True):

    # book-keeping
    existing_mpids = {}
    for b in [False, True]:
        with DefectGenomePcfcMaterialsRester(test_site=b) as mpr:
            for doc in mpr.query_contributions(criteria=mpr.query):
                existing_mpids[doc['mp_cat_id']] = doc['_id']
        if not dup_check_test_site:
            break

    for typ in ['2d', '3d']:

        url = mpfile.hdata['_hdata']['input_url'].format(typ)
        dbfile = url.rsplit('/')[-1]

        if not os.path.exists(dbfile):
            print 'downloading', dbfile, '...'
            url_opener = urllib.URLopener()
            url_opener.retrieve(url, dbfile)

        else:
            print 'unpacking', dbfile, '...'
            with tarfile.open(dbfile, "r:gz") as tar:
                member = tar.getmembers()[0]
                d = json.load(tar.extractfile(member), cls=MontyDecoder)
                print len(d), 'materials loaded.'

                for idx,i in enumerate(d):
                    mpid = i['mpid']
                    print 'adding', mpid, '...'
                    data = RecursiveDict()
                    data['jid'] = i['jid']
                    data['formula'] = i['final_str'].composition.reduced_formula
                    data['spacegroup'] = i['final_str'].get_space_group_info()[0]
                    data['final_energy'] = clean_value(i["fin_en"], 'eV')
                    data['optB88vDW_bandgap'] = clean_value(i["op_gap"], 'eV')
                    data['mbj_bandgap'] = clean_value(i["mbj_gap"], 'eV')
                    data['bulk_modulus'] = clean_value(i["kv"], 'GPa')
                    data['shear_modulus'] = clean_value(i["gv"], 'GPa')
                    mpfile.add_hierarchical_data(nest_dict(data, ['data', typ]), identifier=mpid)
                    mpfile.add_structure(i['final_str'], name=data['formula'], identifier=mpid)

                    if mpid in existing_mpids:
                        cid = existing_mpids[mpid]
                        mpfile.insert_id(mpid, cid)
                        print cid, 'inserted to update', mpid

                    if idx >= nmax-1:
                        break

            print 'DONE with', typ
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2017~2999 - cologler <skyoflw@gmail.com>
# ----------
#
# ----------

import functools
import threading

def _create_wrapper(func, factory, factory_args=[], factory_kwargs={}):
    sync = factory(*factory_args, **factory_kwargs)

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with sync:
            return func(*args, **kwargs)

    return wrapper

def lock(func):
    '''
    use `Lock` to keep func access thread-safety.

    example:

    ``` py
    @lock
    def func(): pass
    ```
    '''
    return _create_wrapper(func, threading.Lock)

def rlock(func):
    '''
    use `RLock` to keep func access thread-safety.

    example:

    ``` py
    @rlock
    def func(): pass
    ```
    '''
    return _create_wrapper(func, threading.RLock)

def semaphore(count: int, bounded: bool=False):
    '''
    use `Semaphore` to keep func access thread-safety.

    example:

    ``` py
    @semaphore(3)
    def func(): pass
    ```
    '''

    lock_type = threading.BoundedSemaphore if bounded else threading.Semaphore

    def _wrap(func):
        return _create_wrapper(func, lock_type, factory_kwargs={'value': count})

    return _wrap

<commit_msg>refactor<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2017~2999 - cologler <skyoflw@gmail.com>
# ----------
#
# ----------

import functools
import threading

def _create_wrapper(func, factory):
    sync = factory()

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with sync:
            return func(*args, **kwargs)

    return wrapper

def lock(func):
    '''
    use `Lock` to keep func access thread-safety.

    example:

    ``` py
    @lock
    def func(): pass
    ```
    '''
    return _create_wrapper(func, threading.Lock)

def rlock(func):
    '''
    use `RLock` to keep func access thread-safety.

    example:

    ``` py
    @rlock
    def func(): pass
    ```
    '''
    return _create_wrapper(func, threading.RLock)

def semaphore(count: int, bounded: bool=False):
    '''
    use `Semaphore` to keep func access thread-safety.

    example:

    ``` py
    @semaphore(3)
    def func(): pass
    ```
    '''

    lock_type = threading.BoundedSemaphore if bounded else threading.Semaphore
    factory = functools.partial(lock_type, value=count)

    def _wrap(func):
        return _create_wrapper(func, factory)

    return _wrap

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# $Id$
#

import unittest
import subprocess
import time
import re
import sys

import psutil

from test_psutil import reap_children, get_test_subprocess
from _posix import ps

def sysctl(cmdline):
    """Expects a sysctl command with an argument and parse the result
    returning only the value of interest.
    """
    p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE)
    result = p.communicate()[0].strip()
    result = result[result.find(": ") + 2:]
    if sys.version_info >= (3,):
        result = str(result, sys.stdout.encoding)
    try:
        return int(result)
    except ValueError:
        return result

def parse_sysctl_vmtotal(output):
    """Parse sysctl vm.vmtotal output returning total and free memory
    values.
    """
    line = output.split('\n')[4]  # our line of interest
    mobj = re.match(r'Virtual\s+Memory.*Total:\s+(\d+)K,\s+Active\s+(\d+)K.*', line)
    total, active = mobj.groups()
    # values are represented in kilo bytes
    total = int(total) * 1024
    active = int(active) * 1024
    free = total - active
    return total, free

class BSDSpecificTestCase(unittest.TestCase):

    def setUp(self):
        self.pid = get_test_subprocess().pid

    def tearDown(self):
        reap_children()

    def test_TOTAL_PHYMEM(self):
        sysctl_hwphymem = sysctl('sysctl hw.physmem')
        self.assertEqual(sysctl_hwphymem, psutil.TOTAL_PHYMEM)

    def test_BOOT_TIME(self):
        s = sysctl('sysctl kern.boottime')
        s = s[s.find(" sec = ") + 7:]
        s = s[:s.find(',')]
        btime = int(s)
        self.assertEqual(btime, psutil.BOOT_TIME)

    def test_avail_phymem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        _sum = sum((sysctl("sysctl vm.stats.vm.v_inactive_count"),
                    sysctl("sysctl vm.stats.vm.v_cache_count"),
                    sysctl("sysctl vm.stats.vm.v_free_count")
                   ))
        _pagesize = sysctl("sysctl hw.pagesize")
        sysctl_avail_phymem = _sum * _pagesize
        psutil_avail_phymem =  psutil.avail_phymem()
        difference = abs(psutil_avail_phymem - sysctl_avail_phymem)
        # On my system both sysctl and psutil report the same values.
        # Let's use a tollerance of 0.5 MB and consider the test as failed
        # if we go over it.
        if difference > (0.5 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                      sysctl_avail_phymem, psutil_avail_phymem, difference))

    def test_total_virtmem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE)
        result = p.communicate()[0].strip()
        if sys.version_info >= (3,):
            result = str(result, sys.stdout.encoding)
        sysctl_total_virtmem, _ = parse_sysctl_vmtotal(result)
        psutil_total_virtmem = psutil.total_virtmem()
        difference = abs(sysctl_total_virtmem - psutil_total_virtmem)

        # On my system I get a difference of 4657152 bytes, probably because
        # the system is consuming memory for this same test.
        # Assuming psutil is right, let's use a tollerance of 10 MB and consider
        # the test as failed if we go over it.
        if difference > (10 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                       sysctl_total_virtmem, psutil_total_virtmem, difference))

    def test_avail_virtmem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE)
        result = p.communicate()[0].strip()
        if sys.version_info >= (3,):
            result = str(result, sys.stdout.encoding)
        _, sysctl_avail_virtmem = parse_sysctl_vmtotal(result)
        psutil_avail_virtmem = psutil.avail_virtmem()
        difference = abs(sysctl_avail_virtmem - psutil_avail_virtmem)
        # let's assume the test is failed if difference is > 0.5 MB
        if difference > (0.5 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                       sysctl_avail_virtmem, psutil_avail_virtmem, difference))

    def test_process_create_time(self):
        cmdline = "ps -o lstart -p %s" %self.pid
        p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE)
        output = p.communicate()[0]
        if sys.version_info >= (3,):
            output = str(output, sys.stdout.encoding)
        start_ps = output.replace('STARTED', '').strip()
        start_psutil = psutil.Process(self.pid).create_time
        start_psutil = time.strftime("%a %b %e %H:%M:%S %Y",
                                     time.localtime(start_psutil))
        self.assertEqual(start_ps, start_psutil)

if __name__ == '__main__':
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.makeSuite(BSDSpecificTestCase))
    unittest.TextTestRunner(verbosity=2).run(test_suite)

<commit_msg>Fix py3 test failure on BSD<commit_after>#!/usr/bin/env python
#
# $Id$
#

import unittest
import subprocess
import time
import re
import sys

import psutil

from test_psutil import reap_children, get_test_subprocess
from _posix import ps

def sysctl(cmdline):
    """Expects a sysctl command with an argument and parse the result
    returning only the value of interest.
    """
    p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE)
    result = p.communicate()[0].strip()
    if sys.version_info >= (3,):
        result = result[result.find(b": ") + 2:]
        result = str(result, sys.stdout.encoding)
    else:
        result = result[result.find(": ") + 2:]
    try:
        return int(result)
    except ValueError:
        return result

def parse_sysctl_vmtotal(output):
    """Parse sysctl vm.vmtotal output returning total and free memory
    values.
    """
    line = output.split('\n')[4]  # our line of interest
    mobj = re.match(r'Virtual\s+Memory.*Total:\s+(\d+)K,\s+Active\s+(\d+)K.*', line)
    total, active = mobj.groups()
    # values are represented in kilo bytes
    total = int(total) * 1024
    active = int(active) * 1024
    free = total - active
    return total, free

class BSDSpecificTestCase(unittest.TestCase):

    def setUp(self):
        self.pid = get_test_subprocess().pid

    def tearDown(self):
        reap_children()

    def test_TOTAL_PHYMEM(self):
        sysctl_hwphymem = sysctl('sysctl hw.physmem')
        self.assertEqual(sysctl_hwphymem, psutil.TOTAL_PHYMEM)

    def test_BOOT_TIME(self):
        s = sysctl('sysctl kern.boottime')
        s = s[s.find(" sec = ") + 7:]
        s = s[:s.find(',')]
        btime = int(s)
        self.assertEqual(btime, psutil.BOOT_TIME)

    def test_avail_phymem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        _sum = sum((sysctl("sysctl vm.stats.vm.v_inactive_count"),
                    sysctl("sysctl vm.stats.vm.v_cache_count"),
                    sysctl("sysctl vm.stats.vm.v_free_count")
                   ))
        _pagesize = sysctl("sysctl hw.pagesize")
        sysctl_avail_phymem = _sum * _pagesize
        psutil_avail_phymem =  psutil.avail_phymem()
        difference = abs(psutil_avail_phymem - sysctl_avail_phymem)
        # On my system both sysctl and psutil report the same values.
        # Let's use a tollerance of 0.5 MB and consider the test as failed
        # if we go over it.
        if difference > (0.5 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                      sysctl_avail_phymem, psutil_avail_phymem, difference))

    def test_total_virtmem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE)
        result = p.communicate()[0].strip()
        if sys.version_info >= (3,):
            result = str(result, sys.stdout.encoding)
        sysctl_total_virtmem, _ = parse_sysctl_vmtotal(result)
        psutil_total_virtmem = psutil.total_virtmem()
        difference = abs(sysctl_total_virtmem - psutil_total_virtmem)

        # On my system I get a difference of 4657152 bytes, probably because
        # the system is consuming memory for this same test.
        # Assuming psutil is right, let's use a tollerance of 10 MB and consider
        # the test as failed if we go over it.
        if difference > (10 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                       sysctl_total_virtmem, psutil_total_virtmem, difference))

    def test_avail_virtmem(self):
        # This test is not particularly accurate and may fail if the OS is
        # consuming memory for other applications.
        # We just want to test that the difference between psutil result
        # and sysctl's is not too high.
        p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE)
        result = p.communicate()[0].strip()
        if sys.version_info >= (3,):
            result = str(result, sys.stdout.encoding)
        _, sysctl_avail_virtmem = parse_sysctl_vmtotal(result)
        psutil_avail_virtmem = psutil.avail_virtmem()
        difference = abs(sysctl_avail_virtmem - psutil_avail_virtmem)
        # let's assume the test is failed if difference is > 0.5 MB
        if difference > (0.5 * 2**20):
            self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
                       sysctl_avail_virtmem, psutil_avail_virtmem, difference))

    def test_process_create_time(self):
        cmdline = "ps -o lstart -p %s" %self.pid
        p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE)
        output = p.communicate()[0]
        if sys.version_info >= (3,):
            output = str(output, sys.stdout.encoding)
        start_ps = output.replace('STARTED', '').strip()
        start_psutil = psutil.Process(self.pid).create_time
        start_psutil = time.strftime("%a %b %e %H:%M:%S %Y",
                                     time.localtime(start_psutil))
        self.assertEqual(start_ps, start_psutil)

if __name__ == '__main__':
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.makeSuite(BSDSpecificTestCase))
    unittest.TextTestRunner(verbosity=2).run(test_suite)

<|endoftext|>