<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import csv
import json
import os

from django.conf import settings
from django.core.management.base import BaseCommand
from django.contrib.gis.geos import GEOSGeometry, MultiPolygon

from scuole.states.models import State, Commissioner

class Command(BaseCommand):
    help = 'Bootstraps state models.'

    def handle(self, *args, **options):

        state_json = os.path.join(
            settings.DATA_FOLDER,
            'state', 'shape', 'tx.geojson')

        self.shape_data = self.load_geojson_file(state_json)

        commissioner_csv = os.path.join(
            settings.DATA_FOLDER,
            'state', 'state.csv')

        self.commissioner_data = self.load_commissioner_file(
            commissioner_csv)

        self.create_state()

    def load_geojson_file(self, file):
        payload = {}

        with open(file, 'r') as f:
            data = json.load(f)

            for feature in data['features']:
                payload = feature['geometry']

        return payload

    def load_commissioner_file(self, file):
        with open(file, 'rU') as f:
            reader = csv.DictReader(f)
            return next(reader)

    def create_state(self):
        self.stdout.write('Creating Texas...')

        geometry = GEOSGeometry(
            json.dumps(self.shape_data))

        # checks to see if the geometry is a multipolygon
        if geometry.geom_typeid == 3:
            geometry = MultiPolygon(geometry)

        state, _ = State.objects.update_or_create(
            slug='tx',
            defaults={
                'name': 'TX',
                'shape': geometry
            }
        )

        commissioner = self.commissioner_data
        self.load_commissioner(state, commissioner)

    def load_commissioner(self, state, commissioner):
        name = commissioner['Full Name']
        Commissioner.objects.update_or_create(
            state=state,
            defaults={
                'name': name,
                'role': commissioner['Role'],
                'email': commissioner['Email Address'],
                'phone_number': commissioner['Phone'],
                'fax_number': commissioner['Fax'],
            }
        )
<commit_msg>changed commissioner file name for consistancy<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import csv
import json
import os

from django.conf import settings
from django.core.management.base import BaseCommand
from django.contrib.gis.geos import GEOSGeometry, MultiPolygon

from scuole.states.models import State, Commissioner

class Command(BaseCommand):
    help = 'Bootstraps state models.'

    def handle(self, *args, **options):

        state_json = os.path.join(
            settings.DATA_FOLDER,
            'state', 'shape', 'tx.geojson')

        self.shape_data = self.load_geojson_file(state_json)

        commissioner_csv = os.path.join(
            settings.DATA_FOLDER,
            'state', 'commissioner.csv')

        self.commissioner_data = self.load_commissioner_file(
            commissioner_csv)

        self.create_state()

    def load_geojson_file(self, file):
        payload = {}

        with open(file, 'r') as f:
            data = json.load(f)

            for feature in data['features']:
                payload = feature['geometry']

        return payload

    def load_commissioner_file(self, file):
        with open(file, 'rU') as f:
            reader = csv.DictReader(f)
            return next(reader)

    def create_state(self):
        self.stdout.write('Creating Texas...')

        geometry = GEOSGeometry(
            json.dumps(self.shape_data))

        # checks to see if the geometry is a multipolygon
        if geometry.geom_typeid == 3:
            geometry = MultiPolygon(geometry)

        state, _ = State.objects.update_or_create(
            slug='tx',
            defaults={
                'name': 'TX',
                'shape': geometry
            }
        )

        commissioner = self.commissioner_data
        self.load_commissioner(state, commissioner)

    def load_commissioner(self, state, commissioner):
        name = commissioner['Full Name']
        Commissioner.objects.update_or_create(
            state=state,
            defaults={
                'name': name,
                'role': commissioner['Role'],
                'email': commissioner['Email Address'],
                'phone_number': commissioner['Phone'],
                'fax_number': commissioner['Fax'],
            }
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Provides an interface to Esprima, implemented in Node.JS.
"""

import warnings
import atexit
import os
import json
import subprocess
from pathlib import Path
from typing import Any, IO, Optional, Iterator

import zmq  # type: ignore

here = Path(__file__).parent.absolute()
esprima_bin = here / 'esprima-interface'
assert esprima_bin.exists()

class Server:
    """
    Provides a convenient interface for accessing the Esprima interface
    server.

    All that's required is a the path of the UNIX domain socket where the
    already instantiated server is running.
    """

    def __init__(self, socket_path: Path) -> None:
        context = zmq.Context()
        socket = self._socket = context.socket(zmq.REQ)
        socket.connect(f"ipc://{socket_path}")

    def check_syntax(self, source: bytes) -> bool:
        return self._communicate(b'c', source)

    def tokenize(self, source: bytes) -> Iterator[Any]:
        return self._communicate(b't', source)

    def exit(self) -> None:
        response = self._communicate(b'x')
        assert response is True

    def _communicate(self, type_code: bytes, payload: bytes=None) -> Any:
        self._socket.send(type_code + payload if payload else type_code)
        response = self._socket.recv()
        return json.loads(response)

# A global instance of the parse server.
_instance: Optional[Server] = None

def get_server() -> Server:
    """
    Retrieves the global server instance.
    """
    if _instance is None:
        return start()
    return _instance

def start() -> Server:
    """
    Starts the global server instance.
    """
    global _instance
    argv0 = str(here / 'esprima-interface')
    socket_path = Path(f'/tmp/esprima-server.{os.getpid()}')
    subprocess.Popen([argv0, '--server', f"ipc://{socket_path}"],
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.DEVNULL,
                     shell=False)

    _instance = Server(socket_path)

    @atexit.register
    def cleanup():
        _instance.exit()
        try:
            socket_path.unlink()
        except FileNotFoundError:
            warnings.warn("Socket {socket_path} already deleted?")

    return _instance

def tokenize(file_obj: IO[bytes]) -> Iterator[Any]:
    """
    Tokenizes a (real!) bytes file using Esprima.
    """
    status = subprocess.run([str(esprima_bin)],
                            check=True,
                            stdin=file_obj,
                            stdout=subprocess.PIPE)
    return json.loads(status.stdout.decode('UTF-8'))

def check_syntax(source_file: IO[bytes]) -> bool:
    status = subprocess.run((str(esprima_bin), '--check-syntax'),
                            check=False,
                            stdin=source_file,
                            stdout=subprocess.PIPE)
    return status.returncode == 0

# Tests the server.
if __name__ == '__main__':
    from sensibility.language.javascript import javascript
    # Time how many times I can tokenize the source code of the server itself.
    parser = get_server()
    import timeit
    with open(here / 'index.js', 'rb') as source_file:
        source = source_file.read()

    timer = timeit.Timer('parser.tokenize(source)', globals=globals())
    samples = timer.repeat(number=1000)
    print("Using server:", *sorted(samples))

    timer = timeit.Timer('javascript.tokenize(source)', globals=globals())
    samples = timer.repeat(number=1000)
    print("Using original:", *sorted(samples))
<commit_msg>Check that JS server is alive when communicating with it.<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Provides an interface to Esprima, implemented in Node.JS.
"""

import warnings
import atexit
import os
import json
import subprocess
from pathlib import Path
from typing import Any, IO, Optional, Iterator

import zmq  # type: ignore

here = Path(__file__).parent.absolute()
esprima_bin = here / 'esprima-interface'
assert esprima_bin.exists()

class Server:
    """
    Provides a convenient interface for accessing the Esprima interface
    server.

    All that's required is a the path of the UNIX domain socket where the
    already instantiated server is running.
    """

    def __init__(self, socket_path: Path, process) -> None:
        context = zmq.Context()
        self._socket = context.socket(zmq.REQ)
        self._socket.connect(f"ipc://{socket_path}")
        self._process = process

    def check_syntax(self, source: bytes) -> bool:
        return self._communicate(b'c', source)

    def tokenize(self, source: bytes) -> Iterator[Any]:
        return self._communicate(b't', source)

    def exit(self) -> None:
        response = self._communicate(b'x')
        assert response is True
        # Remove this reference right here to:
        #  - prevent wonkiness when shutting down
        #  - cause errors if a future request is made.
        del self._process

    def _communicate(self, type_code: bytes, payload: bytes=None) -> Any:
        self._ensure_alive()
        self._socket.send(type_code + payload if payload else type_code)
        response = self._socket.recv()
        return json.loads(response)

    def _ensure_alive(self) -> None:
        if self._process.poll() is not None:
            raise RuntimeError('Server process is dead.')

# A global instance of the parse server.
_instance: Optional[Server] = None

def get_server() -> Server:
    """
    Retrieves the global server instance.
    """
    if _instance is None:
        return start()
    return _instance

def start() -> Server:
    """
    Starts the global server instance.
    """
    global _instance
    argv0 = str(here / 'esprima-interface')
    socket_path = Path(f'/tmp/esprima-server.{os.getpid()}')
    proc = subprocess.Popen([argv0, '--server', f"ipc://{socket_path}"],
                            stdout=subprocess.DEVNULL,
                            stderr=subprocess.DEVNULL,
                            shell=False)

    _instance = Server(socket_path, proc)

    @atexit.register
    def cleanup():
        _instance.exit()
        try:
            socket_path.unlink()
        except FileNotFoundError:
            warnings.warn("Socket {socket_path} already deleted?")

    return _instance

def tokenize(file_obj: IO[bytes]) -> Iterator[Any]:
    """
    Tokenizes a (real!) bytes file using Esprima.
    """
    status = subprocess.run([str(esprima_bin)],
                            check=True,
                            stdin=file_obj,
                            stdout=subprocess.PIPE)
    return json.loads(status.stdout.decode('UTF-8'))

def check_syntax(source_file: IO[bytes]) -> bool:
    status = subprocess.run((str(esprima_bin), '--check-syntax'),
                            check=False,
                            stdin=source_file,
                            stdout=subprocess.PIPE)
    return status.returncode == 0

# Tests the server.
if __name__ == '__main__':
    from sensibility.language.javascript import javascript
    # Time how many times I can tokenize the source code of the server itself.
    parser = get_server()
    import timeit
    with open(here / 'index.js', 'rb') as source_file:
        source = source_file.read()

    timer = timeit.Timer('parser.tokenize(source)', globals=globals())
    samples = timer.repeat(number=1000)
    print("Using server:", *sorted(samples))

    timer = timeit.Timer('javascript.tokenize(source)', globals=globals())
    samples = timer.repeat(number=1000)
    print("Using original:", *sorted(samples))
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import datetime
from scipy.stats import pearsonr

from . import Utils as tsutil

class cPerf:
    def __init__(self):
        self.mErrorStdDev = np.nan;
        self.mErrorMean = np.nan;
        self.mMAE = np.nan;
        self.mMAPE = np.nan;
        self.mSMAPE = np.nan;
        self.mL1 = np.nan;
        self.mL2 = np.nan;
        self.mR2 = np.nan;
        self.mPearsonR = np.nan;
        self.mCount = np.nan;
        self.mName = "No_Name";

    def protect_small_value(self, x , eps):
        if((x < eps) and (x >= 0)):
            return eps;
        if((x > -eps) and (x <= 0)):
            return -eps;
        return x;

    def protect_small_values(self, signal):
        eps = 1.0e-10;
        signal1 = signal.apply(lambda x : self.protect_small_value(x, eps));
        # self.dump_perf_data(signal , signal1);
        return signal1;

    def check_not_nan(self, sig , name):
        #print("check_not_nan");
        if(np.isnan(sig).any()):
            print("PERF_WITH_NAN_IN_SIGNAL" , sig);
            raise tsutil.InternalForecastError("Invalid column for perf ['" + self.mName + "'] '" + name + "'");
        pass

    def compute_R2(self, signal , estimator):
        #return 0.0;
        SST = np.sum((signal.values - np.mean(signal.values))**2);
        SSReg = np.sum((signal.values - estimator.values)**2)
        R2 = 0;
        eps = 1.0e-10;
        if(SST < eps):
            SST = eps;
        R2 = SSReg/SST
        return R2

    def dump_perf_data(self, signal , estimator):
        df = pd.DataFrame();
        df['sig'] = signal.values;
        df['est'] = estimator.values;
        print(df.head());
        print(df.tail());

    def compute(self, signal , estimator, name):
        try:
            # self.dump_perf_data(signal, estimator);
            return self.real_compute(signal, estimator, name);
        except:
            self.dump_perf_data(signal, estimator);
            raise tsutil.InternalForecastError("Failure when computing perf ['" + self.mName + "'] '" + name + "'");
        pass

    def real_compute(self, signal , estimator, name):
        self.mName = name;
        self.check_not_nan(signal.values , "signal")
        self.check_not_nan(estimator.values , "estimator")

        signal_std = np.std(signal);
        estimator_std = np.std(estimator);

        signal1 = self.protect_small_values(signal)
        estimator1 = self.protect_small_values(estimator);

        myerror = (estimator.values - signal.values);
        abs_error = abs(myerror)
        self.mErrorMean = np.mean(myerror)
        self.mErrorStdDev = np.std(myerror)
        self.mMAE = np.mean(abs_error)
        sum_abs = np.abs(signal1.values) + np.abs(estimator1.values)
        self.mMAPE = np.mean(abs(myerror / signal1.values))
        self.mSMAPE = np.mean(abs(myerror) / sum_abs)
        self.mMAPE = round( self.mMAPE , 4 )
        self.mSMAPE = round( self.mSMAPE , 4 )
        self.mL1 = np.mean(abs_error)
        self.mL2 = np.sqrt(np.mean(abs_error ** 2))
        self.mCount = signal.shape[0];
        self.mR2 = self.compute_R2(signal1, estimator1)
        self.mPearsonR = 0.0;
        if((signal_std > 0.0) and (estimator_std > 0.0)):
            (r , pval) = pearsonr(signal1 , estimator1)
            self.mPearsonR = r;
#            print("COMPUTED_PERF_DETAIL " , name, self.mCount ,
#                  self.mErrorMean ,  self.mErrorStdDev ,  self.mMAE ,
#                  self.mMAPE ,  self.mSMAPE , self.mL1 ,  self.mL2 ,
#                  self.mR2 ,  self.mPearsonR);
        pass

    def computeCriterion(self, signal , estimator, criterion):
        self.mCount = signal.shape[0];

        signal1 = self.protect_small_values(signal)
        estimator1 = self.protect_small_values(estimator);

        myerror = (estimator.values - signal.values);
        abs_error = abs(myerror)
        if(criterion == "L1"):
            self.mL1 = np.mean(abs_error)
            return self.mL1;
        if(criterion == "L2"):
            self.mL2 = np.sqrt(np.mean(abs_error ** 2))
            return self.mL2;
        if(criterion == "R2"):
            self.mR2 = self.compute_R2(signal1, estimator1)
            return self.mR2;
        if(criterion == "PEARSONR"):
            (self.mPearsonR , pval) = pearsonr(signal1 , estimator1)
            return self.mPearsonR;
        if(criterion == "MAE"):
            self.mMAE = np.mean(abs_error)
            return self.mAE;
        if(criterion == "SMAPE"):
            self.mSMAPE = np.mean(abs(myerror) / sum_abs)
            self.mSMAPE = round( self.mSMAPE , 4 )
            return self.mSMAPE;
        if(criterion == "COUNT"):
            return self.mCount;

        self.mMAPE = np.mean(abs(myerror / signal1.values))
        self.mMAPE = round( self.mMAPE , 4 )
        return self.mMAPE;

    def getCriterionValue(self, criterion):
        if(criterion == "L1"):
            return self.mL1;
        if(criterion == "L2"):
            return self.mL2;
        if(criterion == "R2"):
            return self.mR2;
        if(criterion == "PEARSONR"):
            return self.mPearsonR;
        if(criterion == "MAE"):
            return self.mAE;
        if(criterion == "SMAPE"):
            return self.mSMAPE;
        if(criterion == "COUNT"):
            return self.mCount;
        return self.mMAPE;

#def date_to_number(x):
#    return  date(int(x) , int(12 * (x - int(x) + 0.01)) + 1 , 1)
<commit_msg>Debugging MAPE computation for too small actual values.<commit_after>import pandas as pd
import numpy as np
import datetime
from scipy.stats import pearsonr

from . import Utils as tsutil

class cPerf:
    def __init__(self):
        self.mErrorStdDev = np.nan;
        self.mErrorMean = np.nan;
        self.mMAE = np.nan;
        self.mMAPE = np.nan;
        self.mSMAPE = np.nan;
        self.mL1 = np.nan;
        self.mL2 = np.nan;
        self.mR2 = np.nan;
        self.mPearsonR = np.nan;
        self.mCount = np.nan;
        self.mName = "No_Name";

    def protect_small_value(self, x , eps):
        if((x < eps) and (x >= 0)):
            return False;
        if((x > -eps) and (x <= 0)):
            return False;
        return True;

    def protect_small_values(self, signal, estimator):
        eps = 1.0e-3;
        keepThis = signal.apply(lambda x : self.protect_small_value(x, eps));
        signal1 =  signal[keepThis];
        estimator1 = estimator[keepThis];
        # self.dump_perf_data(signal , signal1);
        return (signal1 , estimator1);

    def check_not_nan(self, sig , name):
        #print("check_not_nan");
        if(np.isnan(sig).any()):
            print("PERF_WITH_NAN_IN_SIGNAL" , sig);
            raise tsutil.InternalForecastError("INVALID_COLUMN _FOR_PERF ['" + self.mName + "'] '" + name + "'");
        pass

    def compute_R2(self, signal , estimator):
        #return 0.0;
        SST = np.sum((signal.values - np.mean(signal.values))**2);
        SSReg = np.sum((signal.values - estimator.values)**2)
        R2 = 0;
        eps = 1.0e-10;
        if(SST < eps):
            SST = eps;
        R2 = SSReg/SST
        return R2

    def dump_perf_data(self, signal , estimator):
        df = pd.DataFrame();
        df['sig'] = signal.values;
        df['est'] = estimator.values;
        print(df.head());
        print(df.tail());

    def compute(self, signal , estimator, name):
        try:
            # self.dump_perf_data(signal, estimator);
            return self.real_compute(signal, estimator, name);
        except:
            self.dump_perf_data(signal, estimator);
            raise tsutil.InternalForecastError("Failure when computing perf ['" + self.mName + "'] '" + name + "'");
        pass

    def real_compute(self, signal , estimator, name):
        self.mName = name;
        self.check_not_nan(signal.values , "signal")
        self.check_not_nan(estimator.values , "estimator")

        signal_std = np.std(signal);
        estimator_std = np.std(estimator);

        (signal1 , estimator1) = self.protect_small_values(signal, estimator)

        myerror = (estimator.values - signal.values);
        myerror1 = (estimator1.values - signal1.values);
        abs_error = abs(myerror)
        self.mErrorMean = np.mean(myerror)
        self.mErrorStdDev = np.std(myerror)
        self.mMAE = np.mean(abs_error)
        sum_abs = np.abs(signal1.values) + np.abs(estimator1.values)
        self.mMAPE = np.mean(abs(myerror1 / signal1.values))
        self.mSMAPE = np.mean(abs(myerror1) / sum_abs)
        self.mMAPE = round( self.mMAPE , 4 )
        self.mSMAPE = round( self.mSMAPE , 4 )
        self.mL1 = np.mean(abs_error)
        self.mL2 = np.sqrt(np.mean(abs_error ** 2))
        self.mCount = signal.shape[0];
        self.mR2 = self.compute_R2(signal1, estimator1)
        self.mPearsonR = 0.0;
        if((signal_std > 0.0) and (estimator_std > 0.0)):
            (r , pval) = pearsonr(signal , estimator)
            self.mPearsonR = r;
#            print("COMPUTED_PERF_DETAIL " , name, self.mCount ,
#                  self.mErrorMean ,  self.mErrorStdDev ,  self.mMAE ,
#                  self.mMAPE ,  self.mSMAPE , self.mL1 ,  self.mL2 ,
#                  self.mR2 ,  self.mPearsonR);
        pass

    def computeCriterion(self, signal , estimator, criterion):
        self.mCount = signal.shape[0];

        (signal1 , estimator1) = self.protect_small_values(signal, estimator)

        myerror = (estimator.values - signal.values);
        myerror1 = (estimator1.values - signal1.values);
        abs_error = abs(myerror)
        if(criterion == "L1"):
            self.mL1 = np.mean(abs_error)
            return self.mL1;
        if(criterion == "L2"):
            self.mL2 = np.sqrt(np.mean(abs_error ** 2))
            return self.mL2;
        if(criterion == "R2"):
            self.mR2 = self.compute_R2(signal1, estimator1)
            return self.mR2;
        if(criterion == "PEARSONR"):
            (self.mPearsonR , pval) = pearsonr(signal1 , estimator1)
            return self.mPearsonR;
        if(criterion == "MAE"):
            self.mMAE = np.mean(abs_error)
            return self.mAE;
        if(criterion == "SMAPE"):
            self.mSMAPE = np.mean(abs(myerror1) / sum_abs)
            self.mSMAPE = round( self.mSMAPE , 4 )
            return self.mSMAPE;
        if(criterion == "COUNT"):
            return self.mCount;

        self.mMAPE = np.mean(abs(myerror1 / signal1.values))
        self.mMAPE = round( self.mMAPE , 4 )
        return self.mMAPE;

    def getCriterionValue(self, criterion):
        if(criterion == "L1"):
            return self.mL1;
        if(criterion == "L2"):
            return self.mL2;
        if(criterion == "R2"):
            return self.mR2;
        if(criterion == "PEARSONR"):
            return self.mPearsonR;
        if(criterion == "MAE"):
            return self.mAE;
        if(criterion == "SMAPE"):
            return self.mSMAPE;
        if(criterion == "COUNT"):
            return self.mCount;
        return self.mMAPE;

#def date_to_number(x):
#    return  date(int(x) , int(12 * (x - int(x) + 0.01)) + 1 , 1)
<|endoftext|>


<commit_before>Learning how to make commit github<commit_msg>prueba del comit y pull :)<commit_after>Learning how to make commit github

#hola karimito :) prueba del push del git<|endoftext|>


<commit_before>#! /usr/bin/env python3

## Author(s): Kenzo-Hugo Hillion
## Contact(s): kehillio@pasteur.fr
## Python version: 3.5.2+
## Creation : 12-13-2016

'''
Descrition
'''

###########  Import  ###########

# General libraries
import os
import argparse
import sys
import json

# External libraries
import requests

# Class and Objects

###########  Constant(s)  ###########

###########  Class(es)  ###########

class Biotool:

    def __init__(self, name, tool_id, version, description, homepage):
         self.name = name
         self.tool_id = tool_id
         self.version = version
         self.description = description
         self.homepage = homepage
         self.functions = [] # List of Function objects
         self.topics = []    # List of Topic objects
         self.informations = None

class Informations:

    def __init__(self, publication, documentation, credits=None):
        self.publication = publication
        self.documentation = documentation
        self.credits = credits # Credits object
        self.contacts = [] # List of Contact objects

class Credits:

    def __init__(self):
        self.affiliation = []
        self.contributor = []
        self.developer = []
        self.funding = []
        self.infrastructure = []
        self.institution = []

class Contact:

    def __init__(self,contact):
        self.email = contact['contactEmail']
        self.name = contact['contactName']
        # self.role = contact['contactRole']
        # self.tel = contact['contactTel']
        # self.url = contact['contactURL']

class Function:

    def __init__(self,operation):
        self.operation = operation
        self.inputs = [] # List of Inputs objects
        self.outputs = [] # List of Outputs objects

class Data:

    def __init__(self, type, format, description=None):
        self.type = type
        self.format = format
        self.description = description

class Input(Data):

    def __init__(self, type, format, description=None):
        Data.__init__(self, type, format, description)

class Output(Data):

    def __init__(self, type, format, description=None):
        Data.__init__(self, type, format, description)

class Edam:

    def __init__(self, uri, term):
        self.uri = uri
        self.term = term

    def get_edam_id(self):
        '''
        Get the EDAM id from the url
        '''
        return self.uri.split('/')[-1]

class Operation(Edam):

    def __init__(self, uri, term):
        Edam.__init__(self, uri, term)

class Data_type(Edam):

    def __init__(self, uri, term):
        Edam.__init__(self, uri, term)

class Format(Edam):

    def __init__(self, uri, term):
        Edam.__init__(self, uri, term)

class Topic(Edam):

    def __init__(self, uri, term):
        Edam.__init__(self, uri, term)

###########  Function(s)  ###########

def json_from_biotools(tool_id,tool_version):
    '''
    Import JSON of a tool from https://bio.tools

    tool_id: [string]
    tool_version: [string]
    '''
    biotools_link = "https://bio.tools/api/tool/" + tool_id + "/version/" + tool_version
    print ("Loading " + tool_id + ' version ' + tool_version + " from https://bio.tools")
    # Access the entry with requests and get the JSON part
    http_tool = requests.get(biotools_link)
    json_tool = http_tool.json()
    return json_tool

def json_from_file(json_file):
    '''
    Import JSON of a tool from a local file

    json_file: path to the file [string]
    '''
    print ("Loading tool from local JSON file: " + json_file)
    # parse file in JSON format
    with open(json_file,'r') as tool_file:
        json_tool = json.load(tool_file)
    return json_tool

###########  Main  ###########

if __name__ == "__main__":

    ## Parse arguments
    parser = argparse.ArgumentParser(description = 'Generates XML template for Galaxy ')
    subparsers = parser.add_subparsers(dest='COMMAND', help='COMMAND')
    subparsers.required = True

    # Parser for online tool entry
    parser_onl = subparsers.add_parser('online', help='import tool from https://bio.tools')
    parser_onl.add_argument('-i', '--id', help='ID of the tool entry on https://bio.tools',\
                            dest='ID', required=True)
    parser_onl.add_argument('-v', '--version', help='Version of the tool entry', \
                            dest='VERSION', required=True)

    # Parser for local import
    parser_loc = subparsers.add_parser('local', help='import tool from local JSON file')
    parser_loc.add_argument('-j', '--json_file', help='Path to JSON file (Giving a JSON ' + \
                            'file skips the query from https://bio.tools', dest='JSON_FILE', \
                            required=True)

    try:
        args = parser.parse_args()
    except SystemExit:
        if len(sys.argv) == 1:
            parser.print_help()
        elif sys.argv[1] == 'online':
            parser_onl.print_help()
        elif sys.argv[1] == 'local':
            parser_loc.print_help()
        else:
            parser.print_help()
        sys.exit(1)

    ## Extra tests for arguments

    ###########################################################

    ## MAIN

    # Get JSON of the tool
    if args.COMMAND == 'online':
        json_tool = json_from_biotools(args.ID,args.VERSION)
    elif args.COMMAND == 'local':
        json_tool = json_from_file(args.JSON_FILE)
<commit_msg>update classes and add methods to initialize Biotool object<commit_after>#! /usr/bin/env python3

## Author(s): Kenzo-Hugo Hillion
## Contact(s): kehillio@pasteur.fr
## Python version: 3.5.2+
## Creation : 12-13-2016

'''
Descrition
'''

###########  Import  ###########

# General libraries
import os
import argparse
import sys
import json

# External libraries
import requests

# Class and Objects

###########  Constant(s)  ###########

###########  Class(es)  ###########

class Biotool:

    def __init__(self, name, tool_id, version, description, homepage):
         '''
         name: [STRING]
         tool_id: [STRING]
         version: [STRING]
         description: [STRING]
         homepage: URL to homepage [STRING]
         '''
         self.name = name
         self.tool_id = tool_id
         self.version = version
         self.description = description
         self.homepage = homepage
         self.functions = [] # List of Function objects
         self.topics = []    # List of Topic objects
         self.informations = None # Information object

    def set_informations(self, credits, contacts, publications, docs):
        '''
        Add an information object to the biotool

        credits: [DICT]
        contacts: list of contacts [LIST]
        publications: different IDs for publications [DICT]
        doc: link for the document [STRING]
        '''
        self.informations = Informations(publications['publicationsPrimaryID'],\
                                         docs['docsHome'], credits)
        for c in contacts:
            self.informations.contacts.append(Contact(c))

    def add_functions(self, functions):
        '''
        Add functions to the list

        function: [LIST] of [DICT]
        '''
        for f in functions:
            # Create Function object
            function = Function(f['functionName'])
            function.add_inputs(f['input'])
            function.add_outputs(f['output'])
            # Append object to the biotool
            self.functions.append(function)

    def add_topics(self, topics):
        '''
        Add topics to the list
        '''
        for t in topics:
            self.topics.append(Topic(t))

class Informations:

    def __init__(self, publication, documentation, credits):
        self.publication = publication
        self.documentation = documentation
        self.credits = Credits(credits) # Credits object
        self.contacts = [] # List of Contact objects

class Credits:

    def __init__(self, credits):
        self.affiliation = credits['creditsAffiliation']
        self.contributor = credits['creditsContributor']
        self.developer = credits['creditsDeveloper']
        self.funding = credits['creditsFunding']
        self.infrastructure = credits['creditsInfrastructure']
        self.institution = credits['creditsInstitution']

class Contact:

    def __init__(self,contact):
        self.email = contact['contactEmail']
        self.name = contact['contactName']
        # self.role = contact['contactRole']
        # self.tel = contact['contactTel']
        # self.url = contact['contactURL']

class Function:

    def __init__(self,edams):
        '''
        edams: [LIST] of [DICT] of EDAM ontology for operation(s) with uri and term
        '''
        self.operations = [] # List of Operation objects
        for e in edams:
            self.operations.append(Operation(e))
        self.inputs = [] # List of Inputs objects
        self.outputs = [] # List of Outputs objects

    def add_inputs(self, inputs):
        '''
        inputs: [LIST] of [DICT] inputs
        '''
        for i in inputs:
            # Create Input object and appends to the list
            self.inputs.append(Input(i['dataType'],i['dataFormat'],i['dataDescription']))

    def add_outputs(self, outputs):
        '''
        outputs: [LIST] of [DICT] outputs
        '''
        for o in outputs:
            # Create Output object and appends to the list
            self.outputs.append(Output(o['dataType'],o['dataFormat'],o['dataDescription']))

class Data:

    def __init__(self, data_type, formats, description=None):
        '''
        data_type: [DICT] of EDAM ontology for data types with uri and term
        formats: [LIST] of [DICT] of EDAM ontology for data formats with uri and term
        description: [STRING]
        '''
        self.data_type = Data_type(data_type)
        self.formats = [] # List of Format objects
        for f in formats:
            self.formats.append(Format(f))
        self.description = description

class Input(Data):

    def __init__(self, data_type, format, description=None):
        '''
        data_type: [DICT] of EDAM ontology for data types with uri and term
        format: [DICT] of EDAM ontology for data formats with uri and term
        description: [STRING]
        '''
        Data.__init__(self, data_type, format, description)

class Output(Data):

    def __init__(self, data_type, format, description=None):
        '''
        data_type: [DICT] of EDAM ontology for data types with uri and term
        format: [DICT] of EDAM ontology for data formats with uri and term
        description: [STRING]
        '''
        Data.__init__(self, data_type, format, description)

class Edam:

    def __init__(self, edam):
        '''
        edam: [DICT] of EDAM ontology with uri and term
        '''
        self.uri = edam['uri']
        self.term = edam['term']

    def get_edam_id(self):
        '''
        Get the EDAM id from the url
        '''
        return self.uri.split('/')[-1]

class Operation(Edam):

    def __init__(self, edam):
        '''
        edam: [DICT] of EDAM ontology for operations with uri and term
        '''
        Edam.__init__(self, edam)

class Data_type(Edam):

    def __init__(self, edam):
        '''
        edam: [DICT] of EDAM ontology for data type with uri and term
        '''
        Edam.__init__(self, edam)

class Format(Edam):

    def __init__(self, edam):
        '''
        edam: [DICT] of EDAM ontology for formats with uri and term
        '''
        Edam.__init__(self, edam)

class Topic(Edam):

    def __init__(self, edam):
        '''
        edam: [DICT] of EDAM ontology for topics with uri and term
        '''
        Edam.__init__(self, edam)

###########  Function(s)  ###########

def json_from_biotools(tool_id,tool_version):
    '''
    Import JSON of a tool from https://bio.tools

    tool_id: [string]
    tool_version: [string]
    '''
    biotools_link = "https://bio.tools/api/tool/" + tool_id + "/version/" + tool_version
    print ("Loading " + tool_id + ' version ' + tool_version + " from https://bio.tools")
    # Access the entry with requests and get the JSON part
    http_tool = requests.get(biotools_link)
    json_tool = http_tool.json()
    return json_tool

def json_from_file(json_file):
    '''
    Import JSON of a tool from a local file

    json_file: path to the file [string]
    '''
    print ("Loading tool from local JSON file: " + json_file)
    # parse file in JSON format
    with open(json_file,'r') as tool_file:
        json_tool = json.load(tool_file)
    return json_tool

###########  Main  ###########

if __name__ == "__main__":

    ## Parse arguments
    parser = argparse.ArgumentParser(description = 'Generates XML template for Galaxy ')
    subparsers = parser.add_subparsers(dest='COMMAND', help='COMMAND')
    subparsers.required = True

    # Parser for online tool entry
    parser_onl = subparsers.add_parser('online', help='import tool from https://bio.tools')
    parser_onl.add_argument('-i', '--id', help='ID of the tool entry on https://bio.tools',\
                            dest='ID', required=True)
    parser_onl.add_argument('-v', '--version', help='Version of the tool entry', \
                            dest='VERSION', required=True)

    # Parser for local import
    parser_loc = subparsers.add_parser('local', help='import tool from local JSON file')
    parser_loc.add_argument('-j', '--json_file', help='Path to JSON file (Giving a JSON ' + \
                            'file skips the query from https://bio.tools', dest='JSON_FILE', \
                            required=True)

    try:
        args = parser.parse_args()
    except SystemExit:
        if len(sys.argv) == 1:
            parser.print_help()
        elif sys.argv[1] == 'online':
            parser_onl.print_help()
        elif sys.argv[1] == 'local':
            parser_loc.print_help()
        else:
            parser.print_help()
        sys.exit(1)

    ## Extra tests for arguments

    ###########################################################

    ## MAIN

    # Get JSON of the tool
    if args.COMMAND == 'online':
        json_tool = json_from_biotools(args.ID,args.VERSION)
    elif args.COMMAND == 'local':
        json_tool = json_from_file(args.JSON_FILE)
<|endoftext|>


<commit_before>from DataSet import DataSet
import multiprocessing
from random import shuffle
from PIL import Image
import numpy as np
from timeit import default_timer as timer

class Trainer():
    def __init__(self, dataset, flatten, test_p, even_class_n, width, height):
        self.dataset = dataset
        self.normalizer = 255.0
        self.width = width
        self.height = height

        ###all class filenames in a dict, key is class name, value is an array holding tuples of fname and one hot correct label
        self.class_fnames = self.dataset.get_class_fnames()

        n_class_samples = [len(self.class_fnames[c]) for c in self.class_fnames]
        print("CLASS SAMPLES")
        print(n_class_samples)

        ###shuffle the order of the class fnames
        for c in self.class_fnames:
           shuffle(self.class_fnames[c])

        if test_p < 0.0 or test_p > 1.0:
            test_p = 0.15

        ###determine the number of test data points given the desired test proportion
        self.train_data = []
        self.test_data = []
        if even_class_n is True:
            min_samples = min(n_class_samples)
            #min_samples = 800
            print("MIN SAMPLES "+str(min_samples))
            self.test_n = int(min_samples*test_p)
            print("TEST N per class "+str(self.test_n))
            self.train_n = min_samples - self.test_n
            print("TRAIN N per class "+str(self.train_n))
            ###get minimum number of data points from each class, seperate into test and train sets
            for c in self.class_fnames:
                self.train_data.extend(self.class_fnames[c][:self.train_n])
                self.test_data.extend(self.class_fnames[c][self.train_n:self.test_n+self.train_n])

        shuffle(self.train_data)
        shuffle(self.test_data)
        ###set function whether to flatten image for traditional feedforward or keep shape for convolutional
        if flatten is True:
            self.get_img = self.get_img_data_flatten
        else:
            self.get_img = self.get_img_data
        #if class_samples.count(class_samples[0]) == len(class_samples)
        print("TOTAL TEST SAMPLES "+str(len(self.test_data)))
        print("TOTAL TRAIN SAMPLES "+str(len(self.train_data)))
        self.j = 0

    def get_img_data(self, fname):
        img = Image.open(fname).resize((self.width, self.height))
        #return np.expand_dims(np.array(Image.open(fname))/self.normalizer, 0)
        return np.expand_dims(np.array(img)/self.normalizer, 0)

    def get_img_data_flatten(self, fname):
        return np.expand_dims(np.array(Image.open(fname)).flatten()/self.normalizer, 0)

    def get_train_batch(self, batch_size):
        if self.j+batch_size < len(self.train_data):
            batch = self.train_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator, and shuffle train data
            batch = self.train_data[self.j:]
            self.j = 0
            epoch_over = True
            shuffle(self.train_data)
            #print("test_over")
        train_batch, one_hots = self.fnames_to_batch(batch)
        return train_batch, one_hots, epoch_over

    def get_test_batch(self, batch_size):
        if self.j+batch_size < len(self.test_data):
            batch = self.test_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator,
            batch = self.test_data[self.j:]
            self.j = 0
            epoch_over = True
        train_batch, one_hots = self.fnames_to_batch(batch)
        return train_batch, one_hots, epoch_over

    def fnames_to_batch(self, fnames_and_one_hot):
        batch = []
        one_hots = []

        #get image data using PILLOW
        for f in fnames_and_one_hot:
            batch.append(self.get_img(f[0]))
            one_hots.append(f[1])

        batch = np.concatenate(batch, 0)
        one_hots = np.concatenate(one_hots, 0)
        return batch, one_hots

    def get_batch_and_fnames(self, batch_size):
        if self.j+batch_size < len(self.test_data):
            batch = self.test_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator,
            batch = self.test_data[self.j:]
            self.j = 0
            epoch_over = True
        train_batch, one_hots = self.fnames_to_batch(batch)
        #return image data and fnames
        return train_batch, [b[0] for b in batch], epoch_over
<commit_msg>added width and height to flatten, removed some old comments<commit_after>from DataSet import DataSet
import multiprocessing
from random import shuffle
from PIL import Image
import numpy as np
from timeit import default_timer as timer

class Trainer():
    def __init__(self, dataset, flatten, test_p, even_class_n, width, height):
        self.dataset = dataset
        self.normalizer = 255.0
        self.width = width
        self.height = height

        ###all class filenames in a dict, key is class name, value is an array holding tuples of fname and one hot correct label
        self.class_fnames = self.dataset.get_class_fnames()

        n_class_samples = [len(self.class_fnames[c]) for c in self.class_fnames]
        print("CLASS SAMPLES")
        print(n_class_samples)

        ###shuffle the order of the class fnames
        for c in self.class_fnames:
           shuffle(self.class_fnames[c])

        if test_p < 0.0 or test_p > 1.0:
            test_p = 0.15

        ###determine the number of test data points given the desired test proportion
        self.train_data = []
        self.test_data = []
        if even_class_n is True:
            min_samples = min(n_class_samples)
            print("MIN SAMPLES "+str(min_samples))
            self.test_n = int(min_samples*test_p)
            print("TEST N per class "+str(self.test_n))
            self.train_n = min_samples - self.test_n
            print("TRAIN N per class "+str(self.train_n))
            ###get minimum number of data points from each class, seperate into test and train sets
            for c in self.class_fnames:
                self.train_data.extend(self.class_fnames[c][:self.train_n])
                self.test_data.extend(self.class_fnames[c][self.train_n:self.test_n+self.train_n])

        shuffle(self.train_data)
        shuffle(self.test_data)
        ###set function whether to flatten image for traditional feedforward or keep shape for convolutional
        if flatten is True:
            self.get_img = self.get_img_data_flatten
        else:
            self.get_img = self.get_img_data

        print("TOTAL TEST SAMPLES "+str(len(self.test_data)))
        print("TOTAL TRAIN SAMPLES "+str(len(self.train_data)))
        self.j = 0

    def get_img_data(self, fname):
        img = Image.open(fname).resize((self.width, self.height))
        return np.expand_dims(np.array(img)/self.normalizer, 0)

    def get_img_data_flatten(self, fname):
        img = Image.open(fname).resize((self.width, self.height))
        return np.expand_dims(np.array(img).flatten()/self.normalizer, 0)

    def get_train_batch(self, batch_size):
        if self.j+batch_size < len(self.train_data):
            batch = self.train_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator, and shuffle train data
            batch = self.train_data[self.j:]
            self.j = 0
            epoch_over = True
            shuffle(self.train_data)
        train_batch, one_hots = self.fnames_to_batch(batch)
        return train_batch, one_hots, epoch_over

    def get_test_batch(self, batch_size):
        if self.j+batch_size < len(self.test_data):
            batch = self.test_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator,
            batch = self.test_data[self.j:]
            self.j = 0
            epoch_over = True
        train_batch, one_hots = self.fnames_to_batch(batch)
        return train_batch, one_hots, epoch_over

    def fnames_to_batch(self, fnames_and_one_hot):
        batch = []
        one_hots = []

        #get image data using PILLOW
        for f in fnames_and_one_hot:
            batch.append(self.get_img(f[0]))
            one_hots.append(f[1])

        batch = np.concatenate(batch, 0)
        one_hots = np.concatenate(one_hots, 0)
        return batch, one_hots

    def get_batch_and_fnames(self, batch_size):
        if self.j+batch_size < len(self.test_data):
            batch = self.test_data[self.j:self.j+batch_size]
            self.j += batch_size
            epoch_over = False
        else:
            ###at end of train data, use up last of data, reset iterator,
            batch = self.test_data[self.j:]
            self.j = 0
            epoch_over = True
        train_batch, one_hots = self.fnames_to_batch(batch)
        #return image data and fnames
        return train_batch, [b[0] for b in batch], epoch_over
<|endoftext|>


<commit_before>from ACJ import ACJ
import numpy as np
import pickle
import random

if __name__ == "__main__":
    np.set_printoptions(precision=2)
    rounds = 15
    length = 100
    errBase = 0.0
    judges = 3
    true = np.asarray([i+1 for i in range(length)])
    dat = true[:]
    np.random.shuffle(dat)
    acj = ACJ(dat, rounds)
    i = 0
    reviewer = "Me"
    with open(r"acj.pkl", "wb") as output_file:
        pickle.dump(acj, output_file)
    del(acj)
    with open(r"acj.pkl", "rb") as input_file:
        acj = pickle.load(input_file)
    while (True):
        i = i+1
        if (acj.step == 0):
            print(acj.reliability())
        x = acj.nextPair()
        if (x == None):
            break
        err = errBase/np.abs(x[0]-x[1])
        if random.random()<err:
            res = x[0]<x[1]
	    print("hi")
        else:
            res = x[0]>x[1]
        y = []
        y.append(x[1])
        y.append(x[0])
        acj.comp(y, result = not res, reviewer = reviewer)
        #with open(r"acj.pkl", "wb") as output_file:
        #    pickle.dump(acj, output_file)
        #del(acj)
        #with open(r"acj.pkl", "rb") as input_file:
        #    acj = pickle.load(input_file)

    diff = (acj.rankings()[1]-acj.rankings()[1].min())*100/(acj.rankings()[1].max()-acj.rankings()[1].min())
    print(diff)

    rank = acj.rankings()[0]
    val = acj.rankings()[1]
    acc = np.sum(np.abs(true-rank))/length
    worst = np.max(np.abs(true-rank))
    print(rank)
    print(acj.reliability())
    print(acc)
    print(worst)
<commit_msg>Changed method for printing rank<commit_after>from ACJ import ACJ
import numpy as np
import pickle
import random

if __name__ == "__main__":
    np.set_printoptions(precision=2)
    rounds = 15
    length = 100
    errBase = 0.0
    judges = 3
    true = np.asarray([i+1 for i in range(length)])
    dat = true[:]
    np.random.shuffle(dat)
    acj = ACJ(dat, rounds)
    i = 0
    reviewer = "Me"
    with open(r"acj.pkl", "wb") as output_file:
        pickle.dump(acj, output_file)
    del(acj)
    with open(r"acj.pkl", "rb") as input_file:
        acj = pickle.load(input_file)
    while (True):
        i = i+1
        if (acj.step == 0):
            print(acj.reliability())
        x = acj.nextPair()
        if (x == None):
            break
        err = errBase/np.abs(x[0]-x[1])
        if random.random()<err:
            res = x[0]<x[1]
        else:
            res = x[0]>x[1]
        y = []
        y.append(x[1])
        y.append(x[0])
        acj.comp(y, result = not res, reviewer = reviewer)
        #with open(r"acj.pkl", "wb") as output_file:
        #    pickle.dump(acj, output_file)
        #del(acj)
        #with open(r"acj.pkl", "rb") as input_file:
        #    acj = pickle.load(input_file)

    diff = (acj.rankings()[1]-acj.rankings()[1].min())*100/(acj.rankings()[1].max()-acj.rankings()[1].min())
    print(diff)

    rank = acj.rankings()[0]
    val = acj.rankings()[1]
    acc = np.sum(np.abs(true-rank))/length
    worst = np.max(np.abs(true-rank))
    print(rank)
    print(acj.reliability())
    print(acc)
    print(worst)
<|endoftext|>


<commit_before>print("welcome to le name generator where we generate v cheap gud names 4 u ok")
from random import choice
name = ''
name += choice ('bcdfghjklmnpqrstvwxyz')+('aj')+('esh')+('52')
print(name)
<commit_msg>8===D<commit_after>print("welcome to le name generator where we generate v cheap gud names 4 u ok")
from random import choice
name = ''
name += choice ('bcdfghjklmnpqrstvwxyz')+('aj')+('esh')
print(name)
<|endoftext|>


<commit_before>import twisted.python
import http_client
from zope.interface import implements
from twisted.application import service
from twisted.python import modules
from twisted.internet import defer
from twisted.python import log
from twisted.python import rebuild
plugins = modules.getModule('plugins_enabled')
def plugin_method(priority):
    def wrapper(function):
        return(plugins.load().PluginMethod(function=function,
                                           priority=priority))
    return wrapper

class Dispatcher(object):
    implements(service.IServiceCollection)
    def __init__(self):
        self.cache = http_client.HTTPClient("cache")
        self._services = dict()
        self._rehash()

    def addService(self, s):
        if str(s) in self._services:
            raise RuntimeError("Two services with same name not allowed")
        self._services[str(s)] = s

    def removeService(self, s):
        if str(s) in self._services:
            del self._services[str(s)]

    def __iter__(self):
        for s in self._services:
            yield self._services[s]

    def getServiceNamed(self, name):
        if name not in self._services:
            raise KeyError(name)
        return self._services[name]

    @plugin_method('high')
    def cmd_rehash(self, command, parameters, user, channel):
        return self._rehash()

    def _rehash(self):
        log.msg("Beginning rehash")
        _service = service
        for plugin in list(self):
            plugin.disownServiceParent()
        rebuild.updateInstance(self.cache)
        rebuild.updateInstance(self)
        BasePlugin = plugins.load().BasePlugin
        for iterator in plugins.walkModules():
            plugin_module = iterator.load()
            for plugin_name in dir(plugin_module):
                plugin_class = getattr(plugin_module, plugin_name)
                if plugin_class is BasePlugin:
                    continue
                try:
                    if issubclass(plugin_class, BasePlugin):
                        plugin_class().setServiceParent(self)
                except TypeError:
                    continue
        return "Succesfully rehashed"

    @defer.inlineCallbacks
    def __call__(self, result, **kwargs):
        pass
        try:
            plugin_method = getattr(self, result)
        except AttributeError:
            pass
        else:
            if plugin_method is PluginMethod:
                yield plugin_method(**kwargs)
        for plugin in self:
            try:
                plugin_method = getattr(plugin, result)
            except AttributeError:
                pass
            else:
                if plugin_method is PluginMethod:
                    yield plugin_method(**kwargs)

<commit_msg>Make dispatcher point to the right place<commit_after>import twisted.python
import http_client
from zope.interface import implements
from twisted.application import service
from twisted.python import modules
from twisted.internet import defer
from twisted.python import log
from twisted.python import rebuild
import yaml
plugins = modules.getModule('plugins')
def plugin_method(priority):
    def wrapper(function):
        return(plugins.load().PluginMethod(function=function,
                                           priority=priority))
    return wrapper

class Dispatcher(object):
    implements(service.IServiceCollection)
    def __init__(self):
        self.cache = http_client.HTTPClient("cache")
        self._services = dict()
        self._config = dict()
        self._rehash()

    def addService(self, s):
        if str(s) in self._services:
            raise RuntimeError("Two services with same name not allowed")
        self._services[str(s)] = s

    def removeService(self, s):
        if str(s) in self._services:
            del self._services[str(s)]

    def __iter__(self):
        for s in self._services:
            yield self._services[s]

    def getServiceNamed(self, name):
        if name not in self._services:
            raise KeyError(name)
        return self._services[name]

    @plugin_method('high')
    def cmd_rehash(self, command, parameters, user, channel):
        return self._rehash()

    def _rehash(self):
        log.msg("Beginning rehash")
        _service = service
        for plugin in list(self):
            plugin.disownServiceParent()
        with open('config.yaml') as f:
            self._config['plugins'] = yaml.load(f).get('plugins', [])
        rebuild.updateInstance(self.cache)
        rebuild.updateInstance(self)
        BasePlugin = plugins.load().BasePlugin
        for iterator in plugins.walkModules():
            plugin_module = iterator.load()
            prefix, _, name = plugin_module.__name__.partition('.')
            if name not in self._config['plugins']:
                continue
            for plugin_name in dir(plugin_module):
                plugin_class = getattr(plugin_module, plugin_name)
                if plugin_class is BasePlugin:
                    continue
                try:
                    if issubclass(plugin_class, BasePlugin):
                        plugin_class().setServiceParent(self)
                except TypeError:
                    continue
        return "Succesfully rehashed"

    @defer.inlineCallbacks
    def __call__(self, result, **kwargs):
        pass
        try:
            plugin_method = getattr(self, result)
        except AttributeError:
            pass
        else:
            if plugin_method is PluginMethod:
                yield plugin_method(**kwargs)
        for plugin in self:
            try:
                plugin_method = getattr(plugin, result)
            except AttributeError:
                pass
            else:
                if plugin_method is PluginMethod:
                    yield plugin_method(**kwargs)

<|endoftext|>


<commit_before>from object_storage import ObjectStore
from openpathsampling.sample import SampleSet, Sample

import time

class SampleStore(ObjectStore):
    def __init__(self, storage):
        super(SampleStore, self).__init__(storage, Sample, json=False)

    def save(self, sample, idx=None):
        if idx is not None:
            self.storage.trajectories.save(sample.trajectory)
            self.set_object('sample_trajectory', idx, sample.trajectory)

            self.storage.ensembles.save(sample.ensemble)
            self.set_object('sample_ensemble', idx, sample.ensemble)

            self.save_variable('sample_replica', idx, sample.replica)

            if sample.step is None:
                self.save_variable('sample_step', idx, -1)
            else:
                self.save_variable('sample_step', idx, sample.step)

    def load(self, idx):
        '''
        Return a sample from the storage

        Parameters
        ----------
        idx : int
            index of the sample

        Returns
        -------
        sample : Sample
            the sample
        '''
        trajectory_idx = int(self.storage.variables['sample_trajectory_idx'][idx])
        ensemble_idx = int(self.storage.variables['sample_ensemble_idx'][idx])
        replica_idx = int(self.storage.variables['sample_replica'][idx])
        step=self.load_variable('sample_step', idx)

        obj = Sample(
            trajectory=self.storage.trajectories.load(trajectory_idx),
            replica=replica_idx,
            ensemble=self.storage.ensembles.load(ensemble_idx),
            step=step
        )

        return obj

    def by_ensemble(self, ensemble):
        return [ sample for sample in self.iterator() if sample.ensemble == ensemble ]

    def _init(self, units=None):
        super(SampleStore, self)._init(units)

        # New short-hand definition
        self.init_variable('sample_trajectory_idx', 'index', chunksizes=(1, ))
        self.init_variable('sample_ensemble_idx', 'index', chunksizes=(1, ))
        self.init_variable('sample_replica', 'index', chunksizes=(1, ))
        self.init_variable('sample_step', 'index', chunksizes=(1, ))

class SampleSetStore(ObjectStore):

    def __init__(self, storage):
        super(SampleSetStore, self).__init__(storage, SampleSet, json=False)

    def save(self, sample_set, idx=None):
        # Check if all samples are saved
        map(self.storage.samples.save, sample_set)

        values = self.list_to_numpy(sample_set, 'sample')
        self.storage.variables['sampleset_sample_idx'][idx] = values

        self.storage.pathmovechanges.save(sample_set.movepath)
        self.set_object('sampleset_movepath', idx, sample_set.movepath)

    def sample_indices(self, idx):
        '''
        Load sample indices for sample_set with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            ID of the sample_set

        Returns
        -------
        list of int
            list of sample indices
        '''

        # get the values
        values = self.storage.variables['sampleset_sample_idx'][idx]

        # typecast to integer
        return self.list_from_numpy(values, 'index')

    def load(self, idx):
        '''
        Return a sample_set from the storage

        Parameters
        ----------
        idx : int
            index of the sample_set (counts from 0)

        Returns
        -------
        sample_set
            the sample_set
        '''

        movepath_idx = int(self.storage.variables['sampleset_movepath_idx'][idx])

        values = self.storage.variables['sampleset_sample_idx'][idx]

        # typecast to sample
        samples = self.list_from_numpy(values, 'samples')
        sample_set = SampleSet(samples, movepath=self.storage.pathmovechanges.load(movepath_idx))

        return sample_set

    def _init(self, units=None):
        """
        Initialize the associated storage to allow for sampleset storage

        """
        super(SampleSetStore, self)._init()

        self.init_variable('sampleset_sample_idx', 'index', 'sampleset',
            description="sampleset[sampleset][frame] is the sample index (0..nspanshots-1) of frame 'frame' of sampleset 'sampleset'.",
            variable_length = True,
            chunksizes=(1024, )
        )

        self.init_variable('sampleset_movepath_idx', 'index', chunksizes=(1, ))
<commit_msg>updated Sample Store<commit_after>from object_storage import ObjectStore
from openpathsampling.sample import SampleSet, Sample

import time

class SampleStore(ObjectStore):
    def __init__(self, storage):
        super(SampleStore, self).__init__(storage, Sample, json=False)

    def save(self, sample, idx=None):
        if idx is not None:
            self.storage.trajectories.save(sample.trajectory)
            self.set_object('sample_trajectory', idx, sample.trajectory)

            self.storage.ensembles.save(sample.ensemble)
            self.set_object('sample_ensemble', idx, sample.ensemble)

            self.save_variable('sample_replica', idx, sample.replica)

            if sample.step is None:
                self.save_variable('sample_step', idx, -1)
            else:
                self.save_variable('sample_step', idx, sample.step)

            self.save_variable('sample_parent', idx, sample.parent)
            self.save_variable('sample_details', idx, sample.details)
            self.save_variable('sample_valid', idx, sample.valid)
            self.save_variable('sample_accepted', idx, sample.accepted)

    def load(self, idx):
        '''
        Return a sample from the storage

        Parameters
        ----------
        idx : int
            index of the sample

        Returns
        -------
        sample : Sample
            the sample
        '''
        trajectory_idx = int(self.storage.variables['sample_trajectory_idx'][idx])
        ensemble_idx = int(self.storage.variables['sample_ensemble_idx'][idx])
        replica_idx = int(self.storage.variables['sample_replica'][idx])
        parent_idx = int(self.storage.variables['sample_parent'][idx])
        details_idx = int(self.storage.variables['sample_details'][idx])
        step=self.load_variable('sample_step', idx)
        valid=self.load_variable('sample_valid', idx)
        accepted=bool(self.load_variable('sample_accepted', idx))

        obj = Sample(
            trajectory=self.storage.trajectories[trajectory_idx],
            replica=replica_idx,
            ensemble=self.storage.ensembles[ensemble_idx],
            step=step,
            valid=valid,
            parent=self.storage.samples[parent_idx],
            details=self.storage.movedetails[details_idx],
            accepted=accepted
        )

        return obj

    def by_ensemble(self, ensemble):
        return [ sample for sample in self.iterator() if sample.ensemble == ensemble ]

    def _init(self, units=None):
        super(SampleStore, self)._init(units)

        # New short-hand definition
        self.init_variable('sample_trajectory_idx', 'index', chunksizes=(1, ))
        self.init_variable('sample_ensemble_idx', 'index', chunksizes=(1, ))
        self.init_variable('sample_replica', 'index', chunksizes=(1, ))
        self.init_variable('sample_step', 'index', chunksizes=(1, ))
        self.init_variable('sample_parent', 'index', chunksizes=(1, ))
        self.init_variable('sample_valid', 'index', chunksizes=(1, ))
        self.init_variable('sample_details', 'index', chunksizes=(1, ))
        self.init_variable('sample_accepted', 'index', chunksizes=(1, ))

class SampleSetStore(ObjectStore):

    def __init__(self, storage):
        super(SampleSetStore, self).__init__(storage, SampleSet, json=False)

    def save(self, sample_set, idx=None):
        # Check if all samples are saved
        map(self.storage.samples.save, sample_set)

        values = self.list_to_numpy(sample_set, 'sample')
        self.storage.variables['sampleset_sample_idx'][idx] = values

        self.storage.pathmovechanges.save(sample_set.movepath)
        self.set_object('sampleset_movepath', idx, sample_set.movepath)

    def sample_indices(self, idx):
        '''
        Load sample indices for sample_set with ID 'idx' from the storage

        Parameters
        ----------
        idx : int
            ID of the sample_set

        Returns
        -------
        list of int
            list of sample indices
        '''

        # get the values
        values = self.storage.variables['sampleset_sample_idx'][idx]

        # typecast to integer
        return self.list_from_numpy(values, 'index')

    def load(self, idx):
        '''
        Return a sample_set from the storage

        Parameters
        ----------
        idx : int
            index of the sample_set (counts from 0)

        Returns
        -------
        sample_set
            the sample_set
        '''

        movepath_idx = int(self.storage.variables['sampleset_movepath_idx'][idx])

        values = self.storage.variables['sampleset_sample_idx'][idx]

        # typecast to sample
        samples = self.list_from_numpy(values, 'samples')
        sample_set = SampleSet(samples, movepath=self.storage.pathmovechanges.load(movepath_idx))

        return sample_set

    def _init(self, units=None):
        """
        Initialize the associated storage to allow for sampleset storage

        """
        super(SampleSetStore, self)._init()

        self.init_variable('sampleset_sample_idx', 'index', 'sampleset',
            description="sampleset[sampleset][frame] is the sample index (0..nspanshots-1) of frame 'frame' of sampleset 'sampleset'.",
            variable_length = True,
            chunksizes=(1024, )
        )

        self.init_variable('sampleset_movepath_idx', 'index', chunksizes=(1, ))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Module exports :class:`TromansEtAl2019`
"""
import numpy as np
from openquake.hazardlib.gsim.base import registry, GMPE, CoeffsTable
from openquake.hazardlib import const
from openquake.hazardlib.imt import from_string

#: Coefficient tables as per annex B of Abrahamson et al. (2014)
ASK_TAU_COEFFS = CoeffsTable(sa_damping=5, table="""\
    IMT       s3     s4
    pga     0.47   0.36
    pgv     0.38   0.38
    0.010   0.47   0.36
    0.020   0.47   0.36
    0.030   0.47   0.36
    0.050   0.47   0.36
    0.075   0.47   0.36
    0.100   0.47   0.36
    0.150   0.47   0.36
    0.200   0.47   0.36
    0.250   0.47   0.36
    0.300   0.47   0.36
    0.400   0.47   0.36
    0.500   0.47   0.36
    0.750   0.47   0.36
    1.000   0.47   0.36
    1.500   0.47   0.36
    2.000   0.47   0.36
    3.000   0.47   0.36
    4.000   0.47   0.36
    5.000   0.47   0.36
    6.000   0.47   0.36
    7.500   0.47   0.36
    10.00   0.47   0.36
    """)

#: Single station intra-event term for the constant and magnitude-dependent
#: model, as provided in Table 4 of Rodriguez-Marek et al (2013)
PHI_SS_COEFFS = CoeffsTable(sa_damping=5, table="""\
    IMT   const_phiss   phi1m  phi2m
    pgv          0.46    0.49   0.37
    pga          0.46    0.49   0.35
    0.01         0.46    0.49   0.35
    0.1          0.45    0.45   0.43
    0.2          0.48    0.51   0.37
    0.3          0.48    0.51   0.37
    0.5          0.46    0.49   0.37
    1.0          0.45    0.46   0.40
    3.0          0.41    0.41   0.41
    """)

#: Single ststion dfS2S model
DELTA_PHI_S2S = CoeffsTable(sa_damping=5, table="""\
    IMT          dfs2s
    pga         0.0000
    0.01000     0.0000
    0.99999     0.0000
    1.00000     0.0001
    2.00000     0.0069
    3.00000     0.0131
    10.0000     0.0131
    """)

def get_heteroskedastic_tau(imt, mag):
    """
    Returns the magnitude dependent inter-event variability using the
    model of Abrahamson et al (2014).
    :param dict C:
        Coefficients dictionary
    :param float mag:
        Magnitude
    """

    C = ASK_TAU_COEFFS[imt]
    if mag < 5:
        tau = C['s3']
    elif mag <= 7:
        tau = C['s3'] + (C['s4'] - C['s3']) / 2. * (mag - 5.)
    else:
        tau = C['s4']
    return tau

def get_heteroskedastic_phi(imt, mag):
    """
    Returns the heteroskedastic intra-event term, taken as the maximum
    of the constant single-station phi and the magnitude dependent
    single-station phi provided in Table 4 of Rodriguez-Marek et al (2014)
    """
    C = PHI_SS_COEFFS[imt]
    if mag < 5.0:
        mag_phi = C["phi1m"]
    elif mag > 7.0:
        mag_phi = C["phi2m"]
    else:
        mag_phi = C["phi1m"] + (C["phi2m"] - C["phi1m"]) * ((mag - 5.0) / 2.0)
    if mag_phi > C["const_phiss"]:
        return mag_phi
    else:
        return C["const_phiss"]

HETEROSKEDASTIC_PHI = {
    "upper": lambda imt, mag: 1.16 * get_heteroskedastic_phi(imt, mag),
    "central": lambda imt, mag: get_heteroskedastic_phi(imt, mag),
    "lower": lambda imt, mag: 0.84 * get_heteroskedastic_phi(imt, mag)}

HOMOSKEDASTIC_PHI = {
    "upper": lambda imt: 1.16 * PHI_SS_COEFFS[imt]["const_phiss"],
    "central": lambda imt: PHI_SS_COEFFS[imt]["const_phiss"],
    "lower": lambda imt: 0.84 * PHI_SS_COEFFS[imt]["const_phiss"]}

HETEROSKEDASTIC_TAU = {
    "upper": lambda imt, mag: get_heteroskedastic_tau(imt, mag) + 0.075,
    "central": lambda imt, mag: get_heteroskedastic_tau(imt, mag),
    "lower": lambda imt, mag: get_heteroskedastic_tau(imt, mag) - 0.075}

HOMOSKEDASTIC_TAU = {
    "upper": lambda imt: get_heteroskedastic_tau(imt, 6.0) + 0.075,
    "central": lambda imt: get_heteroskedastic_tau(imt, 6.0),
    "lower": lambda imt: get_heteroskedastic_tau(imt, 6.0) - 0.075}

uppernames = '''
DEFINED_FOR_INTENSITY_MEASURE_TYPES
DEFINED_FOR_STANDARD_DEVIATION_TYPES
REQUIRES_SITES_PARAMETERS
REQUIRES_RUPTURE_PARAMETERS
REQUIRES_DISTANCES
'''.split()

class TromansEtAl2019(GMPE):
    """
    Implements a meta GMPE to apply the standard deviation model and
    adjustments described in Tromans et al. (2019), for application to
    a nuclear power plant site in the UK:

    Tromans, I. J., Aldama-Bustos, G., Douglas, J., Lessi-Cheimariou, A.,
    Hunt, S., Davi, M., Musson, R. M. W., Garrard, G., Strasser, F. and
    Robertson, C. (2019) "Probabilistic seismic hazard assessment for a
    new-build nuclear power plant site in the UK", Bulletin of Earthquake
    Engineering, 17: 1- 36

    :param gmpe:
        The GMPE for calculation of the medeian ground motion model

    :param string branch:
        The model defines three branches for the different aleatory
        uncertainty models "lower", "central" and "upper"

    :param float scaling_factor:
        Factor to scale the median values of the GMPE to account for, for
        example, stress drop uncertainty

    :param bool homoskedastic sigma:
        Determines whether to use the homoskedastic uncertainty model (True)
        or the heteroskedastic model (False)

    :param vskappa:
        Apply vs-kappa adjustment factors defined using a dictionary organised
        by IMT, or else none.
    """
    experimental = True

    #: Supported tectonic region type is 'active shallow crust'
    DEFINED_FOR_TECTONIC_REGION_TYPE = const.TRT.ACTIVE_SHALLOW_CRUST

    #: Set of :mod:`intensity measure types <openquake.hazardlib.imt>`
    #: this GSIM can calculate. A set should contain classes from module
    #: :mod:`openquake.hazardlib.imt`.
    DEFINED_FOR_INTENSITY_MEASURE_TYPES = set()

    #: Supported intensity measure component is the geometric mean of two
    #: horizontal components
    DEFINED_FOR_INTENSITY_MEASURE_COMPONENT = const.IMC.RotD50

    #: Supported standard deviation types are inter-event, intra-event
    #: and total
    DEFINED_FOR_STANDARD_DEVIATION_TYPES = set([
        const.StdDev.TOTAL,
        const.StdDev.INTER_EVENT,
        const.StdDev.INTRA_EVENT
    ])

    #: Required site parameter is not set
    REQUIRES_SITES_PARAMETERS = set()

    #: Required rupture parameters are magnitude, others will be taken from
    #: the GMPE
    REQUIRES_RUPTURE_PARAMETERS = set(('mag',))

    #: Required distance measure will be set by the GMPE
    REQUIRES_DISTANCES = set(())

    def __init__(self, gmpe_name, branch="central",
                 homoskedastic_sigma=False,  scaling_factor=None,
                 vskappa=None):
        super().__init__(gmpe_name=gmpe_name)
        self.gmpe = registry[gmpe_name]()
        # Update the required_parameters
        for name in uppernames:
            setattr(self, name,
                    frozenset(getattr(self, name) | getattr(self.gmpe, name)))

        # If the scaling factor take the natural log
        if scaling_factor:
            self.scaling_factor = np.log(scaling_factor)
        else:
            self.scaling_factor = None

        # If vs-kappa is passed as a dictionary then transform to CoeffsTable
        if isinstance(vskappa, dict):
            in_vskappa = {}
            for key in vskappa:
                in_vskappa[from_string(key)] = {"vskappa":
                                                np.log(vskappa[key])}
            self.vskappa = CoeffsTable(sa_damping=5, table=in_vskappa)
        else:
            self.vskappa = None
        self.branch = branch
        self.homoskedastic_sigma = homoskedastic_sigma

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the mean and standard deviations applying, where specified,
        scalar adjustment and vs-kappa adjustment to the mean from the
        original GMPE
        """
        # Retrieve the mean values from the GMPE - cannot avoid calling
        # the stddevs part, but no stddevs will be returned.
        mean = self.gmpe.get_mean_and_stddevs(sites, rup, dists, imt, [])[0]
        # Apply scaling factor
        if self.scaling_factor:
            mean += self.scaling_factor
        # Apply vs-kappa correction
        if self.vskappa:
            mean += self.vskappa[imt]["vskappa"]
        # Get stddevs
        stddevs = self.get_stddevs(stddev_types, imt, rup.mag, mean.shape)
        return mean, stddevs

    def get_stddevs(self, stddev_types, imt, mag, nsites):
        """
        Returns the standard deviations as described in Figure 10 and
        section 4 of Tromans et al. (2019).
        """
        if self.homoskedastic_sigma:
            # Homoskedastic sigma branch
            tau = np.zeros(nsites) + HOMOSKEDASTIC_TAU[self.branch](imt)
            phi = np.zeros(nsites) + HOMOSKEDASTIC_PHI[self.branch](imt)
            phi = np.sqrt(phi ** 2. + DELTA_PHI_S2S[imt]["dfs2s"] ** 2.)
        else:
            # Heteroskedastic sigma branch
            tau = np.zeros(nsites) + HETEROSKEDASTIC_TAU[self.branch](imt, mag)
            phi = np.zeros(nsites) + HETEROSKEDASTIC_PHI[self.branch](imt, mag)
        # Add on the delta phi_d2s
        phi = np.sqrt(phi ** 2. + DELTA_PHI_S2S[imt]["dfs2s"] ** 2.)
        stddevs = []
        for stddev_type in stddev_types:
            assert stddev_type in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES
            if stddev_type == const.StdDev.TOTAL:
                stddevs.append(np.sqrt(tau ** 2. + phi ** 2.))
            elif stddev_type == const.StdDev.INTRA_EVENT:
                stddevs.append(phi)
            elif stddev_type == const.StdDev.INTER_EVENT:
                stddevs.append(tau)
        return stddevs
<commit_msg>Removes experimental warning<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Module exports :class:`TromansEtAl2019`
"""
import numpy as np
from openquake.hazardlib.gsim.base import registry, GMPE, CoeffsTable
from openquake.hazardlib import const
from openquake.hazardlib.imt import from_string

#: Coefficient tables as per annex B of Abrahamson et al. (2014)
ASK_TAU_COEFFS = CoeffsTable(sa_damping=5, table="""\
    IMT       s3     s4
    pga     0.47   0.36
    pgv     0.38   0.38
    0.010   0.47   0.36
    0.020   0.47   0.36
    0.030   0.47   0.36
    0.050   0.47   0.36
    0.075   0.47   0.36
    0.100   0.47   0.36
    0.150   0.47   0.36
    0.200   0.47   0.36
    0.250   0.47   0.36
    0.300   0.47   0.36
    0.400   0.47   0.36
    0.500   0.47   0.36
    0.750   0.47   0.36
    1.000   0.47   0.36
    1.500   0.47   0.36
    2.000   0.47   0.36
    3.000   0.47   0.36
    4.000   0.47   0.36
    5.000   0.47   0.36
    6.000   0.47   0.36
    7.500   0.47   0.36
    10.00   0.47   0.36
    """)

#: Single station intra-event term for the constant and magnitude-dependent
#: model, as provided in Table 4 of Rodriguez-Marek et al (2013)
PHI_SS_COEFFS = CoeffsTable(sa_damping=5, table="""\
    IMT   const_phiss   phi1m  phi2m
    pgv          0.46    0.49   0.37
    pga          0.46    0.49   0.35
    0.01         0.46    0.49   0.35
    0.1          0.45    0.45   0.43
    0.2          0.48    0.51   0.37
    0.3          0.48    0.51   0.37
    0.5          0.46    0.49   0.37
    1.0          0.45    0.46   0.40
    3.0          0.41    0.41   0.41
    """)

#: Single ststion dfS2S model
DELTA_PHI_S2S = CoeffsTable(sa_damping=5, table="""\
    IMT          dfs2s
    pga         0.0000
    0.01000     0.0000
    0.99999     0.0000
    1.00000     0.0001
    2.00000     0.0069
    3.00000     0.0131
    10.0000     0.0131
    """)

def get_heteroskedastic_tau(imt, mag):
    """
    Returns the magnitude dependent inter-event variability using the
    model of Abrahamson et al (2014).
    :param dict C:
        Coefficients dictionary
    :param float mag:
        Magnitude
    """

    C = ASK_TAU_COEFFS[imt]
    if mag < 5:
        tau = C['s3']
    elif mag <= 7:
        tau = C['s3'] + (C['s4'] - C['s3']) / 2. * (mag - 5.)
    else:
        tau = C['s4']
    return tau

def get_heteroskedastic_phi(imt, mag):
    """
    Returns the heteroskedastic intra-event term, taken as the maximum
    of the constant single-station phi and the magnitude dependent
    single-station phi provided in Table 4 of Rodriguez-Marek et al (2014)
    """
    C = PHI_SS_COEFFS[imt]
    if mag < 5.0:
        mag_phi = C["phi1m"]
    elif mag > 7.0:
        mag_phi = C["phi2m"]
    else:
        mag_phi = C["phi1m"] + (C["phi2m"] - C["phi1m"]) * ((mag - 5.0) / 2.0)
    if mag_phi > C["const_phiss"]:
        return mag_phi
    else:
        return C["const_phiss"]

HETEROSKEDASTIC_PHI = {
    "upper": lambda imt, mag: 1.16 * get_heteroskedastic_phi(imt, mag),
    "central": lambda imt, mag: get_heteroskedastic_phi(imt, mag),
    "lower": lambda imt, mag: 0.84 * get_heteroskedastic_phi(imt, mag)}

HOMOSKEDASTIC_PHI = {
    "upper": lambda imt: 1.16 * PHI_SS_COEFFS[imt]["const_phiss"],
    "central": lambda imt: PHI_SS_COEFFS[imt]["const_phiss"],
    "lower": lambda imt: 0.84 * PHI_SS_COEFFS[imt]["const_phiss"]}

HETEROSKEDASTIC_TAU = {
    "upper": lambda imt, mag: get_heteroskedastic_tau(imt, mag) + 0.075,
    "central": lambda imt, mag: get_heteroskedastic_tau(imt, mag),
    "lower": lambda imt, mag: get_heteroskedastic_tau(imt, mag) - 0.075}

HOMOSKEDASTIC_TAU = {
    "upper": lambda imt: get_heteroskedastic_tau(imt, 6.0) + 0.075,
    "central": lambda imt: get_heteroskedastic_tau(imt, 6.0),
    "lower": lambda imt: get_heteroskedastic_tau(imt, 6.0) - 0.075}

uppernames = '''
DEFINED_FOR_INTENSITY_MEASURE_TYPES
DEFINED_FOR_STANDARD_DEVIATION_TYPES
REQUIRES_SITES_PARAMETERS
REQUIRES_RUPTURE_PARAMETERS
REQUIRES_DISTANCES
'''.split()

class TromansEtAl2019(GMPE):
    """
    Implements a meta GMPE to apply the standard deviation model and
    adjustments described in Tromans et al. (2019), for application to
    a nuclear power plant site in the UK:

    Tromans, I. J., Aldama-Bustos, G., Douglas, J., Lessi-Cheimariou, A.,
    Hunt, S., Davi, M., Musson, R. M. W., Garrard, G., Strasser, F. and
    Robertson, C. (2019) "Probabilistic seismic hazard assessment for a
    new-build nuclear power plant site in the UK", Bulletin of Earthquake
    Engineering, 17: 1- 36

    :param gmpe:
        The GMPE for calculation of the medeian ground motion model

    :param string branch:
        The model defines three branches for the different aleatory
        uncertainty models "lower", "central" and "upper"

    :param float scaling_factor:
        Factor to scale the median values of the GMPE to account for, for
        example, stress drop uncertainty

    :param bool homoskedastic sigma:
        Determines whether to use the homoskedastic uncertainty model (True)
        or the heteroskedastic model (False)

    :param vskappa:
        Apply vs-kappa adjustment factors defined using a dictionary organised
        by IMT, or else none.
    """
    #: Supported tectonic region type is 'active shallow crust'
    DEFINED_FOR_TECTONIC_REGION_TYPE = const.TRT.ACTIVE_SHALLOW_CRUST

    #: Set of :mod:`intensity measure types <openquake.hazardlib.imt>`
    #: this GSIM can calculate. A set should contain classes from module
    #: :mod:`openquake.hazardlib.imt`.
    DEFINED_FOR_INTENSITY_MEASURE_TYPES = set()

    #: Supported intensity measure component is the geometric mean of two
    #: horizontal components
    DEFINED_FOR_INTENSITY_MEASURE_COMPONENT = const.IMC.RotD50

    #: Supported standard deviation types are inter-event, intra-event
    #: and total
    DEFINED_FOR_STANDARD_DEVIATION_TYPES = set([
        const.StdDev.TOTAL,
        const.StdDev.INTER_EVENT,
        const.StdDev.INTRA_EVENT
    ])

    #: Required site parameter is not set
    REQUIRES_SITES_PARAMETERS = set()

    #: Required rupture parameters are magnitude, others will be taken from
    #: the GMPE
    REQUIRES_RUPTURE_PARAMETERS = set(('mag',))

    #: Required distance measure will be set by the GMPE
    REQUIRES_DISTANCES = set(())

    def __init__(self, gmpe_name, branch="central",
                 homoskedastic_sigma=False,  scaling_factor=None,
                 vskappa=None):
        super().__init__(gmpe_name=gmpe_name)
        self.gmpe = registry[gmpe_name]()
        # Update the required_parameters
        for name in uppernames:
            setattr(self, name,
                    frozenset(getattr(self, name) | getattr(self.gmpe, name)))

        # If the scaling factor take the natural log
        if scaling_factor:
            self.scaling_factor = np.log(scaling_factor)
        else:
            self.scaling_factor = None

        # If vs-kappa is passed as a dictionary then transform to CoeffsTable
        if isinstance(vskappa, dict):
            in_vskappa = {}
            for key in vskappa:
                in_vskappa[from_string(key)] = {"vskappa":
                                                np.log(vskappa[key])}
            self.vskappa = CoeffsTable(sa_damping=5, table=in_vskappa)
        else:
            self.vskappa = None
        self.branch = branch
        self.homoskedastic_sigma = homoskedastic_sigma

    def get_mean_and_stddevs(self, sites, rup, dists, imt, stddev_types):
        """
        Returns the mean and standard deviations applying, where specified,
        scalar adjustment and vs-kappa adjustment to the mean from the
        original GMPE
        """
        # Retrieve the mean values from the GMPE - cannot avoid calling
        # the stddevs part, but no stddevs will be returned.
        mean = self.gmpe.get_mean_and_stddevs(sites, rup, dists, imt, [])[0]
        # Apply scaling factor
        if self.scaling_factor:
            mean += self.scaling_factor
        # Apply vs-kappa correction
        if self.vskappa:
            mean += self.vskappa[imt]["vskappa"]
        # Get stddevs
        stddevs = self.get_stddevs(stddev_types, imt, rup.mag, mean.shape)
        return mean, stddevs

    def get_stddevs(self, stddev_types, imt, mag, nsites):
        """
        Returns the standard deviations as described in Figure 10 and
        section 4 of Tromans et al. (2019).
        """
        if self.homoskedastic_sigma:
            # Homoskedastic sigma branch
            tau = np.zeros(nsites) + HOMOSKEDASTIC_TAU[self.branch](imt)
            phi = np.zeros(nsites) + HOMOSKEDASTIC_PHI[self.branch](imt)
            phi = np.sqrt(phi ** 2. + DELTA_PHI_S2S[imt]["dfs2s"] ** 2.)
        else:
            # Heteroskedastic sigma branch
            tau = np.zeros(nsites) + HETEROSKEDASTIC_TAU[self.branch](imt, mag)
            phi = np.zeros(nsites) + HETEROSKEDASTIC_PHI[self.branch](imt, mag)
        # Add on the delta phi_d2s
        phi = np.sqrt(phi ** 2. + DELTA_PHI_S2S[imt]["dfs2s"] ** 2.)
        stddevs = []
        for stddev_type in stddev_types:
            assert stddev_type in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES
            if stddev_type == const.StdDev.TOTAL:
                stddevs.append(np.sqrt(tau ** 2. + phi ** 2.))
            elif stddev_type == const.StdDev.INTRA_EVENT:
                stddevs.append(phi)
            elif stddev_type == const.StdDev.INTER_EVENT:
                stddevs.append(tau)
        return stddevs
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
generate_sample.py

Script to find a 2.5 second clip of ambient noise and silence that clip.

*in progress*

Author:
        – Jon Clucas, 2016 (jon.clucas@childmind.org)

© 2016, Child Mind Institute, Apache v2.0 License
Created on Wed Dec 21 16:34:37 2016

@author: jon.clucas
"""
# workaround for local installation : adjust these paths to local installation
#     directories
import sys
sys.path.append("/Volumes/Jon.Clucas/openSMILE/SM_openSMILE")
sys.path.append("/Library/Frameworks/Python.framework/Versions/2.7/lib/"
                "python2.7/site-packages")

import iterate_ursis as iu, noise_replacement as nr, os, pydub, random

def create_sample(in_file):
    """
    Function to iterate through [top_directory]/URSI/[subdirectory]/files
    structure and perform a function on each file.

    Parameters
    ----------
    in_file : string
        absolute path of file

    Returns
    -------
    None

    TODO: break this function apart
    """
    # get file
    print (''.join(["Getting ", in_file]))
    out_file = os.path.join(os.path.dirname(os.path.dirname(in_file)),
                 "sample_silenced", os.path.basename(in_file))
    out_clone = os.path.join(os.path.dirname(os.path.dirname(in_file)),
                 "clone_fill", os.path.basename(in_file))
    # if output directory does not exist, create
    if not os.path.exists(os.path.dirname(out_file)):
        print (''.join(["Making directory: ", os.path.dirname(out_file)]))
        os.makedirs (os.path.dirname(out_file))
    if not os.path.exists(os.path.dirname(out_clone)):
        print (''.join(["Making directory: ", os.path.dirname(out_clone)]))
        os.makedirs (os.path.dirname(out_clone))
    # get ambient clips
    options = nr.get_ambient_clips(in_file)
    # how many?
    print(''.join([str(len(options)), " ambient clips"]))
    remaining_options = []
    for pair in options:
        if ((pair[1] - pair[0]) > 110250):
            remaining_options.append(pair)
    num_options = len(remaining_options)
    print(''.join([str(num_options), " ≥ 2.5 seconds"]))
    if num_options == 0:
        remaining_options = options
        num_options = len(remaining_options)
    chosen_one = remaining_options[random.randrange(0, num_options)]
    print(''.join(['chosen clip : ', str(chosen_one[0]), ":",
          str(chosen_one[1])]))
    original = pydub.AudioSegment.from_wav(in_file)
    clone = nr.grow_mask(original.get_sample_slice(chosen_one[0],
            chosen_one[1]), len(original))
    sample_clip = original.get_sample_slice(0, chosen_one[0])
    sample_clip.append(pydub.AudioSegment.silent(duration=(chosen_one[1] -
                       chosen_one[0])))
    sample_clip.append(original.get_sample_slice(chosen_one[1], None))
    print(''.join(["Exporting ", os.path.basename(os.path.dirname(out_file)),
          "/", os.path.basename(out_file)]))
    sample_clip.export(out_file, format="wav")
    clone_clip = nr.replace_silence(sample_clip, clone, 44.1)
    clone_clip.export(out_clone, format="wav")

def main():
    t_dir = "/Volumes/data/Research/CDB/openSMILE/audio_files"
    starting_files = iu.i_ursi(t_dir, "no_beeps")
    for file in starting_files:
        create_sample(file)

# ============================================================================
if __name__ == '__main__':
    main()<commit_msg>separate functions<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
generate_sample.py

Script to find a 2.5 second clip of ambient noise and silence that clip.

*in progress*

Author:
        – Jon Clucas, 2016 (jon.clucas@childmind.org)

© 2016, Child Mind Institute, Apache v2.0 License
Created on Wed Dec 21 16:34:37 2016

@author: jon.clucas
"""
# workaround for local installation : adjust these paths to local installation
#     directories
import sys
sys.path.append("/Volumes/Jon.Clucas/openSMILE/SM_openSMILE")
sys.path.append("/Library/Frameworks/Python.framework/Versions/2.7/lib/"
                "python2.7/site-packages")

import iterate_ursis as iu, noise_replacement as nr, os, pydub, random

def check_clip_len(options, duration):
    """
    TODO: document
    """
    remaining_options = []
    for pair in options:
        if ((pair[1] - pair[0]) > duration):
            remaining_options.append(pair)
            print(''.join(["Here's an option: ", str(pair[0]), ":", str(pair[
                  1])]))
    return remaining_options

def create_sample(in_file):
    """
    Function to iterate through [top_directory]/URSI/[subdirectory]/files
    structure and perform a function on each file.

    Parameters
    ----------
    in_file : string
        absolute path of file

    Returns
    -------
    None
    """
    # get file
    print (''.join(["Getting ", in_file]))
    # get ambient clips
    options = nr.get_ambient_clips(in_file)
    # how many?
    print(''.join([str(len(options)), " ambient clips"]))
    remaining_options = []
    # start by looking for clips 2.5 seconds or longer
    desired_duration = 110250
    while len(remaining_options) == 0:
        remaining_options = check_clip_len(options, desired_duration)
        # reduce desired duration by .25 seconds
        desired_duration = desired_duration - 11025
    num_options = len(remaining_options)
    print(''.join([str(num_options), " ≥ ", str(round(desired_duration /
          44100, 2)), " seconds"]))
    chosen_one = remaining_options[random.randrange(0, num_options)]
    print(''.join(['chosen clip : ', str(chosen_one[0]), ":",
          str(chosen_one[1]), " (", str(round((chosen_one[1] - chosen_one[0]) /
               44100, 2)), " seconds"]))
    # import original sound
    original = pydub.AudioSegment.from_wav(in_file)
    # create silenced sample
    silenced_sample = create_sample_silenced(in_file, original)
    # create replace silence with clone mask
    create_sample_cloned(in_file, silenced_sample)

def create_sample_cloned(path, original, chosen_one):
    """
    TODO: document
    """
    out_file = out_file_path(path, "clone_fill")
    clone = nr.grow_mask(original.get_sample_slice(chosen_one[0],
            chosen_one[1]), len(original))
    clone_clip = nr.replace_silence(original, clone, 44.1)
    return(export(clone_clip, out_file))

def create_sample_silenced(path, original, chosen_one):
    """
    TODO: document
    """
    out_file = out_file_path(path, "sample_silenced")
    print(''.join(["Building silenced sample 0:", str(chosen_one[0])]))
    sample_clip = original.get_sample_slice(0, chosen_one[0])
    print(''.join(["adding silence ", str(chosen_one[0]), ":", str(chosen_one[
          1])]))
    sample_clip.append(pydub.AudioSegment.silent(duration=(chosen_one[1] -
                       chosen_one[0])))
    print(''.join(["building silenced sample ", str(chosen_one[1]), ":", str(
          len(original))]))
    sample_clip.append(original.get_sample_slice(chosen_one[1], None))
    return(export(sample_clip, out_file))

def export(audio_segment, out_path):
    """
    TODO: document
    """
    print(''.join(["Exporting ", os.path.basename(os.path.dirname(out_path)),
          "/", os.path.basename(out_path)]))
    audio_segment.export(out_path, format="wav")
    return(audio_segment)

def out_file_path(path, method):
    """
    TODO: document
    """
    out_file = os.path.join(os.path.dirname(os.path.dirname(path)), method,
               os.path.basename(path))
    # if output directory does not exist, create
    if not os.path.exists(os.path.dirname(out_file)):
        print (''.join(["Making directory: ", os.path.dirname(out_file)]))
        os.makedirs (os.path.dirname(out_file))
    return out_file

def main():
    t_dir = "/Volumes/data/Research/CDB/openSMILE/audio_files"
    starting_files = iu.i_ursi(t_dir, "no_beeps")
    for file in starting_files:
        create_sample(file)

# ============================================================================
if __name__ == '__main__':
    main()<|endoftext|>