<commit_before>#!/usr/bin/env python3
# coding: utf-8

import os
import sys
import subprocess
import shutil
import time
import json
import argparse
import traceback
import requests
from random import randint
from subprocess import PIPE
from bitcoin import rpc
from bitcoin import SelectParams
from shutil import copyfile

TEST_SWARM_PORT = randint(1024, 65535)
TEST_GATEWAY_PORT = randint(1024, 65535)

BOOTSTRAP_NODES = [
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 0) + "/ipfs/QmVp4tK486CvnamB6K4uhY4vB5sMzEDpCNzeyh9VwBFXhS",
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 1) + "/ipfs/QmWPBKm3sLEPMy8EqrR5nHD2KUzh3TEgfquRjuxHM4h3Pv",
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 2) + "/ipfs/QmPL1X7ZQr2ooQHfuWJwuJndeyaN2DBoYHFg954TUsytvF"
]

BOOTSTAP_MNEMONICS = [
    "today summer matter always angry crumble rib lucky park shoulder police puppy",
    "husband letter control display skin tennis this expand garbage boil pig exchange",
    "resist museum dizzy there pulp suspect dust useless drama grab visa trumpet"
]

class TestFailure(Exception):
    pass

class OpenBazaarTestFramework(object):

    def __init__(self):
        self.nodes = []
        self.bitcoin_api = None

    def setup_nodes(self):
        for i in range(self.num_nodes):
            self.configure_node(i)
            self.start_node(self.nodes[i])

    def setup_network(self):
        if self.bitcoind is not None:
            self.start_bitcoind()
        self.setup_nodes()

    def run_test(self):
        raise NotImplementedError

    def send_bitcoin_cmd(self, *args):
        try:
            return self.bitcoin_api.call(*args)
        except BrokenPipeError:
            self.bitcoin_api = rpc.Proxy(btc_conf_file=self.btc_config)
            return self.send_bitcoin_cmd(*args)

    def configure_node(self, n):
        dir_path = os.path.join(self.temp_dir, "openbazaar-go", str(n))
        args = [self.binary, "init", "-d", dir_path, "--testnet"]
        if n < 3:
            args.extend(["-m", BOOTSTAP_MNEMONICS[n]])
        process = subprocess.Popen(args, stdout=PIPE)
        self.wait_for_init_success(process)
        with open(os.path.join(dir_path, "config")) as cfg:
            config = json.load(cfg)
        config["Addresses"]["Gateway"] = "/ip4/127.0.0.1/tcp/" + str(TEST_GATEWAY_PORT + n)
        config["Addresses"]["Swarm"] = ["/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + n)]
        config["Bootstrap"] = BOOTSTRAP_NODES
        config["Wallet"]["TrustedPeer"] = "127.0.0.1:18444"
        config["Wallet"]["FeeAPI"] = ""
        config["Crosspost-gateways"] = []
        with open(os.path.join(dir_path, "config"), 'w') as outfile:
            outfile.write(json.dumps(config, indent=4))
        node = {
            "data_dir": dir_path,
            "gateway_url": "http://localhost:" + str(TEST_GATEWAY_PORT + n) + "/",
            "swarm_port": str(TEST_SWARM_PORT + n)
        }
        self.nodes.append(node)

    @staticmethod
    def wait_for_init_success(process):
        while True:
            if process.poll() is not None:
                raise Exception("OpenBazaar node initialization failed")
            output = process.stdout
            for o in output:
                if "OpenBazaar repo initialized" in str(o):
                    return

    def start_node(self, node):
        args = [self.binary, "start", "-d", node["data_dir"], *self.options]
        process = subprocess.Popen(args, stdout=PIPE)
        peerId = self.wait_for_start_success(process, node)
        node["peerId"] = peerId

    @staticmethod
    def wait_for_start_success(process, node):
        peerId = ""
        while True:
            if process.poll() is not None:
                raise Exception("OpenBazaar node failed to start")
            output = process.stdout
            for o in output:
                if "Peer ID:" in str(o):
                    peerId = str(o)[str(o).index("Peer ID:") + 10:len(str(o)) - 3]
                if "Gateway/API server listening" in str(o):
                    return peerId

    def start_bitcoind(self):
        SelectParams('regtest')
        dir_path = os.path.join(self.temp_dir, "openbazaar-go", "bitcoin")
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        btc_conf_file = os.path.join(dir_path, "bitcoin.conf")
        copyfile(os.path.join(os.getcwd(), "testdata", "bitcoin.conf"), btc_conf_file)
        self.btc_config = btc_conf_file
        args = [self.bitcoind, "-regtest", "-datadir=" + dir_path, "-debug=net"]
        process = subprocess.Popen(args, stdout=PIPE)
        self.wait_for_bitcoind_start(process, btc_conf_file)
        self.init_blockchain()

    def init_blockchain(self):
        self.send_bitcoin_cmd("generate", 1)
        self.bitcoin_address = self.send_bitcoin_cmd("getnewaddress")
        self.send_bitcoin_cmd("generatetoaddress", 1, self.bitcoin_address)
        self.send_bitcoin_cmd("generate", 125)

    def wait_for_bitcoind_start(self, process, btc_conf_file):
        while True:
            if process.poll() is not None:
                raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
            try:
                self.bitcoin_api = rpc.Proxy(btc_conf_file=btc_conf_file)
                blocks = self.bitcoin_api.getblockcount()
                break # break out of loop on success
            except Exception:
                time.sleep(0.25)
                continue

    def print_logs(self, node, log):
        f = open(os.path.join(node["data_dir"], "logs", log), 'r')
        file_contents = f.read()
        print()
        print("~~~~~~~~~~~~~~~~~~~~~~ " + log + " ~~~~~~~~~~~~~~~~~~~~~~")
        print (file_contents)
        print()
        f.close()

    def teardown(self, delete_data):
        for n in self.nodes:
            requests.post(n["gateway_url"] + "ob/shutdown")
        time.sleep(2)
        if self.bitcoin_api is not None:
            try:
                self.send_bitcoin_cmd("stop")
            except BrokenPipeError:
                pass
        time.sleep(10)
        if delete_data:
            shutil.rmtree(os.path.join(self.temp_dir, "openbazaar-go"))

    def main(self, options=["--disablewallet", "--testnet", "--disableexchangerates"]):
        parser = argparse.ArgumentParser(
                    description="OpenBazaar Test Framework",
                    usage="python3 test_framework.py [options]"
        )
        parser.add_argument('-b', '--binary', required=True, help="the openbazaar-go binary")
        parser.add_argument('-d', '--bitcoind', help="the bitcoind binary")
        parser.add_argument('-t', '--tempdir', action='store_true', help="temp directory to store the data folders", default="/tmp/")
        args = parser.parse_args(sys.argv[1:])
        self.binary = args.binary
        self.temp_dir = args.tempdir
        self.bitcoind = args.bitcoind
        self.options = options

        failure = False
        try:
            self.setup_network()
            self.run_test()
        except TestFailure as e:
            print(repr(e))
            failure = True
        except Exception as e:
            print("Unexpected exception caught during testing: " + repr(e))
            traceback.print_tb(sys.exc_info()[2])
            failure = True

        if not failure:
            self.teardown(True)
        else:
            self.teardown(False)

        if failure:
            sys.exit(1)

<commit_msg>Update qa package bootstrap nodes<commit_after>#!/usr/bin/env python3
# coding: utf-8

import os
import sys
import subprocess
import shutil
import time
import json
import argparse
import traceback
import requests
from random import randint
from subprocess import PIPE
from bitcoin import rpc
from bitcoin import SelectParams
from shutil import copyfile

TEST_SWARM_PORT = randint(1024, 65535)
TEST_GATEWAY_PORT = randint(1024, 65535)

BOOTSTRAP_NODES = [
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 0) + "/ipfs/QmZkcxKxq9pZzqvXHhhsAQuNM9Fj5gx1ecpdzCb8w2NSGE",
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 1) + "/ipfs/QmS5svqgGwFxwY9W5nXBUh1GJ7x8tqpkYfD4kB3MG7mPRv",
    "/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + 2) + "/ipfs/QmchsHCcVAjoCiAQCd8mWcVtS5cwpwRFvyCnAcdoQcfyM7"
]

BOOTSTAP_MNEMONICS = [
    "today summer matter always angry crumble rib lucky park shoulder police puppy",
    "husband letter control display skin tennis this expand garbage boil pig exchange",
    "resist museum dizzy there pulp suspect dust useless drama grab visa trumpet"
]

class TestFailure(Exception):
    pass

class OpenBazaarTestFramework(object):

    def __init__(self):
        self.nodes = []
        self.bitcoin_api = None

    def setup_nodes(self):
        for i in range(self.num_nodes):
            self.configure_node(i)
            self.start_node(self.nodes[i])

    def setup_network(self):
        if self.bitcoind is not None:
            self.start_bitcoind()
        self.setup_nodes()

    def run_test(self):
        raise NotImplementedError

    def send_bitcoin_cmd(self, *args):
        try:
            return self.bitcoin_api.call(*args)
        except BrokenPipeError:
            self.bitcoin_api = rpc.Proxy(btc_conf_file=self.btc_config)
            return self.send_bitcoin_cmd(*args)

    def configure_node(self, n):
        dir_path = os.path.join(self.temp_dir, "openbazaar-go", str(n))
        args = [self.binary, "init", "-d", dir_path, "--testnet"]
        if n < 3:
            args.extend(["-m", BOOTSTAP_MNEMONICS[n]])
        process = subprocess.Popen(args, stdout=PIPE)
        self.wait_for_init_success(process)
        with open(os.path.join(dir_path, "config")) as cfg:
            config = json.load(cfg)
        config["Addresses"]["Gateway"] = "/ip4/127.0.0.1/tcp/" + str(TEST_GATEWAY_PORT + n)
        config["Addresses"]["Swarm"] = ["/ip4/127.0.0.1/tcp/" + str(TEST_SWARM_PORT + n)]
        config["Bootstrap"] = BOOTSTRAP_NODES
        config["Wallet"]["TrustedPeer"] = "127.0.0.1:18444"
        config["Wallet"]["FeeAPI"] = ""
        config["Crosspost-gateways"] = []
        with open(os.path.join(dir_path, "config"), 'w') as outfile:
            outfile.write(json.dumps(config, indent=4))
        node = {
            "data_dir": dir_path,
            "gateway_url": "http://localhost:" + str(TEST_GATEWAY_PORT + n) + "/",
            "swarm_port": str(TEST_SWARM_PORT + n)
        }
        self.nodes.append(node)

    @staticmethod
    def wait_for_init_success(process):
        while True:
            if process.poll() is not None:
                raise Exception("OpenBazaar node initialization failed")
            output = process.stdout
            for o in output:
                if "OpenBazaar repo initialized" in str(o):
                    return

    def start_node(self, node):
        args = [self.binary, "start", "-d", node["data_dir"], *self.options]
        process = subprocess.Popen(args, stdout=PIPE)
        peerId = self.wait_for_start_success(process, node)
        node["peerId"] = peerId

    @staticmethod
    def wait_for_start_success(process, node):
        peerId = ""
        while True:
            if process.poll() is not None:
                raise Exception("OpenBazaar node failed to start")
            output = process.stdout
            for o in output:
                if "Peer ID:" in str(o):
                    peerId = str(o)[str(o).index("Peer ID:") + 10:len(str(o)) - 3]
                if "Gateway/API server listening" in str(o):
                    return peerId

    def start_bitcoind(self):
        SelectParams('regtest')
        dir_path = os.path.join(self.temp_dir, "openbazaar-go", "bitcoin")
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        btc_conf_file = os.path.join(dir_path, "bitcoin.conf")
        copyfile(os.path.join(os.getcwd(), "testdata", "bitcoin.conf"), btc_conf_file)
        self.btc_config = btc_conf_file
        args = [self.bitcoind, "-regtest", "-datadir=" + dir_path, "-debug=net"]
        process = subprocess.Popen(args, stdout=PIPE)
        self.wait_for_bitcoind_start(process, btc_conf_file)
        self.init_blockchain()

    def init_blockchain(self):
        self.send_bitcoin_cmd("generate", 1)
        self.bitcoin_address = self.send_bitcoin_cmd("getnewaddress")
        self.send_bitcoin_cmd("generatetoaddress", 1, self.bitcoin_address)
        self.send_bitcoin_cmd("generate", 125)

    def wait_for_bitcoind_start(self, process, btc_conf_file):
        while True:
            if process.poll() is not None:
                raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
            try:
                self.bitcoin_api = rpc.Proxy(btc_conf_file=btc_conf_file)
                blocks = self.bitcoin_api.getblockcount()
                break # break out of loop on success
            except Exception:
                time.sleep(0.25)
                continue

    def print_logs(self, node, log):
        f = open(os.path.join(node["data_dir"], "logs", log), 'r')
        file_contents = f.read()
        print()
        print("~~~~~~~~~~~~~~~~~~~~~~ " + log + " ~~~~~~~~~~~~~~~~~~~~~~")
        print (file_contents)
        print()
        f.close()

    def teardown(self, delete_data):
        for n in self.nodes:
            requests.post(n["gateway_url"] + "ob/shutdown")
        time.sleep(2)
        if self.bitcoin_api is not None:
            try:
                self.send_bitcoin_cmd("stop")
            except BrokenPipeError:
                pass
        time.sleep(10)
        if delete_data:
            shutil.rmtree(os.path.join(self.temp_dir, "openbazaar-go"))

    def main(self, options=["--disablewallet", "--testnet", "--disableexchangerates"]):
        parser = argparse.ArgumentParser(
                    description="OpenBazaar Test Framework",
                    usage="python3 test_framework.py [options]"
        )
        parser.add_argument('-b', '--binary', required=True, help="the openbazaar-go binary")
        parser.add_argument('-d', '--bitcoind', help="the bitcoind binary")
        parser.add_argument('-t', '--tempdir', action='store_true', help="temp directory to store the data folders", default="/tmp/")
        args = parser.parse_args(sys.argv[1:])
        self.binary = args.binary
        self.temp_dir = args.tempdir
        self.bitcoind = args.bitcoind
        self.options = options

        failure = False
        try:
            self.setup_network()
            self.run_test()
        except TestFailure as e:
            print(repr(e))
            failure = True
        except Exception as e:
            print("Unexpected exception caught during testing: " + repr(e))
            traceback.print_tb(sys.exc_info()[2])
            failure = True

        if not failure:
            self.teardown(True)
        else:
            self.teardown(False)

        if failure:
            sys.exit(1)

<|endoftext|>


<commit_before>import sys
import re

from coalib.bearlib.abstractions.Linter import linter
from coalib.settings.Setting import typed_list
from coalib.results.Result import Result
from coalib.bearlib.aspects import map_setting_to_aspect
from coalib.bearlib.aspects.Formatting import LineLength

from dependency_management.requirements.PipRequirement import PipRequirement

OUTPUT_REGEX = (r'(?P<line>\d+) (?P<column>\d+) '
                r'(?P<message>(?P<origin>\S+).*)')

@linter(executable='pycodestyle',
        )
class PycodestyleBear:
    """
    A wrapper for the tool ``pycodestyle`` formerly known as ``pep8``.
    """
    LANGUAGES = {'Python', 'Python 2', 'Python 3'}
    REQUIREMENTS = {PipRequirement('pycodestyle', '2.2')}
    AUTHORS = {'The coala developers'}
    AUTHORS_EMAILS = {'coala-devel@googlegroups.com'}
    LICENSE = 'AGPL-3.0'
    CAN_DETECT = {'Formatting'}

    @map_setting_to_aspect(
        max_line_length=LineLength.max_line_length,
    )
    def create_arguments(
            self,
            filename, file, config_file,
            pycodestyle_ignore: typed_list(str) = (
                'E121', 'E123', 'E126', 'E133', 'E226',
                'E241', 'E242', 'E704', 'W503', 'W504',
            ),
            pycodestyle_select: typed_list(str) = (),
            max_line_length: int = 79,
            ):
        """
        :param pycodestyle_ignore:
            Comma separated list of errors to ignore.
            See ``pydocstyle`` documentation for a complete list of errors.
        :param pycodestyle_select:
            Comma separated list of errors to detect. If given only
            these errors are going to be detected.
            See ``pydocstyle`` documentation for a complete list of errors.
        :param max_line_length:
            Limit lines to this length. Allows infinite line length when
            set to 0.
        """
        arguments = [r'--format=%(row)d %(col)d %(code)s %(text)s']

        if not max_line_length:
            max_line_length = sys.maxsize

        if pycodestyle_ignore:
            ignore = ','.join(part.strip() for part in pycodestyle_ignore)
            arguments.append('--ignore=' + ignore)

        if pycodestyle_select:
            select = ','.join(part.strip() for part in pycodestyle_select)
            arguments.append('--select=' + select)

        arguments.append('--max-line-length=' + str(max_line_length))

        arguments.append(filename)

        return arguments

    def process_output(self, output, filename, file):
        result = re.match(OUTPUT_REGEX, output)
        if not result:
            return
        line, column, message, rule = result.groups()
        if rule == 'E501':
            aspect = LineLength('py')
        else:
            aspect = None
        yield Result.from_values(
            origin='{} ({})'.format(self.name, rule),
            message=message,
            file=filename,
            line=int(line),
            column=int(column),
            aspect=aspect,
            )
<commit_msg>PycodestyleBear: Allow non-coverage for no output<commit_after>import sys
import re

from coalib.bearlib.abstractions.Linter import linter
from coalib.settings.Setting import typed_list
from coalib.results.Result import Result
from coalib.bearlib.aspects import map_setting_to_aspect
from coalib.bearlib.aspects.Formatting import LineLength

from dependency_management.requirements.PipRequirement import PipRequirement

OUTPUT_REGEX = (r'(?P<line>\d+) (?P<column>\d+) '
                r'(?P<message>(?P<origin>\S+).*)')

@linter(executable='pycodestyle',
        )
class PycodestyleBear:
    """
    A wrapper for the tool ``pycodestyle`` formerly known as ``pep8``.
    """
    LANGUAGES = {'Python', 'Python 2', 'Python 3'}
    REQUIREMENTS = {PipRequirement('pycodestyle', '2.2')}
    AUTHORS = {'The coala developers'}
    AUTHORS_EMAILS = {'coala-devel@googlegroups.com'}
    LICENSE = 'AGPL-3.0'
    CAN_DETECT = {'Formatting'}

    @map_setting_to_aspect(
        max_line_length=LineLength.max_line_length,
    )
    def create_arguments(
            self,
            filename, file, config_file,
            pycodestyle_ignore: typed_list(str) = (
                'E121', 'E123', 'E126', 'E133', 'E226',
                'E241', 'E242', 'E704', 'W503', 'W504',
            ),
            pycodestyle_select: typed_list(str) = (),
            max_line_length: int = 79,
            ):
        """
        :param pycodestyle_ignore:
            Comma separated list of errors to ignore.
            See ``pydocstyle`` documentation for a complete list of errors.
        :param pycodestyle_select:
            Comma separated list of errors to detect. If given only
            these errors are going to be detected.
            See ``pydocstyle`` documentation for a complete list of errors.
        :param max_line_length:
            Limit lines to this length. Allows infinite line length when
            set to 0.
        """
        arguments = [r'--format=%(row)d %(col)d %(code)s %(text)s']

        if not max_line_length:
            max_line_length = sys.maxsize

        if pycodestyle_ignore:
            ignore = ','.join(part.strip() for part in pycodestyle_ignore)
            arguments.append('--ignore=' + ignore)

        if pycodestyle_select:
            select = ','.join(part.strip() for part in pycodestyle_select)
            arguments.append('--select=' + select)

        arguments.append('--max-line-length=' + str(max_line_length))

        arguments.append(filename)

        return arguments

    def process_output(self, output, filename, file):
        if not output:  # backwards compatible no results
            return
        result = re.match(OUTPUT_REGEX, output)
        if not result:  # backwards compatible no results
            self.warn('{}: Unexpected output {}'.format(filename, output))
            return
        line, column, message, rule = result.groups()
        if rule == 'E501':
            aspect = LineLength('py')
        else:
            aspect = None
        yield Result.from_values(
            origin='{} ({})'.format(self.name, rule),
            message=message,
            file=filename,
            line=int(line),
            column=int(column),
            aspect=aspect,
            )
<|endoftext|>


<commit_before>import json
import os.path
import uuid
from nose.tools import nottest

import re
from casexml.apps.case.mock import CaseIndex, CaseStructure
from casexml.apps.case.tests.util import assert_user_doesnt_have_cases, \
    assert_user_has_cases
from casexml.apps.phone.models import get_properly_wrapped_sync_log
from casexml.apps.phone.tests.test_sync_mode import SyncBaseTest
from corehq.form_processor.tests import run_with_all_backends
from corehq.util.test_utils import softer_assert

@nottest
def get_test_file_json(filename):
    base = os.path.dirname(__file__)
    file_path = 'data'
    path = os.path.join(base, file_path, '%s.json' % filename)
    with open(path) as f:
        file_contents = f.read()

    return json.loads(file_contents)

@nottest
def test_generator(test_name, skip=False):
    @softer_assert
    def test(self):
        if skip:
            self.skipTest(skip)
        self.build_case_structures(test_name)
        desired_cases = self._get_test(test_name).get('outcome', [])
        undesired_cases = [case for case in self.ALL_CASES if case not in desired_cases]
        sync_log = get_properly_wrapped_sync_log(self.sync_log._id)
        self.assertEqual(sync_log.case_ids_on_phone, set(desired_cases))
        assert_user_has_cases(self, self.user, desired_cases)
        assert_user_doesnt_have_cases(self, self.user, undesired_cases)
    return run_with_all_backends(test)

class TestSequenceMeta(type):

    def __new__(mcs, name, bases, dict):
        tests_to_run = get_test_file_json('case_relationship_tests')
        run_single_tests = filter(lambda t: t.get('only', False), tests_to_run)
        if run_single_tests:
            tests_to_run = run_single_tests

        for test in [(test['name'], test.get('skip', False)) for test in tests_to_run]:
            # Create a new testcase that the test runner is able to find
            dict["test_%s" % re.sub("\s", "_", test[0])] = test_generator(test[0], test[1])

        return type.__new__(mcs, name, bases, dict)

class IndexTreeTest(SyncBaseTest):
    """Fetch all testcases from data/case_relationship_tests.json and run them

    Each testcase is structured as follows:
    {
        "name": The name of the test,
        "owned": Cases whose owner id is set to the user. Cases not in this list are owned by someone else,
        "subcases": [ A list of ordered pairs e.g. ["a","b"] means "a creates a child index pointing to b"],
        "extensions": [ A list of ordered pairs e.g. ["a","b"] means "a creates an extension index pointing to b"],
        "closed": A list of the closed cases,
        "outcome": When syncing all the cases, which cases should be sent to the phone,
        "only": For debugging purposes, run a single test by adding 'only': true to those tests,
    }
    """
    __metaclass__ = TestSequenceMeta

    ALL_CASES = ['a', 'b', 'c', 'd', 'e']

    @property
    def all_tests(self):
        """All the test cases in a dict"""
        all_tests = {}
        tests = get_test_file_json('case_relationship_tests')
        for test in tests:
            all_tests[test['name']] = test
        return all_tests

    def _get_test(self, test_name):
        return self.all_tests[test_name]

    def build_case_structures(self, test_name):
        test = self._get_test(test_name)
        case_structures = []
        indices = {case: [] for case in self.ALL_CASES}

        for i, subcase in enumerate(test.get('subcases', [])):
            indices[subcase[0]].append(CaseIndex(
                related_structure=CaseStructure(
                    attrs={
                        'create': False,
                        'owner_id': self.user.user_id if subcase[1] in test.get('owned', []) else uuid.uuid4().hex,
                        'close': subcase[1] in test.get('closed', []),
                    },
                    case_id=subcase[1],
                ),
                relationship='child',
                identifier='child_{}'.format(i),
            ))

        for i, extension in enumerate(test.get('extensions', [])):
            indices[extension[0]].append(CaseIndex(
                related_structure=CaseStructure(
                    attrs={
                        'create': False,
                        'owner_id': (self.user.user_id if extension[1]
                                     in test.get('owned', []) else uuid.uuid4().hex),
                        'close': extension[1] in test.get('closed', []),
                    },
                    case_id=extension[1],
                ),
                relationship='extension',
                identifier='extension_{}'.format(i),
            ))

        for case in self.ALL_CASES:
            case_structures.append(CaseStructure(
                attrs={
                    'create': True,
                    'owner_id': self.user.user_id if case in test.get('owned', []) else uuid.uuid4().hex,
                    'close': case in test.get('closed', []),
                },
                case_id=case,
                indices=indices[case],
            ))

        self.factory.create_or_update_cases(case_structures)
<commit_msg>Correctly set test __name__ so nose sees it<commit_after>import json
import os.path
import uuid
from nose.tools import nottest

import re
from casexml.apps.case.mock import CaseIndex, CaseStructure
from casexml.apps.case.tests.util import assert_user_doesnt_have_cases, \
    assert_user_has_cases
from casexml.apps.phone.models import get_properly_wrapped_sync_log
from casexml.apps.phone.tests.test_sync_mode import SyncBaseTest
from corehq.form_processor.tests import run_with_all_backends
from corehq.util.test_utils import softer_assert

@nottest
def get_test_file_json(filename):
    base = os.path.dirname(__file__)
    file_path = 'data'
    path = os.path.join(base, file_path, '%s.json' % filename)
    with open(path) as f:
        file_contents = f.read()

    return json.loads(file_contents)

def get_test_name(test_name):
    return str("test_%s" % re.sub("\s", "_", test_name))

@nottest
def test_generator(test_name, skip=False):
    @softer_assert
    def test(self):
        if skip:
            self.skipTest(skip)
        self.build_case_structures(test_name)
        desired_cases = self._get_test(test_name).get('outcome', [])
        undesired_cases = [case for case in self.ALL_CASES if case not in desired_cases]
        sync_log = get_properly_wrapped_sync_log(self.sync_log._id)
        self.assertEqual(sync_log.case_ids_on_phone, set(desired_cases))
        assert_user_has_cases(self, self.user, desired_cases)
        assert_user_doesnt_have_cases(self, self.user, undesired_cases)

    test.__name__ = get_test_name(test_name)
    return run_with_all_backends(test)

class TestSequenceMeta(type):

    def __new__(mcs, name, bases, dict):
        tests_to_run = get_test_file_json('case_relationship_tests')
        run_single_tests = filter(lambda t: t.get('only', False), tests_to_run)
        if run_single_tests:
            tests_to_run = run_single_tests

        for test in [(test['name'], test.get('skip', False)) for test in tests_to_run]:
            # Create a new testcase that the test runner is able to find
            dict[get_test_name(test[0])] = test_generator(test[0], test[1])

        return type.__new__(mcs, name, bases, dict)

class IndexTreeTest(SyncBaseTest):
    """Fetch all testcases from data/case_relationship_tests.json and run them

    Each testcase is structured as follows:
    {
        "name": The name of the test,
        "owned": Cases whose owner id is set to the user. Cases not in this list are owned by someone else,
        "subcases": [ A list of ordered pairs e.g. ["a","b"] means "a creates a child index pointing to b"],
        "extensions": [ A list of ordered pairs e.g. ["a","b"] means "a creates an extension index pointing to b"],
        "closed": A list of the closed cases,
        "outcome": When syncing all the cases, which cases should be sent to the phone,
        "only": For debugging purposes, run a single test by adding 'only': true to those tests,
    }
    """
    __metaclass__ = TestSequenceMeta

    ALL_CASES = ['a', 'b', 'c', 'd', 'e']

    @property
    def all_tests(self):
        """All the test cases in a dict"""
        all_tests = {}
        tests = get_test_file_json('case_relationship_tests')
        for test in tests:
            all_tests[test['name']] = test
        return all_tests

    def _get_test(self, test_name):
        return self.all_tests[test_name]

    def build_case_structures(self, test_name):
        test = self._get_test(test_name)
        case_structures = []
        indices = {case: [] for case in self.ALL_CASES}

        for i, subcase in enumerate(test.get('subcases', [])):
            indices[subcase[0]].append(CaseIndex(
                related_structure=CaseStructure(
                    attrs={
                        'create': False,
                        'owner_id': self.user.user_id if subcase[1] in test.get('owned', []) else uuid.uuid4().hex,
                        'close': subcase[1] in test.get('closed', []),
                    },
                    case_id=subcase[1],
                ),
                relationship='child',
                identifier='child_{}'.format(i),
            ))

        for i, extension in enumerate(test.get('extensions', [])):
            indices[extension[0]].append(CaseIndex(
                related_structure=CaseStructure(
                    attrs={
                        'create': False,
                        'owner_id': (self.user.user_id if extension[1]
                                     in test.get('owned', []) else uuid.uuid4().hex),
                        'close': extension[1] in test.get('closed', []),
                    },
                    case_id=extension[1],
                ),
                relationship='extension',
                identifier='extension_{}'.format(i),
            ))

        for case in self.ALL_CASES:
            case_structures.append(CaseStructure(
                attrs={
                    'create': True,
                    'owner_id': self.user.user_id if case in test.get('owned', []) else uuid.uuid4().hex,
                    'close': case in test.get('closed', []),
                },
                case_id=case,
                indices=indices[case],
            ))

        self.factory.create_or_update_cases(case_structures)
<|endoftext|>


<commit_before># It turns out that 2,400,000 cells is too large for Google Docs & Excel.
import csv
import argparse

def analyze(csv_file):
  f = open(csv_file)
  lines = f.readlines()
  f.close()
  headers = lines.pop(0)
  messages = []
  for line in lines:
    name, email, year, age, replies, length = map(int, line.split(","))
    messages.append({"name": name, "email": email, "year": year, "age": age, "replies": replies, "length": length})
  print "Done parsing CSV {0} messages".format(len(messages))

  x_by_y(messages, "name", "year")
  x_by_y(messages, "email", "year")
  x_by_y(messages, "length", "year")
  x_by_y(messages, "name", "replies", 5)
  x_by_y(messages, "email", "replies", 5)
  x_by_y(messages, "length", "replies", 5)

def x_by_y(messages, x, y, max_y=None):
  x_totals = {}
  for m in messages:
    new_y = m[y]
    if max_y and new_y > max_y:
      new_y = max_y + 1
    if new_y not in x_totals:
      x_totals[new_y] = {"count": 0, "val": 0}
    x_totals[new_y]["val"] += m[x]
    x_totals[new_y]["count"] += 1

  data = []
  total_x, total_count = 0, 0
  for y_val in x_totals:
    totals = x_totals[y_val]
    data.append([y_val, float(totals["val"]) / totals["count"]])
    total_x += totals["val"]
    total_count += totals["count"]

  print "{0} total={1}, avg={2}".format(x, total_x, float(total_x)/total_count)
  write_csv("2ch_{0}_{1}.csv".format(x, y), (y, x), data)

def write_csv(fname, headers, list_of_lists):
  f = open(fname, 'wb')
  writer = csv.writer(f)
  writer.writerow(headers)
  for l in list_of_lists:
    writer.writerow(l)
  f.close()

if __name__=="__main__":
  parser = argparse.ArgumentParser(description="2ch CSV analyzer")
  parser.add_argument('-c', '--csv-path', required=True)
  args = parser.parse_args()
  analyze(args.csv_path)
<commit_msg>calculate std dev for message length 2ch<commit_after># It turns out that 2,400,000 cells is too large for Google Docs & Excel.
import csv
import argparse

def analyze(csv_file):
  f = open(csv_file)
  lines = f.readlines()
  f.close()
  headers = lines.pop(0)
  messages = []
  for line in lines:
    name, email, year, age, replies, length = map(int, line.split(","))
    messages.append({"name": name, "email": email, "year": year, "age": age, "replies": replies, "length": length})
  print "Done parsing CSV {0} messages".format(len(messages))

  x_by_y(messages, "name", "year")
  x_by_y(messages, "email", "year")
  x_by_y(messages, "length", "year")
  x_by_y(messages, "name", "replies", 5)
  x_by_y(messages, "email", "replies", 5)
  x_by_y(messages, "length", "replies", 5)
  std_dev(messages, "length")

def x_by_y(messages, x, y, max_y=None):
  x_totals = {}
  for m in messages:
    new_y = m[y]
    if max_y and new_y > max_y:
      new_y = max_y + 1
    if new_y not in x_totals:
      x_totals[new_y] = {"count": 0, "val": 0}
    x_totals[new_y]["val"] += m[x]
    x_totals[new_y]["count"] += 1

  data = []
  total_x, total_count = 0, 0
  for y_val in x_totals:
    totals = x_totals[y_val]
    data.append([y_val, float(totals["val"]) / totals["count"]])
    total_x += totals["val"]
    total_count += totals["count"]

  print "{0} total={1}, avg={2}".format(x, total_x, float(total_x)/total_count)
  write_csv("2ch_{0}_{1}.csv".format(x, y), (y, x), data)

def std_dev(messages, x):
  x_total = 0
  for m in messages:
    x_total += m[x]
  x_mean = float(x_total) / len(messages)

  x_diffs = 0
  for m in messages:
    x_diffs += (m[x] - x_mean)**2
  sd = (float(x_diffs) / len(messages))**0.5
  print "Std dev of {0}={1}".format(x, sd)

def write_csv(fname, headers, list_of_lists):
  f = open(fname, 'wb')
  writer = csv.writer(f)
  writer.writerow(headers)
  for l in list_of_lists:
    writer.writerow(l)
  f.close()

if __name__=="__main__":
  parser = argparse.ArgumentParser(description="2ch CSV analyzer")
  parser.add_argument('-c', '--csv-path', required=True)
  args = parser.parse_args()
  analyze(args.csv_path)
<|endoftext|>


<commit_before># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

import os
import unittest

from openquake.db.models import OqCalculation

from tests.utils import helpers

class ClassicalRiskQATestCase(unittest.TestCase):
    """Single site QA tests for the Classical Risk calculator."""

    def test_classical_psha_based_risk(self):
        """Run the full hazard+risk job, serialize all results to the db,
        and verify them against expected values."""

        expected_results = None

        helpers.run_job(helpers.demo_file(
            os.path.join('classical_psha_based_risk', 'config.gem')))

        calc_id = OqCalculation.objects.latest('id')
<commit_msg>Solid draft of the classical risk QA test; still debugging result discrepancies<commit_after># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

import os
import unittest

from openquake.db.models import LossCurveData
from openquake.db.models import OqCalculation

from tests.utils import helpers

class ClassicalRiskQATestCase(unittest.TestCase):
    """Single site QA tests for the Classical Risk calculator."""

    def test_classical_psha_based_risk(self):
        """Run the full hazard+risk job, serialize all results to the db,
        and verify them against expected values."""

        self.maxDiff =None

        expected_lc_poes = [
            0.03881,
            0.03879,
            0.03793,
            0.03485,
            0.03059,
            0.02644,
            0.02282,
            0.01975,
            0.01717,
            0.01501,
            0.01323,
            0.01054,
            0.00862,
            0.00713,
            0.00591,
            0.00491,
            0.00353,
            0.00274,
            0.00219,
            0.00168,
            0.00119,
            0.00051,
            0.00026,
            0.00019,
            0.00015,
            0.00009,
            0.00005,
            0.00003,
            0.00001,
            0.00001,
            0.00000,
        ]

        helpers.run_job(helpers.demo_file(
            os.path.join('classical_psha_based_risk', 'config.gem')),
            debug='debug')

        calculation = OqCalculation.objects.latest('id')

        loss_curve = LossCurveData.objects.get(
            loss_curve__output__oq_calculation=calculation.id)

        self.assertEqual(expected_lc_poes, loss_curve.poes)
<|endoftext|>


<commit_before># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The genre whitelist can be specified like so in .beetsconfig:

    [lastgenre]
    whitelist=/path/to/genres.txt

The included (default) genre list was produced by scraping Wikipedia.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import logging
import pylast
import os
import yaml

from beets import plugins
from beets import ui
from beets.util import normpath, plurality
from beets import config
from beets import library

log = logging.getLogger('beets')

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

# Core genre identification routine.

def _tags_for(obj):
    """Given a pylast entity (album or track), returns a list of
    tag names for that entity. Returns an empty list if the entity is
    not found or another error occurs.
    """
    try:
        res = super(pylast.Album, obj).get_top_tags()
    except PYLAST_EXCEPTIONS as exc:
        log.debug(u'last.fm error: %s' % unicode(exc))
        return []

    tags = []
    multiple = config['lastgenre']['multiple'].get(bool)
    if multiple:
        min_weight = config['lastgenre']['min_weight'].get(int)
        max_genres = config['lastgenre']['max_genres'].get(int)
    else:
        min_weight = -1
        max_genres = 1

    for el in res:
        # pylast 0.5.x does not use Album.getTopTags, so we don't have a
        # weight for album tags.  However:  Album.getInfo (they use that)
        # returns only ~5 tags, so we use maximum weight for them
        if isinstance(el, pylast.TopItem):
            weight = int(el.weight)
            tag = el.item.get_name().lower()
        else:
            weight = 100
            tag = el.get_name().lower()
        if _is_allowed(tag):
          if min_weight > -1 and min_weight > weight and len(tags) > 0:
            return tags
          log.debug(u'lastfm.tag (min. {}): {} [{}]'.format(min_weight, tag, weight))
          tags.append(tag)
          if len(tags) == max_genres:
            return tags
    return tags

def _is_allowed(genre):
    """Determine whether the genre is present in the whitelist,
    returning a boolean.
    """
    if genre is None:
        return False
    if not options['whitelist'] or genre in options['whitelist']:
        return True
    return False

def _strings_to_genre(tags):
    """Given a list of strings, return a genre. Returns the first string
    that is present in the genre whitelist (or the canonicalization
    tree) or None if no tag is suitable.
    """
    if not tags:
        return None

    if options.get('c14n'):
        # Use the canonicalization tree.
        tags = find_parents(tags[0], options['branches'])

    tags = [t.title() for t in tags]
    if config['lastgenre']['multiple']:
      return u', '.join(tags[:config['lastgenre']['max_genres'].get(int)])
    else:
      return tags[0]

def fetch_genre(lastfm_obj):
    """Return the genre for a pylast entity or None if no suitable genre
    can be found. Ex. 'Electronic, House, Dance'
    """
    return _strings_to_genre(_tags_for(lastfm_obj))

# Canonicalization tree processing.

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [unicode(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx + 1]))
        except ValueError:
            continue
    return [candidate]

# Cached entity lookups.

_genre_cache = {}

def _cached_lookup(entity, method, *args):
    """Get a genre based on the named entity using the callable `method`
    whose arguments are given in the sequence `args`. The genre lookup
    is cached based on the entity name and the arguments.
    """
    # Shortcut if we're missing metadata.
    if any(not s for s in args):
        return None

    key = u'{0}.{1}'.format(entity, u'-'.join(unicode(a) for a in args))
    if key in _genre_cache:
        return _genre_cache[key]
    else:
        genre = fetch_genre(method(*args))
        _genre_cache[key] = genre
        return genre

def fetch_album_genre(obj):
    """Return the album genre for this Item or Album.
    """
    return _cached_lookup(u'album', LASTFM.get_album, obj.albumartist,
                          obj.album)

def fetch_album_artist_genre(obj):
    """Return the album artist genre for this Item or Album.
    """
    return _cached_lookup(u'artist', LASTFM.get_artist, obj.albumartist)

def fetch_artist_genre(item):
    """Returns the track artist genre for this Item.
    """
    return _cached_lookup(u'artist', LASTFM.get_artist, item.artist)

def fetch_track_genre(obj):
    """Returns the track genre for this Item.
    """
    return _cached_lookup(u'track', LASTFM.get_track, obj.artist, obj.title)

# Main plugin logic.

options = {
    'whitelist': None,
    'branches': None,
    'c14n': False,
}
class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()

        self.config.add({
            'whitelist': os.path.join(os.path.dirname(__file__), 'genres.txt'),
            'multiple': False,
            'min_weight': 10,
            'max_genres': 3,
            'fallback': None,
            'canonical': None,
            'source': 'album',
            'force': True,
            'auto': True,
        })

        if self.config['auto']:
            self.import_stages = [self.imported]

        # Read the whitelist file.
        wl_filename = self.config['whitelist'].as_filename()
        whitelist = set()
        with open(wl_filename) as f:
            for line in f:
                line = line.decode('utf8').strip().lower()
                if line:
                    whitelist.add(line)
        options['whitelist'] = whitelist

        # Read the genres tree for canonicalization if enabled.
        c14n_filename = self.config['canonical'].get()
        if c14n_filename is not None:
            c14n_filename = c14n_filename.strip()
            if not c14n_filename:
                c14n_filename = C14N_TREE
            c14n_filename = normpath(c14n_filename)

            genres_tree = yaml.load(open(c14n_filename, 'r'))
            branches = []
            flatten_tree(genres_tree, [], branches)
            options['branches'] = branches
            options['c14n'] = True

    @property
    def sources(self):
        """A tuple of allowed genre sources. May contain 'track',
        'album', or 'artist.'
        """
        source = self.config['source'].as_choice(('track', 'album', 'artist'))
        if source == 'track':
            return 'track', 'album', 'artist'
        elif source == 'album':
            return 'album', 'artist'
        elif source == 'artist':
            return 'artist',

    def _get_genre(self, obj):
        """Get the genre string for an Album or Item object based on
        self.sources. Return a `(genre, source)` pair. The
        prioritization order is:
            - track (for Items only)
            - album
            - artist
            - original
            - fallback
            - None
        """
        # Shortcut to existing genre if not forcing.
        if not self.config['force'] and _is_allowed(obj.genre):
            return obj.genre, 'keep'

        # Track genre (for Items only).
        if isinstance(obj, library.Item):
            if 'track' in self.sources:
                result = fetch_track_genre(obj)
                if result:
                    return result, 'track'

        # Album genre.
        if 'album' in self.sources:
            result = fetch_album_genre(obj)
            if result:
                return result, 'album'

        # Artist (or album artist) genre.
        if 'artist' in self.sources:
            result = None
            if isinstance(obj, library.Item):
                result = fetch_artist_genre(obj)
            elif obj.albumartist != 'Various Artists':
                result = fetch_album_artist_genre(obj)
            else:
                # For "Various Artists", pick the most popular track genre.
                item_genres = []
                for item in obj.items():
                    item_genre = None
                    if 'track' in self.sources:
                        item_genre = fetch_track_genre(item)
                    if not item_genre:
                        item_genre = fetch_artist_genre(item)
                    if item_genre:
                        item_genres.append(item_genre)
                if item_genres:
                    result, _ = plurality(item_genres)

            if result:
                return result, 'artist'

        # Filter the existing genre.
        if obj.genre:
            result = _strings_to_genre([obj.genre])
            if result:
                return result, 'original'

        # Fallback string.
        fallback = self.config['fallback'].get()
        if fallback:
            return fallback, 'fallback'

        return None, None

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres')
        lastgenre_cmd.parser.add_option('-f', '--force', dest='force',
                              action='store_true',
                              default=False,
                              help='re-download genre when already present')
        lastgenre_cmd.parser.add_option('-s', '--source', dest='source',
                              type='string',
                              help='genre source: artist, album, or track')
        def lastgenre_func(lib, opts, args):
            write = config['import']['write'].get(bool)
            self.config.set_args(opts)

            for album in lib.albums(ui.decargs(args)):
                album.genre, src = self._get_genre(album)
                log.info(u'genre for album {0} - {1} ({2}): {3}'.format(
                    album.albumartist, album.album, src, album.genre
                ))
                album.store()

                for item in album.items():
                    # If we're using track-level sources, also look up each
                    # track on the album.
                    if 'track' in self.sources:
                        item.genre, src = self._get_genre(item)
                        item.store()
                        log.info(u'genre for track {0} - {1} ({2}): {3}'.format(
                            item.artist, item.title, src, item.genre
                        ))

                    if write:
                        item.write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        """Event hook called when an import task finishes."""
        if task.is_album:
            album = session.lib.get_album(task.album_id)
            album.genre, src = self._get_genre(album)
            log.debug(u'added last.fm album genre ({0}): {1}'.format(
                  src, album.genre))
            album.store()

            if 'track' in self.sources:
                for item in album.items():
                    item.genre, src = self._get_genre(item)
                    log.debug(u'added last.fm item genre ({0}): {1}'.format(
                          src, item.genre))
                    item.store()

        else:
            item = task.item
            item.genre, src = self._get_genre(item)
            log.debug(u'added last.fm item genre ({0}): {1}'.format(
                  src, item.genre))
            item.store()
<commit_msg>Bugfix for Album.get_top_tracks workaround.<commit_after># This file is part of beets.
# Copyright 2013, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Gets genres for imported music based on Last.fm tags.

Uses a provided whitelist file to determine which tags are valid genres.
The genre whitelist can be specified like so in .beetsconfig:

    [lastgenre]
    whitelist=/path/to/genres.txt

The included (default) genre list was produced by scraping Wikipedia.
The scraper script used is available here:
https://gist.github.com/1241307
"""
import logging
import pylast
import os
import yaml

from beets import plugins
from beets import ui
from beets.util import normpath, plurality
from beets import config
from beets import library

log = logging.getLogger('beets')

LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY)
C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml')

PYLAST_EXCEPTIONS = (
    pylast.WSError,
    pylast.MalformedResponseError,
    pylast.NetworkError,
)

# Core genre identification routine.

def _tags_for(obj):
    """Given a pylast entity (album or track), returns a list of
    tag names for that entity. Returns an empty list if the entity is
    not found or another error occurs.
    """
    try:
        if isinstance(el, pylast.Album):
            res = super(pylast.Album, obj).get_top_tags()
        else:
            res = obj.get_top_tags()
    except PYLAST_EXCEPTIONS as exc:
        log.debug(u'last.fm error: %s' % unicode(exc))
        return []

    tags = []
    multiple = config['lastgenre']['multiple'].get(bool)
    if multiple:
        min_weight = config['lastgenre']['min_weight'].get(int)
        max_genres = config['lastgenre']['max_genres'].get(int)
    else:
        min_weight = -1
        max_genres = 1

    for el in res:
        # pylast 0.5.x does not use Album.getTopTags, so we don't have a
        # weight for album tags.  However:  Album.getInfo (they use that)
        # returns only ~5 tags, so we use maximum weight for them
        if isinstance(el, pylast.TopItem):
            weight = int(el.weight)
            tag = el.item.get_name().lower()
        else:
            weight = 100
            tag = el.get_name().lower()
        if _is_allowed(tag):
          if min_weight > -1 and min_weight > weight and len(tags) > 0:
            return tags
          log.debug(u'lastfm.tag (min. {}): {} [{}]'.format(min_weight, tag, weight))
          tags.append(tag)
          if len(tags) == max_genres:
            return tags
    return tags

def _is_allowed(genre):
    """Determine whether the genre is present in the whitelist,
    returning a boolean.
    """
    if genre is None:
        return False
    if not options['whitelist'] or genre in options['whitelist']:
        return True
    return False

def _strings_to_genre(tags):
    """Given a list of strings, return a genre. Returns the first string
    that is present in the genre whitelist (or the canonicalization
    tree) or None if no tag is suitable.
    """
    if not tags:
        return None

    if options.get('c14n'):
        # Use the canonicalization tree.
        tags = find_parents(tags[0], options['branches'])

    tags = [t.title() for t in tags]
    if config['lastgenre']['multiple']:
      return u', '.join(tags[:config['lastgenre']['max_genres'].get(int)])
    else:
      return tags[0]

def fetch_genre(lastfm_obj):
    """Return the genre for a pylast entity or None if no suitable genre
    can be found. Ex. 'Electronic, House, Dance'
    """
    return _strings_to_genre(_tags_for(lastfm_obj))

# Canonicalization tree processing.

def flatten_tree(elem, path, branches):
    """Flatten nested lists/dictionaries into lists of strings
    (branches).
    """
    if not path:
        path = []

    if isinstance(elem, dict):
        for (k, v) in elem.items():
            flatten_tree(v, path + [k], branches)
    elif isinstance(elem, list):
        for sub in elem:
            flatten_tree(sub, path, branches)
    else:
        branches.append(path + [unicode(elem)])

def find_parents(candidate, branches):
    """Find parents genre of a given genre, ordered from the closest to
    the further parent.
    """
    for branch in branches:
        try:
            idx = branch.index(candidate.lower())
            return list(reversed(branch[:idx + 1]))
        except ValueError:
            continue
    return [candidate]

# Cached entity lookups.

_genre_cache = {}

def _cached_lookup(entity, method, *args):
    """Get a genre based on the named entity using the callable `method`
    whose arguments are given in the sequence `args`. The genre lookup
    is cached based on the entity name and the arguments.
    """
    # Shortcut if we're missing metadata.
    if any(not s for s in args):
        return None

    key = u'{0}.{1}'.format(entity, u'-'.join(unicode(a) for a in args))
    if key in _genre_cache:
        return _genre_cache[key]
    else:
        genre = fetch_genre(method(*args))
        _genre_cache[key] = genre
        return genre

def fetch_album_genre(obj):
    """Return the album genre for this Item or Album.
    """
    return _cached_lookup(u'album', LASTFM.get_album, obj.albumartist,
                          obj.album)

def fetch_album_artist_genre(obj):
    """Return the album artist genre for this Item or Album.
    """
    return _cached_lookup(u'artist', LASTFM.get_artist, obj.albumartist)

def fetch_artist_genre(item):
    """Returns the track artist genre for this Item.
    """
    return _cached_lookup(u'artist', LASTFM.get_artist, item.artist)

def fetch_track_genre(obj):
    """Returns the track genre for this Item.
    """
    return _cached_lookup(u'track', LASTFM.get_track, obj.artist, obj.title)

# Main plugin logic.

options = {
    'whitelist': None,
    'branches': None,
    'c14n': False,
}
class LastGenrePlugin(plugins.BeetsPlugin):
    def __init__(self):
        super(LastGenrePlugin, self).__init__()

        self.config.add({
            'whitelist': os.path.join(os.path.dirname(__file__), 'genres.txt'),
            'multiple': False,
            'min_weight': 10,
            'max_genres': 3,
            'fallback': None,
            'canonical': None,
            'source': 'album',
            'force': True,
            'auto': True,
        })

        if self.config['auto']:
            self.import_stages = [self.imported]

        # Read the whitelist file.
        wl_filename = self.config['whitelist'].as_filename()
        whitelist = set()
        with open(wl_filename) as f:
            for line in f:
                line = line.decode('utf8').strip().lower()
                if line:
                    whitelist.add(line)
        options['whitelist'] = whitelist

        # Read the genres tree for canonicalization if enabled.
        c14n_filename = self.config['canonical'].get()
        if c14n_filename is not None:
            c14n_filename = c14n_filename.strip()
            if not c14n_filename:
                c14n_filename = C14N_TREE
            c14n_filename = normpath(c14n_filename)

            genres_tree = yaml.load(open(c14n_filename, 'r'))
            branches = []
            flatten_tree(genres_tree, [], branches)
            options['branches'] = branches
            options['c14n'] = True

    @property
    def sources(self):
        """A tuple of allowed genre sources. May contain 'track',
        'album', or 'artist.'
        """
        source = self.config['source'].as_choice(('track', 'album', 'artist'))
        if source == 'track':
            return 'track', 'album', 'artist'
        elif source == 'album':
            return 'album', 'artist'
        elif source == 'artist':
            return 'artist',

    def _get_genre(self, obj):
        """Get the genre string for an Album or Item object based on
        self.sources. Return a `(genre, source)` pair. The
        prioritization order is:
            - track (for Items only)
            - album
            - artist
            - original
            - fallback
            - None
        """
        # Shortcut to existing genre if not forcing.
        if not self.config['force'] and _is_allowed(obj.genre):
            return obj.genre, 'keep'

        # Track genre (for Items only).
        if isinstance(obj, library.Item):
            if 'track' in self.sources:
                result = fetch_track_genre(obj)
                if result:
                    return result, 'track'

        # Album genre.
        if 'album' in self.sources:
            result = fetch_album_genre(obj)
            if result:
                return result, 'album'

        # Artist (or album artist) genre.
        if 'artist' in self.sources:
            result = None
            if isinstance(obj, library.Item):
                result = fetch_artist_genre(obj)
            elif obj.albumartist != 'Various Artists':
                result = fetch_album_artist_genre(obj)
            else:
                # For "Various Artists", pick the most popular track genre.
                item_genres = []
                for item in obj.items():
                    item_genre = None
                    if 'track' in self.sources:
                        item_genre = fetch_track_genre(item)
                    if not item_genre:
                        item_genre = fetch_artist_genre(item)
                    if item_genre:
                        item_genres.append(item_genre)
                if item_genres:
                    result, _ = plurality(item_genres)

            if result:
                return result, 'artist'

        # Filter the existing genre.
        if obj.genre:
            result = _strings_to_genre([obj.genre])
            if result:
                return result, 'original'

        # Fallback string.
        fallback = self.config['fallback'].get()
        if fallback:
            return fallback, 'fallback'

        return None, None

    def commands(self):
        lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres')
        lastgenre_cmd.parser.add_option('-f', '--force', dest='force',
                              action='store_true',
                              default=False,
                              help='re-download genre when already present')
        lastgenre_cmd.parser.add_option('-s', '--source', dest='source',
                              type='string',
                              help='genre source: artist, album, or track')
        def lastgenre_func(lib, opts, args):
            write = config['import']['write'].get(bool)
            self.config.set_args(opts)

            for album in lib.albums(ui.decargs(args)):
                album.genre, src = self._get_genre(album)
                log.info(u'genre for album {0} - {1} ({2}): {3}'.format(
                    album.albumartist, album.album, src, album.genre
                ))
                album.store()

                for item in album.items():
                    # If we're using track-level sources, also look up each
                    # track on the album.
                    if 'track' in self.sources:
                        item.genre, src = self._get_genre(item)
                        item.store()
                        log.info(u'genre for track {0} - {1} ({2}): {3}'.format(
                            item.artist, item.title, src, item.genre
                        ))

                    if write:
                        item.write()

        lastgenre_cmd.func = lastgenre_func
        return [lastgenre_cmd]

    def imported(self, session, task):
        """Event hook called when an import task finishes."""
        if task.is_album:
            album = session.lib.get_album(task.album_id)
            album.genre, src = self._get_genre(album)
            log.debug(u'added last.fm album genre ({0}): {1}'.format(
                  src, album.genre))
            album.store()

            if 'track' in self.sources:
                for item in album.items():
                    item.genre, src = self._get_genre(item)
                    log.debug(u'added last.fm item genre ({0}): {1}'.format(
                          src, item.genre))
                    item.store()

        else:
            item = task.item
            item.genre, src = self._get_genre(item)
            log.debug(u'added last.fm item genre ({0}): {1}'.format(
                  src, item.genre))
            item.store()
<|endoftext|>


<commit_before>#
# setup.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from setuptools import setup, find_packages

VERSION             = '0.6.0'

AUTHOR              = 'Antoine Mercadal'
MAIL                = 'antoine.mercadal@archipelproject.org'
URL                 = 'http://archipelproject.org'
LICENSE             = 'AGPL'
NAME                = 'archipel-agent-action-scheduler'
SHORTDESCRIPTION    = "Creation and management of planified tasks for hypervisors and virtual machines"
LONGDESCRIPTION     = ""
ENTRY_POINTS        = { 'archipel.plugin.virtualmachine' : [
                            'factory=archipelagentactionscheduler:make_archipel_plugin'],
                        'archipel.plugin.hypervisor' : [
                            'factory=archipelagentactionscheduler:make_archipel_plugin'],
                        'archipel.plugin' : [
                            'version=archipelagentactionscheduler:version']}

RPM_REQUIRED_DEPS   = "archipel-core, APScheduler"

## HACK FOR DEPS IN RPMS
from setuptools.command.bdist_rpm import bdist_rpm
def custom_make_spec_file(self):
    spec = self._original_make_spec_file()
    lineDescription = "%description"
    spec.insert(spec.index(lineDescription) - 1, "requires: %s" % RPM_REQUIRED_DEPS)
    return spec
bdist_rpm._original_make_spec_file = bdist_rpm._make_spec_file
bdist_rpm._make_spec_file = custom_make_spec_file
## END OF HACK

setup(name=NAME,
      version=VERSION,
      description=SHORTDESCRIPTION,
      long_description=LONGDESCRIPTION,
      classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Environment :: No Input/Output (Daemon)',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: End Users/Desktop',
        'Intended Audience :: Science/Research',
        'Intended Audience :: System Administrators',
        'Intended Audience :: Telecommunications Industry',
        'License :: OSI Approved :: GNU Affero General Public License v3',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Topic :: Internet',
        'Topic :: System :: Emulators',
        'Topic :: System :: Operating System'],
      keywords='archipel, virtualization, libvirt, orchestration',
      author=AUTHOR,
      author_email=MAIL,
      url=URL,
      license=LICENSE,
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
        "apscheduler>=1.3.1",
        "archipel-core>=0.6.0beta"
      ],
      entry_points=ENTRY_POINTS,
      data_files=[
        ('configuration' , ["configuration/plugin.conf"])
        ]
      )
<commit_msg>APScheduler 3.X break the api<commit_after>#
# setup.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from setuptools import setup, find_packages

VERSION             = '0.6.0'

AUTHOR              = 'Antoine Mercadal'
MAIL                = 'antoine.mercadal@archipelproject.org'
URL                 = 'http://archipelproject.org'
LICENSE             = 'AGPL'
NAME                = 'archipel-agent-action-scheduler'
SHORTDESCRIPTION    = "Creation and management of planified tasks for hypervisors and virtual machines"
LONGDESCRIPTION     = ""
ENTRY_POINTS        = { 'archipel.plugin.virtualmachine' : [
                            'factory=archipelagentactionscheduler:make_archipel_plugin'],
                        'archipel.plugin.hypervisor' : [
                            'factory=archipelagentactionscheduler:make_archipel_plugin'],
                        'archipel.plugin' : [
                            'version=archipelagentactionscheduler:version']}

RPM_REQUIRED_DEPS   = "archipel-core, APScheduler"

## HACK FOR DEPS IN RPMS
from setuptools.command.bdist_rpm import bdist_rpm
def custom_make_spec_file(self):
    spec = self._original_make_spec_file()
    lineDescription = "%description"
    spec.insert(spec.index(lineDescription) - 1, "requires: %s" % RPM_REQUIRED_DEPS)
    return spec
bdist_rpm._original_make_spec_file = bdist_rpm._make_spec_file
bdist_rpm._make_spec_file = custom_make_spec_file
## END OF HACK

setup(name=NAME,
      version=VERSION,
      description=SHORTDESCRIPTION,
      long_description=LONGDESCRIPTION,
      classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Environment :: No Input/Output (Daemon)',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: End Users/Desktop',
        'Intended Audience :: Science/Research',
        'Intended Audience :: System Administrators',
        'Intended Audience :: Telecommunications Industry',
        'License :: OSI Approved :: GNU Affero General Public License v3',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Topic :: Internet',
        'Topic :: System :: Emulators',
        'Topic :: System :: Operating System'],
      keywords='archipel, virtualization, libvirt, orchestration',
      author=AUTHOR,
      author_email=MAIL,
      url=URL,
      license=LICENSE,
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
        "apscheduler<3",
        "archipel-core>=0.6.0beta"
      ],
      entry_points=ENTRY_POINTS,
      data_files=[
        ('configuration' , ["configuration/plugin.conf"])
        ]
      )
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess

import qrcode
import sqlite3
import urllib

import click

@click.group()
def cli():
    pass

DB_TYPE_GOOGLE_AUTHENTICATOR = "GA"

# utils
# NOTE: this is just a heuristic that worked well for me (for nice labelling)
def extract_issuer_from_email(email):
    if "@" in email:
        return email.split("@")[1]

    if "-" in email:
        return email.split("-")[0]

    if ":" in email:
        return email.split(":")[0]

# db parsing handlers
def parse_google_authenticator_db(db_path):
    db = sqlite3.connect(db_path)

    for account in db.execute("SELECT * FROM accounts"):
        id, email, secret, _, _, _, issuer = account[:7]

        if issuer is None:
            issuer = extract_issuer_from_email(email)

        label = issuer

        # template: otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&issuer=Example
        line = "otpauth://totp/%s:%s?secret=%s&issuer=%s" % (urllib.quote(label), urllib.quote(email), secret, urllib.quote(issuer))

        yield (id, label, issuer, email, secret, line)

# db pulling handlers
def pull_google_authenticator_db(dest):
    cmdline = "adb pull /data/data/net.singular.authenticator/databases/databases %s" % dest

    print "executing '%s', output:" % cmdline
    pipe = subprocess.Popen(cmdline.split(), stdout=subprocess.PIPE)
    print pipe.stdout.read()
    print
    print "done."

@cli.command(name="generate", help="Generate QR Codes for 2FA database entries")
@click.option("--db", help="Path to database file", required=True)
@click.option("--type", help="database type (default: %s)" % DB_TYPE_GOOGLE_AUTHENTICATOR,
              default=DB_TYPE_GOOGLE_AUTHENTICATOR)
def generate(db, type):
    DB_TYPE_HANDLERS = {
        DB_TYPE_GOOGLE_AUTHENTICATOR: parse_google_authenticator_db
    }

    db_type_handler = DB_TYPE_HANDLERS[type]

    for id, label, issuer, email, secret, line in db_type_handler(db):
        filename = "%s.png" % label

        print "writing QR Code for '%s' => %s" % (label, filename)

        qrcode.make(line).save(file(filename, "wb"), "png")

@cli.command(name="pull", help="Pull 2FA database file from your device")
@click.option("--type", help="database type (default: %s)" % DB_TYPE_GOOGLE_AUTHENTICATOR,
              default=DB_TYPE_GOOGLE_AUTHENTICATOR)
@click.option("--dest", help="destination filename (default=db.bin)", default="db.bin")
def pull(type, dest):
    DB_TYPE_HANDLERS = {
        DB_TYPE_GOOGLE_AUTHENTICATOR: pull_google_authenticator_db
    }

    db_type_handler = DB_TYPE_HANDLERS[type]

    print "pulling db of type '%s' => %s" % (type, dest)
    db_type_handler(dest)

if __name__ == '__main__':
    cli()
<commit_msg>2fa_migrator - added error message/exitcode when no handler is found<commit_after>#!/usr/bin/env python
import sys
import subprocess

import qrcode
import sqlite3
import urllib

import click

@click.group()
def cli():
    pass

DB_TYPE_GOOGLE_AUTHENTICATOR = "GA"

# utils
# NOTE: this is just a heuristic that worked well for me (for nice labelling)
def extract_issuer_from_email(email):
    if "@" in email:
        return email.split("@")[1]

    if "-" in email:
        return email.split("-")[0]

    if ":" in email:
        return email.split(":")[0]

# db parsing handlers
def parse_google_authenticator_db(db_path):
    db = sqlite3.connect(db_path)

    for account in db.execute("SELECT * FROM accounts"):
        id, email, secret, _, _, _, issuer = account[:7]

        if issuer is None:
            issuer = extract_issuer_from_email(email)

        label = issuer

        # template: otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&issuer=Example
        line = "otpauth://totp/%s:%s?secret=%s&issuer=%s" % (urllib.quote(label), urllib.quote(email), secret, urllib.quote(issuer))

        yield (id, label, issuer, email, secret, line)

# db pulling handlers
def pull_google_authenticator_db(dest):
    cmdline = "adb pull /data/data/net.singular.authenticator/databases/databases %s" % dest

    print "executing '%s', output:" % cmdline
    pipe = subprocess.Popen(cmdline.split(), stdout=subprocess.PIPE)
    print pipe.stdout.read()
    print
    print "done."

@cli.command(name="generate", help="Generate QR Codes for 2FA database entries")
@click.option("--db", help="Path to database file", required=True)
@click.option("--type", help="database type (default: %s)" % DB_TYPE_GOOGLE_AUTHENTICATOR,
              default=DB_TYPE_GOOGLE_AUTHENTICATOR)
def generate(db, type):
    DB_TYPE_HANDLERS = {
        DB_TYPE_GOOGLE_AUTHENTICATOR: parse_google_authenticator_db
    }

    if type not in DB_TYPE_HANDLERS:
        sys.stderr.write("No handler for db type '%s' for generate! implement it! :)\n" % type)
        sys.exit(1)

    db_type_handler = DB_TYPE_HANDLERS[type]

    for id, label, issuer, email, secret, line in db_type_handler(db):
        filename = "%s.png" % label

        print "writing QR Code for '%s' => %s" % (label, filename)

        qrcode.make(line).save(file(filename, "wb"), "png")

@cli.command(name="pull", help="Pull 2FA database file from your device")
@click.option("--type", help="database type (default: %s)" % DB_TYPE_GOOGLE_AUTHENTICATOR,
              default=DB_TYPE_GOOGLE_AUTHENTICATOR)
@click.option("--dest", help="destination filename (default=db.bin)", default="db.bin")
def pull(type, dest):
    DB_TYPE_HANDLERS = {
        DB_TYPE_GOOGLE_AUTHENTICATOR: pull_google_authenticator_db
    }

    if type not in DB_TYPE_HANDLERS:
        sys.stderr.write("No handler for db type '%s' for pull! implement it! :)\n" % type)
        sys.exit(1)

    db_type_handler = DB_TYPE_HANDLERS[type]

    print "pulling db of type '%s' => %s" % (type, dest)
    db_type_handler(dest)

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation and G. Weatherill
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Strong motion utilities.
"""
# WARNING: this module is intended to collect functions used in various places
# throughout the code. Consequently, try to limit the amount of stuff here and in
# particular the amount of imports, which might slow down the code unnecessarily
import os
import sys
import re
import numpy as np
from scipy.integrate import cumtrapz
from scipy.constants import g

from openquake.hazardlib.gsim import get_available_gsims
from openquake.hazardlib.scalerel.peer import PeerMSR
from openquake.hazardlib.gsim.gmpe_table import GMPETable
from openquake.hazardlib.gsim.base import GMPE

if sys.version_info[0] >= 3:
    import pickle
else:
    import cPickle as pickle  # pylint: disable=import-error

# Get a list of the available GSIMs
AVAILABLE_GSIMS = get_available_gsims()

def check_gsim_list(gsim_list):
    """
    Check the GSIM models or strings in `gsim_list`, and return a dict of
    gsim names (str) mapped to their :class:`openquake.hazardlib.Gsim`.
    Raises error if any Gsim in the list is supported in OpenQuake.

    If a Gsim is passed as instance, its string representation is inferred
    from the class name and optional arguments. If a Gsim is passed as string,
    the associated class name is fetched from the OpenQuake available Gsims.

    :param gsim_list: list of GSIM names (str) or OpenQuake Gsims
    :return: a dict of GSIM names (str) mapped to the associated GSIM
    """
    output_gsims = {}
    for gs in gsim_list:
        if isinstance(gs, GMPE):
            # retrieve the name of an instantated GMPE via `_get_gmpe_name`:
            output_gsims[_get_gmpe_name(gs)] = gs
        elif gs.startswith("GMPETable"):
            # Get filename
            match = re.match(r'^GMPETable\(([^)]+?)\)$', gs)
            filepath = match.group(1).split("=")[1]
            output_gsims[gs] = GMPETable(gmpe_table=filepath)
        elif gs not in AVAILABLE_GSIMS:
            raise ValueError('%s Not supported by OpenQuake' % gs)
        else:
            output_gsims[gs] = AVAILABLE_GSIMS[gs]()
    return output_gsims

def _get_gmpe_name(gsim):
    """
    Returns the name of the GMPE given an instance of the class
    """
    if gsim.__class__.__name__.startswith("GMPETable"):
        match = re.match(r'^GMPETable\(([^)]+?)\)$', str(gsim))
        filepath = match.group(1).split("=")[1][1:-1]
        return 'GMPETable(gmpe_table=%s)' % filepath
    else:
        gsim_name = gsim.__class__.__name__
        additional_args = []
        # Try to build the GSIM with arguments. The idea si to provide a sort
        # of unique representation which might be used to get back the GSIM from
        # string. So please, NO fancy stuff (replacements, case changee, and so on.
        # Maybe quote string arguments in the future?)
        for key in gsim.__dict__:
            if key.startswith("kwargs"):
                continue
            val = str(gsim.__dict__[key])
            additional_args.append("{:s}={:s}".format(key, val))
        if len(additional_args):
            gsim_name_str = "({:s})".format(", ".join(additional_args))
            return gsim_name + gsim_name_str
        else:
            return gsim_name

def get_time_vector(time_step, number_steps):
    """
    General SMTK utils
    """
    return np.cumsum(time_step * np.ones(number_steps, dtype=float)) - time_step

def nextpow2(nval):
    m_f = np.log2(nval)
    m_i = np.ceil(m_f)
    return int(2.0 ** m_i)

def convert_accel_units(acceleration, from_, to_='cm/s/s'):  # noqa
    """
    Converts acceleration from/to different units

    :param acceleration: the acceleration (numeric or numpy array)
    :param from_: unit of `acceleration`: string in "g", "m/s/s", "m/s**2",
        "m/s^2", "cm/s/s", "cm/s**2" or "cm/s^2"
    :param to_: new unit of `acceleration`: string in "g", "m/s/s", "m/s**2",
        "m/s^2", "cm/s/s", "cm/s**2" or "cm/s^2". When missing, it defaults
        to "cm/s/s"

    :return: acceleration converted to the given units (by default, 'cm/s/s')
    """
    m_sec_square = ("m/s/s", "m/s**2", "m/s^2")
    cm_sec_square = ("cm/s/s", "cm/s**2", "cm/s^2")
    acceleration = np.asarray(acceleration)
    if from_ == 'g':
        if to_ == 'g':
            return acceleration
        if to_ in m_sec_square:
            return acceleration * g
        if to_ in cm_sec_square:
            return acceleration * (100 * g)
    elif from_ in m_sec_square:
        if to_ == 'g':
            return acceleration / g
        if to_ in m_sec_square:
            return acceleration
        if to_ in cm_sec_square:
            return acceleration * 100
    elif from_ in cm_sec_square:
        if to_ == 'g':
            return acceleration / (100 * g)
        if to_ in m_sec_square:
            return acceleration / 100
        if to_ in cm_sec_square:
            return acceleration

    raise ValueError("Unrecognised time history units. "
                     "Should take either ''g'', ''m/s/s'' or ''cm/s/s''")

def get_velocity_displacement(time_step, acceleration, units="cm/s/s",
                              velocity=None, displacement=None):
    """
    Returns the velocity and displacment time series using simple integration
    :param float time_step:
        Time-series time-step (s)
    :param numpy.ndarray acceleration:
        Acceleration time-history
    :returns:
        velocity - Velocity Time series (cm/s)
        displacement - Displacement Time series (cm)
    """
    acceleration = convert_accel_units(acceleration, units)
    if velocity is None:
        velocity = time_step * cumtrapz(acceleration, initial=0.)
    if displacement is None:
        displacement = time_step * cumtrapz(velocity, initial=0.)
    return velocity, displacement

def _save_image(filename, fig, format='png', dpi=300, **kwargs):  # noqa
    """
    Saves the matplotlib figure `fig` to `filename`. Wrapper around `fig.savefig`
    with `dpi=300` by default and `format` inferred from `filename` extension
    or, if no extension is found, set as "png".
    If filename is empty this function does nothing and return

    :param str filename: str, the file path
    :param figure: a :class:`matplotlib.figure.Figure` (e.g. via
        `matplotlib.pyplot.figure()`)
    :param format: string, the image format. Default: 'png'. This argument is
        ignored if `filename` has a file extension, as `format` will be set
        equal to the extension without leading dot.
    :param str kwargs: additional keyword arguments to pass to `fig.savefig`.
        For details, see:
        https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.savefig.html
    """
    if not filename:
        return

    name, ext = os.path.splitext(filename)
    if ext:
        format = ext[1:]  # noqa
    else:
        filename = name + '.' + format

    fig.savefig(filename, dpi=dpi, format=format, **kwargs)

def load_pickle(pickle_file):
    """
    Python 2 & 3 compatible way of loading a Python Pickle file
    """
    try:
        with open(pickle_file, 'rb') as f:
            pickle_data = pickle.load(f)
    except UnicodeDecodeError as e:
        with open(pickle_file, 'rb') as f:
            pickle_data = pickle.load(f, encoding='latin1')
    except Exception as e:
        print('Unable to load data ', pickle_file, ':', e)
        raise
    return pickle_data

# Moved from sm_database: Mechanism type to Rake conversion:
MECHANISM_TYPE = {
    "Normal": -90.0,
    "Strike-Slip": 0.0,
    "Reverse": 90.0,
    "Oblique": 0.0,
    "Unknown": 0.0,
    "N": -90.0,  # Flatfile conventions
    "S": 0.0,
    "R": 90.0,
    "U": 0.0,
    "NF": -90.,  # ESM flatfile conventions
    "SS": 0.,
    "TF": 90.,
    "NS": -45.,  # Normal with strike-slip component
    "TS": 45.,  # Reverse with strike-slip component
    "O": 0.0
}

DIP_TYPE = {
    "Normal": 60.0,
    "Strike-Slip": 90.0,
    "Reverse": 35.0,
    "Oblique": 60.0,
    "Unknown": 90.0,
    "N": 60.0, # Flatfile conventions
    "S": 90.0,
    "R": 35.0,
    "U": 90.0,
    "NF": 60., # ESM flatfile conventions
    "SS": 90.,
    "TF": 35.,
    "NS": 70., # Normal with strike-slip component
    "TS": 45., # Reverse with strike-slip component
    "O": 90.0
}

# mean utilities (geometric, arithmetic, ...):
SCALAR_XY = {
    "Geometric": lambda x, y: np.sqrt(x * y),
    "Arithmetic": lambda x, y: (x + y) / 2.,
    "Larger": lambda x, y: np.max(np.array([x, y]), axis=0),
    "Vectorial": lambda x, y: np.sqrt(x ** 2. + y ** 2.)
}

DEFAULT_MSR = PeerMSR()

def get_interpolated_period(target_period, periods, values):
    """
    Returns the spectra interpolated in loglog space
    :param float target_period:
        Period required for interpolation
    :param np.ndarray periods:
        Spectral Periods
    :param np.ndarray values:
        Ground motion values
    """
    if (target_period < np.min(periods)) or (target_period > np.max(periods)):
        return None, "Period not within calculated range %s"
    lval = np.where(periods <= target_period)[0][-1]
    uval = np.where(periods >= target_period)[0][0]

    if (uval - lval) == 0:
        return values[lval]

    d_y = np.log10(values[uval]) - np.log10(values[lval])
    d_x = np.log10(periods[uval]) - np.log10(periods[lval])
    return 10.0 ** (
        np.log10(values[lval]) +
        (np.log10(target_period) - np.log10(periods[lval])) * d_y / d_x
        )
<commit_msg>refactoring docstrings<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2017 GEM Foundation and G. Weatherill
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Strong motion utilities.
"""
# WARNING: this module is intended to collect functions used in various places
# throughout the code. Consequently, try to limit the amount of stuff here and in
# particular the amount of imports, which might slow down the code unnecessarily
import os
import sys
import re
import numpy as np
from scipy.integrate import cumtrapz
from scipy.constants import g

from openquake.hazardlib.gsim import get_available_gsims
from openquake.hazardlib.scalerel.peer import PeerMSR
from openquake.hazardlib.gsim.gmpe_table import GMPETable
from openquake.hazardlib.gsim.base import GMPE

if sys.version_info[0] >= 3:
    import pickle
else:
    import cPickle as pickle  # pylint: disable=import-error

# Get a list of the available GSIMs
AVAILABLE_GSIMS = get_available_gsims()

_gmpetable_regex = re.compile(r'^GMPETable\(([^)]+?)\)$')

def check_gsim_list(gsim_list):
    """
    Check the GSIM models or strings in `gsim_list`, and return a dict of
    gsim names (str) mapped to their :class:`openquake.hazardlib.Gsim`.
    Raises error if any Gsim in the list is supported in OpenQuake.

    If a Gsim is passed as instance, its string representation is inferred
    from the class name and optional arguments. If a Gsim is passed as string,
    the associated class name is fetched from the OpenQuake available Gsims.

    :param gsim_list: list of GSIM names (str) or OpenQuake Gsims
    :return: a dict of GSIM names (str) mapped to the associated GSIM
    """
    output_gsims = {}
    for gs in gsim_list:
        if isinstance(gs, GMPE):
            output_gsims[_get_gmpe_name(gs)] = gs  # get name of GMPE instance
        elif gs in AVAILABLE_GSIMS:
            output_gsims[gs] = AVAILABLE_GSIMS[gs]()
        else:
            match = _gmpetable_regex.match(gs)  # GMPETable ?
            if match:
                filepath = match.group(1).split("=")[1]  # get table filename
                output_gsims[gs] = GMPETable(gmpe_table=filepath)
            else:
                raise ValueError('%s Not supported by OpenQuake' % gs)

    return output_gsims

def _get_gmpe_name(gsim):
    """
    Returns the name of the GMPE given an instance of the class
    """
    match = _gmpetable_regex.match(str(gsim))  # GMPETable ?
    if match:
        filepath = match.group(1).split("=")[1][1:-1]
        return 'GMPETable(gmpe_table=%s)' % filepath
    else:
        gsim_name = gsim.__class__.__name__
        additional_args = []
        # Build the GSIM string by showing name and arguments. Keep things
        # simple (no replacements, no case changes) as we might want to be able
        # to get back the GSIM from its string in the future.
        for key in gsim.__dict__:
            if key.startswith("kwargs"):
                continue
            val = str(gsim.__dict__[key])  # quoting strings with json maybe?
            additional_args.append("{:s}={:s}".format(key, val))
        if len(additional_args):
            gsim_name_str = "({:s})".format(", ".join(additional_args))
            return gsim_name + gsim_name_str
        else:
            return gsim_name

def get_time_vector(time_step, number_steps):
    """
    General SMTK utils
    """
    return np.cumsum(time_step * np.ones(number_steps, dtype=float)) - time_step

def nextpow2(nval):
    m_f = np.log2(nval)
    m_i = np.ceil(m_f)
    return int(2.0 ** m_i)

def convert_accel_units(acceleration, from_, to_='cm/s/s'):  # noqa
    """
    Converts acceleration from/to different units

    :param acceleration: the acceleration (numeric or numpy array)
    :param from_: unit of `acceleration`: string in "g", "m/s/s", "m/s**2",
        "m/s^2", "cm/s/s", "cm/s**2" or "cm/s^2"
    :param to_: new unit of `acceleration`: string in "g", "m/s/s", "m/s**2",
        "m/s^2", "cm/s/s", "cm/s**2" or "cm/s^2". When missing, it defaults
        to "cm/s/s"

    :return: acceleration converted to the given units (by default, 'cm/s/s')
    """
    m_sec_square = ("m/s/s", "m/s**2", "m/s^2")
    cm_sec_square = ("cm/s/s", "cm/s**2", "cm/s^2")
    acceleration = np.asarray(acceleration)
    if from_ == 'g':
        if to_ == 'g':
            return acceleration
        if to_ in m_sec_square:
            return acceleration * g
        if to_ in cm_sec_square:
            return acceleration * (100 * g)
    elif from_ in m_sec_square:
        if to_ == 'g':
            return acceleration / g
        if to_ in m_sec_square:
            return acceleration
        if to_ in cm_sec_square:
            return acceleration * 100
    elif from_ in cm_sec_square:
        if to_ == 'g':
            return acceleration / (100 * g)
        if to_ in m_sec_square:
            return acceleration / 100
        if to_ in cm_sec_square:
            return acceleration

    raise ValueError("Unrecognised time history units. "
                     "Should take either ''g'', ''m/s/s'' or ''cm/s/s''")

def get_velocity_displacement(time_step, acceleration, units="cm/s/s",
                              velocity=None, displacement=None):
    """
    Returns the velocity and displacment time series using simple integration
    :param float time_step:
        Time-series time-step (s)
    :param numpy.ndarray acceleration:
        Acceleration time-history
    :returns:
        velocity - Velocity Time series (cm/s)
        displacement - Displacement Time series (cm)
    """
    acceleration = convert_accel_units(acceleration, units)
    if velocity is None:
        velocity = time_step * cumtrapz(acceleration, initial=0.)
    if displacement is None:
        displacement = time_step * cumtrapz(velocity, initial=0.)
    return velocity, displacement

def _save_image(filename, fig, format='png', dpi=300, **kwargs):  # noqa
    """
    Saves the matplotlib figure `fig` to `filename`. Wrapper around `fig.savefig`
    with `dpi=300` by default and `format` inferred from `filename` extension
    or, if no extension is found, set as "png".
    If filename is empty this function does nothing and return

    :param str filename: str, the file path
    :param figure: a :class:`matplotlib.figure.Figure` (e.g. via
        `matplotlib.pyplot.figure()`)
    :param format: string, the image format. Default: 'png'. This argument is
        ignored if `filename` has a file extension, as `format` will be set
        equal to the extension without leading dot.
    :param str kwargs: additional keyword arguments to pass to `fig.savefig`.
        For details, see:
        https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.savefig.html
    """
    if not filename:
        return

    name, ext = os.path.splitext(filename)
    if ext:
        format = ext[1:]  # noqa
    else:
        filename = name + '.' + format

    fig.savefig(filename, dpi=dpi, format=format, **kwargs)

def load_pickle(pickle_file):
    """
    Python 2 & 3 compatible way of loading a Python Pickle file
    """
    try:
        with open(pickle_file, 'rb') as f:
            pickle_data = pickle.load(f)
    except UnicodeDecodeError as e:
        with open(pickle_file, 'rb') as f:
            pickle_data = pickle.load(f, encoding='latin1')
    except Exception as e:
        print('Unable to load data ', pickle_file, ':', e)
        raise
    return pickle_data

# Moved from sm_database: Mechanism type to Rake conversion:
MECHANISM_TYPE = {
    "Normal": -90.0,
    "Strike-Slip": 0.0,
    "Reverse": 90.0,
    "Oblique": 0.0,
    "Unknown": 0.0,
    "N": -90.0,  # Flatfile conventions
    "S": 0.0,
    "R": 90.0,
    "U": 0.0,
    "NF": -90.,  # ESM flatfile conventions
    "SS": 0.,
    "TF": 90.,
    "NS": -45.,  # Normal with strike-slip component
    "TS": 45.,  # Reverse with strike-slip component
    "O": 0.0
}

DIP_TYPE = {
    "Normal": 60.0,
    "Strike-Slip": 90.0,
    "Reverse": 35.0,
    "Oblique": 60.0,
    "Unknown": 90.0,
    "N": 60.0, # Flatfile conventions
    "S": 90.0,
    "R": 35.0,
    "U": 90.0,
    "NF": 60., # ESM flatfile conventions
    "SS": 90.,
    "TF": 35.,
    "NS": 70., # Normal with strike-slip component
    "TS": 45., # Reverse with strike-slip component
    "O": 90.0
}

# mean utilities (geometric, arithmetic, ...):
SCALAR_XY = {
    "Geometric": lambda x, y: np.sqrt(x * y),
    "Arithmetic": lambda x, y: (x + y) / 2.,
    "Larger": lambda x, y: np.max(np.array([x, y]), axis=0),
    "Vectorial": lambda x, y: np.sqrt(x ** 2. + y ** 2.)
}

DEFAULT_MSR = PeerMSR()

def get_interpolated_period(target_period, periods, values):
    """
    Returns the spectra interpolated in loglog space
    :param float target_period:
        Period required for interpolation
    :param np.ndarray periods:
        Spectral Periods
    :param np.ndarray values:
        Ground motion values
    """
    if (target_period < np.min(periods)) or (target_period > np.max(periods)):
        return None, "Period not within calculated range %s"
    lval = np.where(periods <= target_period)[0][-1]
    uval = np.where(periods >= target_period)[0][0]

    if (uval - lval) == 0:
        return values[lval]

    d_y = np.log10(values[uval]) - np.log10(values[lval])
    d_x = np.log10(periods[uval]) - np.log10(periods[lval])
    return 10.0 ** (
        np.log10(values[lval]) +
        (np.log10(target_period) - np.log10(periods[lval])) * d_y / d_x
        )
<|endoftext|>