<commit_before>"""
"Makefile" for doit to test the whole package with Jenkins.

This file gets run with the ``cea test`` command as well as by the Jenkins continuous integration server. It runs all
the unit tests in the ``cea/tests/`` folder as well as some of the CEA scripts, to make sure they at least run through.

In order to run reference cases besides the one called "open", you will need to set up authentication to the private
GitHub repository. To do this, you need to create a GitHub [authentication token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/).
Then, create a text file called ``cea_github.auth`` in your home directory (e.g. ``C:\Users\your-user-name`` for Windows systems or equivalent on
POSIX systems, ask your administrator if you don't know what this is). The file should contain two lines, the first being
your GitHub user name the second the authentication token.

The reference cases can be found here: https://github.com/architecture-building-systems/cea-reference-case/archive/master.zip
"""
import os
import shutil
import sys
import zipfile
import tempfile
import cea.inputlocator
import cea.config

import requests

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"

__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

REPOSITORY_URL = "https://github.com/architecture-building-systems/cea-reference-case/archive/%s.zip"
REPOSITORY_NAME = "master"

if 'JOB_NAME' in os.environ:
    # this script is being run as part of a Jenkins job
    ARCHIVE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), '%JOB_NAME%/cea-reference-case.zip'))
    REFERENCE_CASE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), '%JOB_NAME%/cea-reference-case'))
else:
    ARCHIVE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), 'cea-reference-case.zip'))
    REFERENCE_CASE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), 'cea-reference-case'))

REFERENCE_CASES = {
    'open': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME, "reference-case-open",
                         "baseline"),
    'zug/baseline': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME, "reference-case-zug",
                                 "baseline"),
    'zurich/baseline': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME,
                                    "reference-case-zurich",
                                    "baseline"),
    'zurich/masterplan': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME,
                                      "reference-case-zurich",
                                      "masterplan")}

REFERENCE_CASES_DATA = {
    'open': {'databases': 'CH', 'weather': 'Zug-inducity_1990_2010_TMY', 'latitude': 47.1628017306431, 'longitude': 8.31,
             'radiation': 'open.baseline.radiation.csv',
             'properties_surfaces': 'open.baseline.properties_surfaces.csv'},
    'zug/baseline': {'databases': 'CH', 'weather': 'Zug-inducity_1990_2010_TMY', 'latitude': 47.1628017306431, 'longitude': 8.31,
                     'radiation': 'zug.baseline.radiation.csv',
                     'properties_surfaces': 'zug.baseline.properties_surfaces.csv'},
    'zurich/baseline': {'databases': 'CH', 'weather': 'Zuerich-Kloten_1990_2010_TMY', 'latitude': 46.9524055556, 'longitude': 7.43958333333,
                        'radiation': 'hq.baseline.radiation.csv',
                        'properties_surfaces': 'hq.baseline.properties_surfaces.csv'},
    'zurich/masterplan': {'databases': 'CH', 'weather': 'Zuerich-ETHZ_1990-2010_TMY', 'latitude': 46.9524055556, 'longitude': 7.43958333333,
                          'radiation': 'hq.masterplan.radiation.csv',
                          'properties_surfaces': 'hq.masterplan.properties_surfaces.csv'}}

# set to github user and personal access token in main
_user = None
_token = None
_reference_cases = []

def get_github_auth():
    """
    get the username / token for github from a file in the home directory
    called "cea_github.auth". The first line contains the user, the second line
    the personal access token.

    :return: (user, token)
    """
    if _user and _token:
        return _user, _token

    with open(os.path.expanduser('~/cea_github.auth')) as f:
        user, token = map(str.strip, f.readlines())
    return user, token

def task_run_unit_tests():
    """run the unittests"""
    def run_unit_tests():
        import unittest
        import os

        testsuite = unittest.defaultTestLoader.discover(os.path.dirname(__file__))
        result = unittest.TextTestRunner(verbosity=1).run(testsuite)
        return result.wasSuccessful()
    return {
        'name': 'run_unit_tests',
        'actions': [run_unit_tests],
        'task_dep': [],
        'verbosity': 1
    }

def task_download_reference_cases():
    """Download the (current) state of the reference cases"""
    def download_reference_cases():
        if os.path.exists(REFERENCE_CASE_PATH):
            print('Removing folder: %s' % REFERENCE_CASE_PATH)
            shutil.rmtree(REFERENCE_CASE_PATH)
            assert not os.path.exists(REFERENCE_CASE_PATH), 'FAILED to remove folder %s' % REFERENCE_CASE_PATH
        # extract the bundled reference case (we will use this anyways
        import cea.examples
        archive = zipfile.ZipFile(os.path.join(os.path.dirname(cea.examples.__file__), 'reference-case-open.zip'))
        archive.extractall(os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME))

        if len([rc for rc in _reference_cases if rc.lower() != 'open']):
            if os.path.exists(ARCHIVE_PATH):
                os.remove(ARCHIVE_PATH)
            r = requests.get(REPOSITORY_URL % REPOSITORY_NAME, auth=get_github_auth())
            assert r.ok, 'could not download the reference case'
            with open(ARCHIVE_PATH, 'wb') as f:
                f.write(r.content)
            # extract the reference cases to the temp folder
            archive = zipfile.ZipFile(ARCHIVE_PATH)
            archive.extractall(REFERENCE_CASE_PATH)

    return {
        'name' : 'download_reference_cases',
        'actions': [download_reference_cases],
    }

def task_run_data_initializer():
    """Run the data initializer for each reference case"""

    def run_data_initializer(scenario_path, databases_path):
        import cea.datamanagement.data_initializer
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        config.data_initializer.databases_path = databases_path
        cea.datamanagement.data_initializer.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        databases_path = REFERENCE_CASES_DATA[reference_case]['databases']

        yield {
            'name': 'run_data_initializer:%s' % reference_case,
            'task_dep': ['download_reference_cases'],
            'actions': [
                (run_data_initializer, [], {
                    'scenario_path': scenario_path,
                    'databases_path': databases_path})],
        }

def task_run_data_helper():
    """Run the data helper for each reference case"""
    import cea.datamanagement.archetypes_mapper

    def run_data_helper(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.datamanagement.archetypes_mapper.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_data_helper:%s' % reference_case,
            'task_dep': ['run_data_initializer:%s' % reference_case],
            'actions': [
                (run_data_helper, [], {
                    'scenario_path': scenario_path})],
        }

def task_run_demand():
    """run the demand script for each reference cases and weather file"""
    import cea.demand.demand_main

    def run_demand(scenario_path, weather):
        import cea.config
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path

        # make sure weather file is copied to inputs first
        import cea.datamanagement.weather_helper
        import cea.demand.schedule_maker.schedule_maker
        config.weather_helper.weather = weather
        cea.datamanagement.weather_helper.main(config)
        cea.demand.schedule_maker.schedule_maker.main(config)
        cea.demand.demand_main.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        weather = REFERENCE_CASES_DATA[reference_case]['weather']

        yield {
            'name': 'run_demand:%(reference_case)s' % locals(),
            'task_dep': ['download_reference_cases', 'run_data_helper:%s' % reference_case],
            'actions': [(run_demand, [], {
                'scenario_path': scenario_path,
                'weather': weather
            })],
        }

def task_run_embodied_energy():
    """Run the embodied energy script for each reference case"""
    import cea.analysis.lca.embodied

    def run_embodied_energy(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        config.emissions.year_to_calculate = 2050
        cea.analysis.lca.embodied.main(config=config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_embodied_energy:%(reference_case)s' % locals(),
            'task_dep': ['download_reference_cases', 'run_data_helper:%s' % reference_case],
            'actions': [(run_embodied_energy, [scenario_path])],
        }

def task_run_emissions_operation():
    """run the emissions operation script for each reference case"""
    import cea.analysis.lca.operation

    def run_emissions_operation(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.analysis.lca.operation.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_emissions_operation:%(reference_case)s' % locals(),
            'task_dep': ['run_demand:%(reference_case)s' % locals()],
            'actions': [(run_emissions_operation, [], {
                'scenario_path': scenario_path,
            })],
        }

def task_run_emissions_mobility():
    """run the emissions mobility script for each reference case"""
    import cea.analysis.lca.mobility

    def run_emissions_mobility(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.analysis.lca.mobility.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_emissions_mobility:%(reference_case)s' % locals(),
            'task_dep': ['run_demand:%(reference_case)s' % locals()],
            'actions': [(run_emissions_mobility, [], {
                'scenario_path': scenario_path,
            })],
        }

def task_run_thermal_network():
    """run the thermal_network for the included reference case"""
    def run_thermal_network():
        import cea.technologies.thermal_network.thermal_network as thermal_network
        from cea.technologies.network_layout.main import NetworkLayout, layout_network

        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        locator = cea.inputlocator.InputLocator(scenario=REFERENCE_CASES['open'])
        config.scenario = locator.scenario
        config.multiprocessing = True
        config.thermal_network.start_t = 100
        config.thermal_network.stop_t = 200
        config.thermal_network.network_type = 'DH'
        config.network_layout.network_type = 'DH'

        # first, create the network layout
        network_layout = NetworkLayout(config.network_layout)
        layout_network(network_layout, locator, [])
        thermal_network.main(config)

    return {
        'name': 'run_thermal_network',
        'task_dep': ['run_demand:open'],
        'actions': [(run_thermal_network, [], {})],
    }

def main(config):
    from doit.api import DoitMain
    from doit.task import dict_to_task
    from doit.cmd_base import TaskLoader

    global _reference_cases
    if not config.test.reference_cases:
        _reference_cases = ['open']
    elif 'all' in config.test.reference_cases:
        _reference_cases = REFERENCE_CASES.keys()
    else:
        _reference_cases = [rc for rc in config.test.reference_cases if rc in REFERENCE_CASES.keys()]

    if 'all' in config.test.tasks:
        tasks = [t[len('task_'):] for t in globals().keys() if t.startswith('task_')]
    else:
        tasks = config.test.tasks

    class CeaTaskLoader(TaskLoader):
        """Add functionality to only run specific tasks"""
        def load_tasks(self, cmd, opt_values, pos_args):
            task_list = []
            for task_name in tasks:
                print('Configuring task %s' % task_name)
                func = globals()['task_' + task_name]
                if isinstance(func(), dict):
                    task_dict = func()
                    task_dict['verbosity'] = config.test.verbosity
                    task_list.append(dict_to_task(task_dict))
                else:
                    for task_dict in func():
                        task_dict['verbosity'] = config.test.verbosity
                        task_list.append(dict_to_task(task_dict))
            task_config = {'verbosity': config.test.verbosity}
            return task_list, task_config

    sys.exit(DoitMain(CeaTaskLoader()).run([]))

if __name__ == '__main__':
    main(cea.config.Configuration())
<commit_msg>Update dodo.py<commit_after>"""
"Makefile" for doit to test the whole package with Jenkins.

This file gets run with the ``cea test`` command as well as by the Jenkins continuous integration server. It runs all
the unit tests in the ``cea/tests/`` folder as well as some of the CEA scripts, to make sure they at least run through.

In order to run reference cases besides the one called "open", you will need to set up authentication to the private
GitHub repository. To do this, you need to create a GitHub [authentication token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/).
Then, create a text file called ``cea_github.auth`` in your home directory (e.g. ``C:\Users\your-user-name`` for Windows systems or equivalent on
POSIX systems, ask your administrator if you don't know what this is). The file should contain two lines, the first being
your GitHub user name the second the authentication token.

The reference cases can be found here: https://github.com/architecture-building-systems/cea-reference-case/archive/master.zip
"""
import os
import shutil
import sys
import zipfile
import tempfile
import cea.inputlocator
import cea.config

import requests

__author__ = "Daren Thomas"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Daren Thomas"]
__license__ = "MIT"
__version__ = "0.1"

__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

REPOSITORY_URL = "https://github.com/architecture-building-systems/cea-reference-case/archive/%s.zip"
REPOSITORY_NAME = "master"

if 'JOB_NAME' in os.environ:
    # this script is being run as part of a Jenkins job
    ARCHIVE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), '%JOB_NAME%/cea-reference-case.zip'))
    REFERENCE_CASE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), '%JOB_NAME%/cea-reference-case'))
else:
    ARCHIVE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), 'cea-reference-case.zip'))
    REFERENCE_CASE_PATH = os.path.expandvars(os.path.join(tempfile.gettempdir(), 'cea-reference-case'))

REFERENCE_CASES = {
    'open': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME, "reference-case-open",
                         "baseline"),
    'zug/baseline': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME, "reference-case-zug",
                                 "baseline"),
    'zurich/baseline': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME,
                                    "reference-case-zurich",
                                    "baseline"),
    'zurich/masterplan': os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME,
                                      "reference-case-zurich",
                                      "masterplan")}

REFERENCE_CASES_DATA = {
    'open': {'databases': 'CH', 'weather': 'Zug-inducity_1990_2010_TMY', 'latitude': 47.1628017306431, 'longitude': 8.31,
             'radiation': 'open.baseline.radiation.csv',
             'properties_surfaces': 'open.baseline.properties_surfaces.csv'},
    'zug/baseline': {'databases': 'CH', 'weather': 'Zug-inducity_1990_2010_TMY', 'latitude': 47.1628017306431, 'longitude': 8.31,
                     'radiation': 'zug.baseline.radiation.csv',
                     'properties_surfaces': 'zug.baseline.properties_surfaces.csv'},
    'zurich/baseline': {'databases': 'CH', 'weather': 'Zuerich-Kloten_1990_2010_TMY', 'latitude': 46.9524055556, 'longitude': 7.43958333333,
                        'radiation': 'hq.baseline.radiation.csv',
                        'properties_surfaces': 'hq.baseline.properties_surfaces.csv'},
    'zurich/masterplan': {'databases': 'CH', 'weather': 'Zuerich-ETHZ_1990-2010_TMY', 'latitude': 46.9524055556, 'longitude': 7.43958333333,
                          'radiation': 'hq.masterplan.radiation.csv',
                          'properties_surfaces': 'hq.masterplan.properties_surfaces.csv'}}

# set to github user and personal access token in main
_user = None
_token = None
_reference_cases = []

def get_github_auth():
    """
    get the username / token for github from a file in the home directory
    called "cea_github.auth". The first line contains the user, the second line
    the personal access token.

    :return: (user, token)
    """
    if _user and _token:
        return _user, _token

    with open(os.path.expanduser('~/cea_github.auth')) as f:
        user, token = map(str.strip, f.readlines())
    return user, token

def task_run_unit_tests():
    """run the unittests"""
    def run_unit_tests():
        import unittest
        import os

        testsuite = unittest.defaultTestLoader.discover(os.path.dirname(__file__))
        result = unittest.TextTestRunner(verbosity=1).run(testsuite)
        return result.wasSuccessful()
    return {
        'name': 'run_unit_tests',
        'actions': [run_unit_tests],
        'task_dep': [],
        'verbosity': 1
    }

def task_download_reference_cases():
    """Download the (current) state of the reference cases"""
    def download_reference_cases():
        if os.path.exists(REFERENCE_CASE_PATH):
            print('Removing folder: %s' % REFERENCE_CASE_PATH)
            shutil.rmtree(REFERENCE_CASE_PATH)
            assert not os.path.exists(REFERENCE_CASE_PATH), 'FAILED to remove folder %s' % REFERENCE_CASE_PATH
        # extract the bundled reference case (we will use this anyways
        import cea.examples
        archive = zipfile.ZipFile(os.path.join(os.path.dirname(cea.examples.__file__), 'reference-case-open.zip'))
        archive.extractall(os.path.join(REFERENCE_CASE_PATH, "cea-reference-case-%s" % REPOSITORY_NAME))

        if len([rc for rc in _reference_cases if rc.lower() != 'open']):
            if os.path.exists(ARCHIVE_PATH):
                os.remove(ARCHIVE_PATH)
            r = requests.get(REPOSITORY_URL % REPOSITORY_NAME, auth=get_github_auth())
            assert r.ok, 'could not download the reference case'
            with open(ARCHIVE_PATH, 'wb') as f:
                f.write(r.content)
            # extract the reference cases to the temp folder
            archive = zipfile.ZipFile(ARCHIVE_PATH)
            archive.extractall(REFERENCE_CASE_PATH)

    return {
        'name' : 'download_reference_cases',
        'actions': [download_reference_cases],
    }

def task_run_data_initializer():
    """Run the data initializer for each reference case"""

    def run_data_initializer(scenario_path, databases_path):
        import cea.datamanagement.data_initializer
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        config.data_initializer.databases_path = databases_path
        cea.datamanagement.data_initializer.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        databases_path = REFERENCE_CASES_DATA[reference_case]['databases']

        yield {
            'name': 'run_data_initializer:%s' % reference_case,
            'task_dep': ['download_reference_cases'],
            'actions': [
                (run_data_initializer, [], {
                    'scenario_path': scenario_path,
                    'databases_path': databases_path})],
        }

def task_run_archetypes_mapper():
    """Run the data helper for each reference case"""
    import cea.datamanagement.archetypes_mapper

    def run_archetypes_mapper(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.datamanagement.archetypes_mapper.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_archetypes_mapper:%s' % reference_case,
            'task_dep': ['run_data_initializer:%s' % reference_case],
            'actions': [
                (run_archetypes_mapper, [], {
                    'scenario_path': scenario_path})],
        }

def task_run_demand():
    """run the demand script for each reference cases and weather file"""
    import cea.demand.demand_main

    def run_demand(scenario_path, weather):
        import cea.config
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path

        # make sure weather file is copied to inputs first
        import cea.datamanagement.weather_helper
        import cea.demand.schedule_maker.schedule_maker
        config.weather_helper.weather = weather
        cea.datamanagement.weather_helper.main(config)
        cea.demand.schedule_maker.schedule_maker.main(config)
        cea.demand.demand_main.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        weather = REFERENCE_CASES_DATA[reference_case]['weather']

        yield {
            'name': 'run_demand:%(reference_case)s' % locals(),
            'task_dep': ['download_reference_cases', 'run_archetypes_mapper:%s' % reference_case],
            'actions': [(run_demand, [], {
                'scenario_path': scenario_path,
                'weather': weather
            })],
        }

def task_run_embodied_energy():
    """Run the embodied energy script for each reference case"""
    import cea.analysis.lca.embodied

    def run_embodied_energy(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        config.emissions.year_to_calculate = 2050
        cea.analysis.lca.embodied.main(config=config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_embodied_energy:%(reference_case)s' % locals(),
            'task_dep': ['download_reference_cases', 'run_archetypes_mapper:%s' % reference_case],
            'actions': [(run_embodied_energy, [scenario_path])],
        }

def task_run_emissions_operation():
    """run the emissions operation script for each reference case"""
    import cea.analysis.lca.operation

    def run_emissions_operation(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.analysis.lca.operation.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_emissions_operation:%(reference_case)s' % locals(),
            'task_dep': ['run_demand:%(reference_case)s' % locals()],
            'actions': [(run_emissions_operation, [], {
                'scenario_path': scenario_path,
            })],
        }

def task_run_emissions_mobility():
    """run the emissions mobility script for each reference case"""
    import cea.analysis.lca.mobility

    def run_emissions_mobility(scenario_path):
        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        config.scenario = scenario_path
        cea.analysis.lca.mobility.main(config)

    for reference_case, scenario_path in REFERENCE_CASES.items():
        if _reference_cases and reference_case not in _reference_cases:
            continue

        yield {
            'name': 'run_emissions_mobility:%(reference_case)s' % locals(),
            'task_dep': ['run_demand:%(reference_case)s' % locals()],
            'actions': [(run_emissions_mobility, [], {
                'scenario_path': scenario_path,
            })],
        }

def task_run_thermal_network():
    """run the thermal_network for the included reference case"""
    def run_thermal_network():
        import cea.technologies.thermal_network.thermal_network as thermal_network
        from cea.technologies.network_layout.main import NetworkLayout, layout_network

        config = cea.config.Configuration(cea.config.DEFAULT_CONFIG)
        locator = cea.inputlocator.InputLocator(scenario=REFERENCE_CASES['open'])
        config.scenario = locator.scenario
        config.multiprocessing = True
        config.thermal_network.start_t = 100
        config.thermal_network.stop_t = 200
        config.thermal_network.network_type = 'DH'
        config.network_layout.network_type = 'DH'

        # first, create the network layout
        network_layout = NetworkLayout(config.network_layout)
        layout_network(network_layout, locator, [])
        thermal_network.main(config)

    return {
        'name': 'run_thermal_network',
        'task_dep': ['run_demand:open'],
        'actions': [(run_thermal_network, [], {})],
    }

def main(config):
    from doit.api import DoitMain
    from doit.task import dict_to_task
    from doit.cmd_base import TaskLoader

    global _reference_cases
    if not config.test.reference_cases:
        _reference_cases = ['open']
    elif 'all' in config.test.reference_cases:
        _reference_cases = REFERENCE_CASES.keys()
    else:
        _reference_cases = [rc for rc in config.test.reference_cases if rc in REFERENCE_CASES.keys()]

    if 'all' in config.test.tasks:
        tasks = [t[len('task_'):] for t in globals().keys() if t.startswith('task_')]
    else:
        tasks = config.test.tasks

    class CeaTaskLoader(TaskLoader):
        """Add functionality to only run specific tasks"""
        def load_tasks(self, cmd, opt_values, pos_args):
            task_list = []
            for task_name in tasks:
                print('Configuring task %s' % task_name)
                func = globals()['task_' + task_name]
                if isinstance(func(), dict):
                    task_dict = func()
                    task_dict['verbosity'] = config.test.verbosity
                    task_list.append(dict_to_task(task_dict))
                else:
                    for task_dict in func():
                        task_dict['verbosity'] = config.test.verbosity
                        task_list.append(dict_to_task(task_dict))
            task_config = {'verbosity': config.test.verbosity}
            return task_list, task_config

    sys.exit(DoitMain(CeaTaskLoader()).run([]))

if __name__ == '__main__':
    main(cea.config.Configuration())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2018 SUNET
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from __future__ import absolute_import

import imp
import time
from saml2.config import SPConfig
from pwgen import pwgen
from eduid_common.session import session

from eduid_common.api.utils import urlappend

import logging
logger = logging.getLogger(__name__)

def get_saml2_config(module_path):

    module = imp.load_source('saml2_settings', module_path)

    conf = SPConfig()
    conf.load(module.SAML_CONFIG)
    return conf

def get_location(http_info):
    """Extract the redirect URL from a pysaml2 http_info object"""
    assert 'headers' in http_info
    headers = http_info['headers']

    assert len(headers) == 1
    header_name, header_value = headers[0]
    assert header_name == 'Location'
    return header_value

def get_saml_attribute(session_info, attr_name):
    """
    Get value from a SAML attribute received from the SAML IdP.

    session_info is a pysaml2 response.session_info(). This is a dictionary like
        {'mail': ['user@example.edu'],
         'eduPersonPrincipalName': ['gadaj-fifib@idp.example.edu']
      }

    :param session_info: SAML attributes received by pysaml2 client.
    :param attr_name: The attribute to look up
    :returns: Attribute values

    :type session_info: dict()
    :type attr_name: string()
    :rtype: [string()]
    """
    if not 'ava' in session_info:
        raise ValueError('SAML attributes (ava) not found in session_info')

    attributes = session_info['ava']

    logger.debug('SAML attributes received: %s' % attributes)

    attr_name = attr_name.lower()
    # Look for the canonicalized attribute in the SAML assertion attributes
    for saml_attr, local_fields in attributes.items():
        if saml_attr.lower() == attr_name:
            return attributes[saml_attr]

def no_authn_views(app, paths):
    """
    :param app: Flask app
    :type app: flask.Flask
    :param paths: Paths that does not require authentication
    :type paths: list

    :return: Flask app
    :rtype: flask.Flask
    """
    app_root = app.config.get('APPLICATION_ROOT')
    if app_root is None:
        app_root = ''
    for path in paths:
        no_auth_regex = '^{!s}$'.format(urlappend(app_root, path))
        if no_auth_regex not in app.config['NO_AUTHN_URLS']:
            app.config['NO_AUTHN_URLS'].append(no_auth_regex)
    return app

def generate_password(length=12):
    return pwgen(int(length), no_capitalize=True, no_symbols=True)

def check_implicit_login():
    """
    Check that the user, though not properly authenticated, has been recognized
    by some app with access to the shared session
    Must be called within a request context.

    Used after signup or for idp actions.

    :return: The eppn in case the check is successful, None otherwise
    """
    eppn = session.common.eppn
    if eppn is None:
        eppn = session.get('user_eppn', None)
    timestamp = session.implicit_login.ts
    logger.debug('Trying to authenticate user {} with timestamp {!r}'.format(eppn, timestamp))
    # check that the eppn and timestamp have been set in the session
    if eppn is None or timestamp is None:
        return None
    # check timestamp to make sure it is within -300..900
    now = int(time.time())
    ts = int(timestamp, 16)
    if (ts < now - 300) or (ts > now + 900):
        logger.debug('Auth token timestamp {} out of bounds ({} seconds from {})'.format(
            timestamp, ts - now, now))
        return None
    return eppn

<commit_msg>backwardscompat for token logins<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2018 SUNET
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from __future__ import absolute_import

import imp
import time
from saml2.config import SPConfig
from pwgen import pwgen
from eduid_common.session import session

from eduid_common.api.utils import urlappend

import logging
logger = logging.getLogger(__name__)

def get_saml2_config(module_path):

    module = imp.load_source('saml2_settings', module_path)

    conf = SPConfig()
    conf.load(module.SAML_CONFIG)
    return conf

def get_location(http_info):
    """Extract the redirect URL from a pysaml2 http_info object"""
    assert 'headers' in http_info
    headers = http_info['headers']

    assert len(headers) == 1
    header_name, header_value = headers[0]
    assert header_name == 'Location'
    return header_value

def get_saml_attribute(session_info, attr_name):
    """
    Get value from a SAML attribute received from the SAML IdP.

    session_info is a pysaml2 response.session_info(). This is a dictionary like
        {'mail': ['user@example.edu'],
         'eduPersonPrincipalName': ['gadaj-fifib@idp.example.edu']
      }

    :param session_info: SAML attributes received by pysaml2 client.
    :param attr_name: The attribute to look up
    :returns: Attribute values

    :type session_info: dict()
    :type attr_name: string()
    :rtype: [string()]
    """
    if not 'ava' in session_info:
        raise ValueError('SAML attributes (ava) not found in session_info')

    attributes = session_info['ava']

    logger.debug('SAML attributes received: %s' % attributes)

    attr_name = attr_name.lower()
    # Look for the canonicalized attribute in the SAML assertion attributes
    for saml_attr, local_fields in attributes.items():
        if saml_attr.lower() == attr_name:
            return attributes[saml_attr]

def no_authn_views(app, paths):
    """
    :param app: Flask app
    :type app: flask.Flask
    :param paths: Paths that does not require authentication
    :type paths: list

    :return: Flask app
    :rtype: flask.Flask
    """
    app_root = app.config.get('APPLICATION_ROOT')
    if app_root is None:
        app_root = ''
    for path in paths:
        no_auth_regex = '^{!s}$'.format(urlappend(app_root, path))
        if no_auth_regex not in app.config['NO_AUTHN_URLS']:
            app.config['NO_AUTHN_URLS'].append(no_auth_regex)
    return app

def generate_password(length=12):
    return pwgen(int(length), no_capitalize=True, no_symbols=True)

def check_implicit_login():
    """
    Check that the user, though not properly authenticated, has been recognized
    by some app with access to the shared session
    Must be called within a request context.

    Used after signup or for idp actions.

    :return: The eppn in case the check is successful, None otherwise
    """
    eppn = session.common.eppn
    if eppn is None:
        eppn = session.get('user_eppn', None)
    timestamp = session.implicit_login.ts
    logger.debug('Trying to authenticate user {} with timestamp {!r}'.format(eppn, timestamp))
    # check that the eppn and timestamp have been set in the session
    if eppn is None or timestamp is None:
        return None
    # check timestamp to make sure it is within -300..900
    now = int(time.time())
    ts = int(timestamp, 16)
    if (ts < now - 300) or (ts > now + 900):
        logger.debug('Auth token timestamp {} out of bounds ({} seconds from {})'.format(
            timestamp, ts - now, now))
        return None
    return eppn

# XXX the code below is deprecated and only kept fr the transition to implicit
# logins with data in the session. Please remove after the transition

import six
from hashlib import sha256
from nacl import secret, encoding
import nacl.exceptions

def verify_auth_token(shared_key, eppn, token, nonce, timestamp, usage, generator=sha256):
    """
    Authenticate a user with a token.
    Used after signup or for idp actions.
    Authentication is done using a shared key in the configuration of the
    authn and signup applications or another shared key in the configuration of idp and actions.
    :param shared_key: Applications shared key
    :param eppn: the identifier of the user as string
    :param token: authentication token as string
    :param nonce: a public nonce for this authentication request as string
    :param timestamp: unixtime of signup application as hex string
    :param usage: The intended usage of the token, to safeguard against tokens being maliciously
                  sent to another token consumer than intended
    :param generator: hash function to use (default: SHA-256)
    :return: bool, True on valid authentication
    """
    logger.debug('Trying to authenticate user {} with auth token {!r}'.format(eppn, token))
    if six.PY2:
        shared_key = shared_key.encode('ascii')

    # check timestamp to make sure it is within -300..900 seconds from now
    now = int(time.time())
    ts = int(timestamp, 16)
    if (ts < now - 300) or (ts > now + 900):
        logger.debug('Auth token timestamp {} out of bounds ({} seconds from {})'.format(
            timestamp, ts - now, now))
        return False

    # try to open secret box
    try:
        box = secret.SecretBox(encoding.URLSafeBase64Encoder.decode(shared_key))
        plaintext = box.decrypt(token.encode('ascii'), encoder=encoding.URLSafeBase64Encoder)
        expected = '{}|{}|{}'.format(usage, timestamp, eppn).encode('ascii')
        logger.debug('Comparing plaintext {!r} with expected {!r}'.format(plaintext, expected))
        return plaintext == expected
    except (LookupError,  ValueError, nacl.exceptions.CryptoError) as e:
        logger.debug('Secretbox decryption failed, error: ' + str(e))

    # Fall back to HMAC validation

    # verify there is a long enough nonce
    if len(nonce) < 16:
        logger.warning('Auth token nonce {} too short'.format(nonce))
        return False

    # verify token format
    data = u'{0}|{1}|{2}|{3}'.format(shared_key, eppn, nonce, timestamp)
    hashed = generator(data.encode('ascii'))
    expected = hashed.hexdigest()
    if len(expected) != len(token):
        logger.warning('Auth token bad length')
        return False

    # constant time comparision of the hash, courtesy of
    # http://rdist.root.org/2009/05/28/timing-attack-in-google-keyczar-library/
    result = 0
    if isinstance(expected, six.binary_type):
        expected = expected.decode('ascii')
    if isinstance(token, six.binary_type):
        token = token.decode('ascii')
    for x, y in zip(expected, token):
        result |= ord(x) ^ ord(str(y))
    logger.debug('Auth token match result: {}'.format(result == 0))
    return result == 0
<|endoftext|>


<commit_before>"""
  This example illustrates the synthesis of a sound figure.

  The sound figure is defined by a grayscale PNG image. Various examples
  are located in the directory figures.
"""

import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import sfs

# parameters
dx = 0.10  # secondary source distance
N = 60  # number of secondary sources
pw_angle = [90, 45]  # traveling direction of plane wave
f = 2000  # frequency

# angular frequency
omega = 2 * np.pi * f

# normal vector of plane wave
npw = sfs.util.direction_vector(np.radians(pw_angle[0]), np.radians(pw_angle[1]))

# spatial grid
x = sfs.util.strict_arange(-3, 3, 0.02, endpoint=True)
y = sfs.util.strict_arange(-3, 3, 0.02, endpoint=True)
grid = np.meshgrid(x, y, 0, sparse=True)

# --------------------------------------------------------------------------------
# main
# --------------------------------------------------------------------------------

# get secondary source positions
x0, n0, a0 = sfs.array.cube(N, dx)

# driving function for sound figure
figure = np.array(Image.open('figures/tree.png'))  # read image from file
figure = np.rot90(figure)  # turn 0deg to the top
d = sfs.mono.soundfigure.wfs_3d_pw(omega, x0, n0, figure, npw=npw)

# compute synthesized sound field
p = sfs.mono.synthesized.generic(omega, x0, n0, d * a0, grid,
                                 source=sfs.mono.source.point)

# plot and save synthesized sound field
plt.figure(figsize=(10, 10))
sfs.plot.soundfield(2.5e-9 * p, grid, colorbar=False, cmap=plt.cm.BrBG)
plt.title('Synthesized Sound Field')
plt.savefig('soundfigure.png')

# plot and save level of synthesized sound field
plt.figure(figsize=(12.5, 12.5))
Lp = 20*np.log10(abs(p) / abs(p[len(x)//2, len(y)//2]))
plt.imshow(Lp, origin='lower',
           extent=[min(x), max(x), min(y), max(y)],
           vmin=-50, vmax=0, aspect='equal')
plt.title('Level of Synthesized Sound Field')
plt.xlabel('x (m)')
plt.ylabel('y (m)')
cbar = plt.colorbar(shrink=0.8)
cbar.set_label('dB')
plt.savefig('soundfigure_level.png')
<commit_msg>Update soundfigures example<commit_after>"""This example illustrates the synthesis of a sound figure.

The sound figure is defined by a grayscale PNG image. Various example
images are located in the "figures/" directory.

"""
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import sfs

dx = 0.10  # secondary source distance
N = 60  # number of secondary sources
pw_angle = [90, 45]  # traveling direction of plane wave
f = 2000  # frequency

# angular frequency
omega = 2 * np.pi * f

# normal vector of plane wave
npw = sfs.util.direction_vector(*np.radians(pw_angle))

# spatial grid
grid = sfs.util.xyz_grid([-3, 3], [-3, 3], 0, spacing=0.02)

# get secondary source positions
x0, n0, a0 = sfs.array.cube(N, dx)

# driving function for sound figure
figure = np.array(Image.open('figures/tree.png'))  # read image from file
figure = np.rot90(figure)  # turn 0deg to the top
d = sfs.mono.soundfigure.wfs_3d_pw(omega, x0, n0, figure, npw=npw)

# compute synthesized sound field
p = sfs.mono.synthesized.generic(omega, x0, n0, d * a0, grid,
                                 source=sfs.mono.source.point)

# plot and save synthesized sound field
plt.figure(figsize=(10, 10))
sfs.plot.soundfield(p, grid, xnorm=[0, -2.2, 0], cmap='BrBG', colorbar=False,
                    vmin=-1, vmax=1)
plt.title('Synthesized Sound Field')
plt.savefig('soundfigure.png')

# plot and save level of synthesized sound field
plt.figure(figsize=(12.5, 12.5))
im = sfs.plot.level(p, grid, xnorm=[0, -2.2, 0], colorbar=False,
                    vmin=-50, vmax=0)
plt.title('Level of Synthesized Sound Field')
cbar = plt.colorbar(im, label='dB', shrink=0.8)
plt.savefig('soundfigure_level.png')
<|endoftext|>


<commit_before>'''
Created on Mar 15, 2011

@author: jsalvatier
'''
import numpy as np
from core import *

__all__ = ['NpTrace', 'MultiTrace']

class NpTrace(object):
    """
    encapsulates the recording of a process chain
    """
    def __init__(self, vars):
        self.f = compilef(vars)
        self.vars = vars
        self.varnames =map(str, vars)
        self.samples = dict((v, ListArray()) for v in self.varnames)

    def record(self, point):
        """
        records the position of a chain at a certain point in time
        """
        for var, value in zip(self.varnames, self.f(point)):
            self.samples[var].append(value)
        return self

    def __getitem__(self, key):
        try :
            return self.point(key)
        except ValueError:
            pass
        except TypeError:
            pass
        return self.samples[str(key)].value

    def point(self, index):
        return dict((k, v.value[index]) for (k,v) in self.samples.iteritems())

def stats(trace, alpha=0.05, start=0, batches=100, chain=None, quantiles=(2.5, 25, 50, 75, 97.5)):
    """
    Generate posterior statistics from trace.

    :Parameters:
    name : string
      The name of the tallyable object.

    alpha : float
      The alpha level for generating posterior intervals. Defaults to
      0.05.

    start : int
      The starting index from which to summarize (each) chain. Defaults
      to zero.

    batches : int
      Batch size for calculating standard deviation for non-independent
      samples. Defaults to 100.

    chain : int
      The index for which chain to summarize. Defaults to None (all
      chains).

    quantiles : tuple or list
      The desired quantiles to be calculated. Defaults to (2.5, 25, 50, 75, 97.5).
    """

    def var_stats(trace):

            x = trace[start:]

            n = len(x)
            if not n:
                print('Cannot generate statistics for zero-length trace.')
                return

            stats =  {
                'n': n,
                'standard deviation': x.std(0),
                'mean': x.mean(0),
                '%s%s HPD interval' % (int(100*(1-alpha)),'%'): hpd(x, alpha),
                'mc error': batchsd(x, batches),
                'quantiles': calc_quantiles(x, qlist=quantiles)
            }

            return stats

    try:
        # For entire Trace object
        stats = {}
        for v in trace.varnames:
            try:
                stats[v] = var_stats(trace[v])
            except:
                print('Could not generate output statistics for ' + v)
        return stats

    except AttributeError:
        # For trace of single variable
        return var_stats(trace)

def summary(trace, alpha=0.05, start=0, batches=100, roundto=3):
    """
    Generate a pretty-printed summary of the node.

    :Parameters:
    alpha : float
      The alpha level for generating posterior intervals. Defaults to
      0.05.

    start : int
      The starting index from which to summarize (each) chain. Defaults
      to zero.

    batches : int
      Batch size for calculating standard deviation for non-independent
      samples. Defaults to 100.

    roundto : int
      The number of digits to round posterior statistics.
    """

    def var_summary(statdict):

        size = np.size(statdict['mean'])

        # Initialize buffer
        buffer = []

        # Index to interval label
        iindex = [key.split()[-1] for key in statdict.keys()].index('interval')
        interval = statdict.keys()[iindex]

        # Print basic stats
        buffer += ['Mean             SD               MC Error        %s' % interval]
        buffer += ['-'*len(buffer[-1])]

        indices = range(size)
        if len(indices)==1:
            indices = [None]

        for index in indices:
            # Extract statistics and convert to string
            m = str(round(statdict['mean'][index], roundto))
            sd = str(round(statdict['standard deviation'][index], roundto))
            mce = str(round(statdict['mc error'][index], roundto))
            hpd = str(statdict[interval][index].squeeze().round(roundto))

            # Build up string buffer of values
            valstr = m
            valstr += ' '*(17-len(m)) + sd
            valstr += ' '*(17-len(sd)) + mce
            valstr += ' '*(len(buffer[-1]) - len(valstr) - len(hpd)) + hpd

            buffer += [valstr]

        buffer += ['']*2

        # Print quantiles
        buffer += ['Posterior quantiles:','']

        buffer += ['2.5             25              50              75             97.5']
        buffer += [' |---------------|===============|===============|---------------|']

        for index in indices:
            quantile_str = ''
            for i,q in enumerate((2.5, 25, 50, 75, 97.5)):
                qstr = str(round(statdict['quantiles'][q][index], roundto))
                quantile_str += qstr + ' '*(17-i-len(qstr))
            buffer += [quantile_str.strip()]

        buffer += ['']

        print('\t' + '\n\t'.join(buffer))

    # Calculate statistics
    s = stats(trace, alpha=alpha, start=start, batches=batches)

    try:
        for var in trace.varnames:

            try:
                vstats = s[var]

                print('\n%s:' % var)
                print(' ')

                var_summary(vstats)
            except KeyError:

                pass

    except AttributeError:

        var_summary(s)

class ListArray(object):
    def __init__(self):
        self.vals = []

    @property
    def value(self):
        if len(self.vals) > 1:
            self.vals = [np.concatenate(self.vals, axis =0)]
        return self.vals[0]

    def append(self, v):
        self.vals.append(v[np.newaxis])

class MultiTrace(object):
    def __init__(self, traces, vars = None):
        try :
            self.traces = list(traces)
        except TypeError:
            if vars is None:
                raise ValueError("vars can't be None if trace count specified")
            self.traces = [NpTrace(vars) for _ in xrange(traces)]

    def __getitem__(self, key):
        return [h[key] for h in self.traces]
    def point(self, index):
        return [h.point(index) for h in self.traces]

    def combined(self):
        h = NpTrace(self.traces[0].vars)
        for k in self.traces[0].samples:
            h.samples[k].vals = [s[k] for s in self.traces]
        return h

def calc_min_interval(x, alpha):
    """Internal method to determine the minimum interval of
    a given width

    Assumes that x is sorted numpy array.
    """

    n = len(x)
    cred_mass = 1.0-alpha

    interval_idx_inc = int(np.floor( cred_mass*n ))
    n_intervals = n - interval_idx_inc
    interval_width = x[interval_idx_inc:] - x[:n_intervals]

    if len(interval_width)==0:
        print('Too few elements for interval calculation')
        return [None,None]

    min_idx = np.argmin(interval_width)
    hdi_min = x[min_idx]
    hdi_max = x[min_idx+interval_idx_inc]
    return hdi_min, hdi_max

def hpd(x, alpha):
    """Calculate highest posterior density (HPD) of array for given alpha. The HPD is the
    minimum width Bayesian credible interval (BCI).

    :Arguments:
      x : Numpy array
          An array containing MCMC samples
      alpha : float
          Desired probability of type I error

    """

    # Make a copy of trace
    x = x.copy()

    # For multivariate node
    if x.ndim>1:

        # Transpose first, then sort
        tx = tr(x, range(x.ndim)[1:]+[0])
        dims = np.shape(tx)

        # Container list for intervals
        intervals = np.resize(0.0, dims[:-1]+(2,))

        for index in make_indices(dims[:-1]):

            try:
                index = tuple(index)
            except TypeError:
                pass

            # Sort trace
            sx = np.sort(tx[index])

            # Append to list
            intervals[index] = calc_min_interval(sx, alpha)

        # Transpose back before returning
        return np.array(intervals)

    else:
        # Sort univariate node
        sx = np.sort(x)

        return np.array(calc_min_interval(sx, alpha))

def batchsd(trace, batches=5):
    """
    Calculates the simulation standard error, accounting for non-independent
    samples. The trace is divided into batches, and the standard deviation of
    the batch means is calculated.
    """

    if len(np.shape(trace)) > 1:

        dims = np.shape(trace)
        #ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:]))))
        ttrace = np.transpose([t.ravel() for t in trace])

        return np.reshape([batchsd(t, batches) for t in ttrace], dims[1:])

    else:
        if batches == 1: return np.std(trace)/np.sqrt(len(trace))

        try:
            batched_traces = np.resize(trace, (batches, len(trace)/batches))
        except ValueError:
            # If batches do not divide evenly, trim excess samples
            resid = len(trace) % batches
            batched_traces = np.resize(trace[:-resid], (batches, len(trace)/batches))

        means = np.mean(batched_traces, 1)

        return np.std(means)/np.sqrt(batches)

def calc_quantiles(x, qlist=(2.5, 25, 50, 75, 97.5)):
    """Returns a dictionary of requested quantiles from array

    :Arguments:
      x : Numpy array
          An array containing MCMC samples
      qlist : tuple or list
          A list of desired quantiles (defaults to (2.5, 25, 50, 75, 97.5))

    """

    # Make a copy of trace
    x = x.copy()

    # For multivariate node
    if x.ndim>1:
        # Transpose first, then sort, then transpose back
        sx = np.sort(x.T).T
    else:
        # Sort univariate node
        sx = np.sort(x)

    try:
        # Generate specified quantiles
        quants = [sx[int(len(sx)*q/100.0)] for q in qlist]

        return dict(zip(qlist, quants))

    except IndexError:
        print("Too few elements for quantile calculation")<commit_msg>calc_interval now throws ValueError when there are insufficient elements.<commit_after>'''
Created on Mar 15, 2011

@author: jsalvatier
'''
import numpy as np
from core import *

__all__ = ['NpTrace', 'MultiTrace']

class NpTrace(object):
    """
    encapsulates the recording of a process chain
    """
    def __init__(self, vars):
        self.f = compilef(vars)
        self.vars = vars
        self.varnames =map(str, vars)
        self.samples = dict((v, ListArray()) for v in self.varnames)

    def record(self, point):
        """
        records the position of a chain at a certain point in time
        """
        for var, value in zip(self.varnames, self.f(point)):
            self.samples[var].append(value)
        return self

    def __getitem__(self, key):
        try :
            return self.point(key)
        except ValueError:
            pass
        except TypeError:
            pass
        return self.samples[str(key)].value

    def point(self, index):
        return dict((k, v.value[index]) for (k,v) in self.samples.iteritems())

def stats(trace, alpha=0.05, start=0, batches=100, chain=None, quantiles=(2.5, 25, 50, 75, 97.5)):
    """
    Generate posterior statistics from trace.

    :Parameters:
    name : string
      The name of the tallyable object.

    alpha : float
      The alpha level for generating posterior intervals. Defaults to
      0.05.

    start : int
      The starting index from which to summarize (each) chain. Defaults
      to zero.

    batches : int
      Batch size for calculating standard deviation for non-independent
      samples. Defaults to 100.

    chain : int
      The index for which chain to summarize. Defaults to None (all
      chains).

    quantiles : tuple or list
      The desired quantiles to be calculated. Defaults to (2.5, 25, 50, 75, 97.5).
    """

    def var_stats(trace):

            x = trace[start:]

            n = len(x)
            if not n:
                print('Cannot generate statistics for zero-length trace.')
                return

            stats =  {
                'n': n,
                'standard deviation': x.std(0),
                'mean': x.mean(0),
                '%s%s HPD interval' % (int(100*(1-alpha)),'%'): hpd(x, alpha),
                'mc error': batchsd(x, batches),
                'quantiles': calc_quantiles(x, qlist=quantiles)
            }

            return stats

    try:
        # For entire Trace object
        stats = {}
        for v in trace.varnames:
            try:
                stats[v] = var_stats(trace[v])
            except:
                print('Could not generate output statistics for ' + v)
        return stats

    except AttributeError:
        # For trace of single variable
        return var_stats(trace)

def summary(trace, alpha=0.05, start=0, batches=100, roundto=3):
    """
    Generate a pretty-printed summary of the node.

    :Parameters:
    alpha : float
      The alpha level for generating posterior intervals. Defaults to
      0.05.

    start : int
      The starting index from which to summarize (each) chain. Defaults
      to zero.

    batches : int
      Batch size for calculating standard deviation for non-independent
      samples. Defaults to 100.

    roundto : int
      The number of digits to round posterior statistics.
    """

    def var_summary(statdict):

        size = np.size(statdict['mean'])

        # Initialize buffer
        buffer = []

        # Index to interval label
        iindex = [key.split()[-1] for key in statdict.keys()].index('interval')
        interval = statdict.keys()[iindex]

        # Print basic stats
        buffer += ['Mean             SD               MC Error        %s' % interval]
        buffer += ['-'*len(buffer[-1])]

        indices = range(size)
        if len(indices)==1:
            indices = [None]

        for index in indices:
            # Extract statistics and convert to string
            m = str(round(statdict['mean'][index], roundto))
            sd = str(round(statdict['standard deviation'][index], roundto))
            mce = str(round(statdict['mc error'][index], roundto))
            hpd = str(statdict[interval][index].squeeze().round(roundto))

            # Build up string buffer of values
            valstr = m
            valstr += ' '*(17-len(m)) + sd
            valstr += ' '*(17-len(sd)) + mce
            valstr += ' '*(len(buffer[-1]) - len(valstr) - len(hpd)) + hpd

            buffer += [valstr]

        buffer += ['']*2

        # Print quantiles
        buffer += ['Posterior quantiles:','']

        buffer += ['2.5             25              50              75             97.5']
        buffer += [' |---------------|===============|===============|---------------|']

        for index in indices:
            quantile_str = ''
            for i,q in enumerate((2.5, 25, 50, 75, 97.5)):
                qstr = str(round(statdict['quantiles'][q][index], roundto))
                quantile_str += qstr + ' '*(17-i-len(qstr))
            buffer += [quantile_str.strip()]

        buffer += ['']

        print('\t' + '\n\t'.join(buffer))

    # Calculate statistics
    s = stats(trace, alpha=alpha, start=start, batches=batches)

    try:
        for var in trace.varnames:

            try:
                vstats = s[var]

                print('\n%s:' % var)
                print(' ')

                var_summary(vstats)
            except KeyError:

                pass

    except AttributeError:

        var_summary(s)

class ListArray(object):
    def __init__(self):
        self.vals = []

    @property
    def value(self):
        if len(self.vals) > 1:
            self.vals = [np.concatenate(self.vals, axis =0)]
        return self.vals[0]

    def append(self, v):
        self.vals.append(v[np.newaxis])

class MultiTrace(object):
    def __init__(self, traces, vars = None):
        try :
            self.traces = list(traces)
        except TypeError:
            if vars is None:
                raise ValueError("vars can't be None if trace count specified")
            self.traces = [NpTrace(vars) for _ in xrange(traces)]

    def __getitem__(self, key):
        return [h[key] for h in self.traces]
    def point(self, index):
        return [h.point(index) for h in self.traces]

    def combined(self):
        h = NpTrace(self.traces[0].vars)
        for k in self.traces[0].samples:
            h.samples[k].vals = [s[k] for s in self.traces]
        return h

def calc_min_interval(x, alpha):
    """Internal method to determine the minimum interval of
    a given width

    Assumes that x is sorted numpy array.
    """

    n = len(x)
    cred_mass = 1.0-alpha

    interval_idx_inc = int(np.floor( cred_mass*n ))
    n_intervals = n - interval_idx_inc
    interval_width = x[interval_idx_inc:] - x[:n_intervals]

    if len(interval_width)==0:
        raise ValueError('Too few elements for interval calculation')

    min_idx = np.argmin(interval_width)
    hdi_min = x[min_idx]
    hdi_max = x[min_idx+interval_idx_inc]
    return hdi_min, hdi_max

def hpd(x, alpha):
    """Calculate highest posterior density (HPD) of array for given alpha. The HPD is the
    minimum width Bayesian credible interval (BCI).

    :Arguments:
      x : Numpy array
          An array containing MCMC samples
      alpha : float
          Desired probability of type I error

    """

    # Make a copy of trace
    x = x.copy()

    # For multivariate node
    if x.ndim>1:

        # Transpose first, then sort
        tx = tr(x, range(x.ndim)[1:]+[0])
        dims = np.shape(tx)

        # Container list for intervals
        intervals = np.resize(0.0, dims[:-1]+(2,))

        for index in make_indices(dims[:-1]):

            try:
                index = tuple(index)
            except TypeError:
                pass

            # Sort trace
            sx = np.sort(tx[index])

            # Append to list
            intervals[index] = calc_min_interval(sx, alpha)

        # Transpose back before returning
        return np.array(intervals)

    else:
        # Sort univariate node
        sx = np.sort(x)

        return np.array(calc_min_interval(sx, alpha))

def batchsd(trace, batches=5):
    """
    Calculates the simulation standard error, accounting for non-independent
    samples. The trace is divided into batches, and the standard deviation of
    the batch means is calculated.
    """

    if len(np.shape(trace)) > 1:

        dims = np.shape(trace)
        #ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:]))))
        ttrace = np.transpose([t.ravel() for t in trace])

        return np.reshape([batchsd(t, batches) for t in ttrace], dims[1:])

    else:
        if batches == 1: return np.std(trace)/np.sqrt(len(trace))

        try:
            batched_traces = np.resize(trace, (batches, len(trace)/batches))
        except ValueError:
            # If batches do not divide evenly, trim excess samples
            resid = len(trace) % batches
            batched_traces = np.resize(trace[:-resid], (batches, len(trace)/batches))

        means = np.mean(batched_traces, 1)

        return np.std(means)/np.sqrt(batches)

def calc_quantiles(x, qlist=(2.5, 25, 50, 75, 97.5)):
    """Returns a dictionary of requested quantiles from array

    :Arguments:
      x : Numpy array
          An array containing MCMC samples
      qlist : tuple or list
          A list of desired quantiles (defaults to (2.5, 25, 50, 75, 97.5))

    """

    # Make a copy of trace
    x = x.copy()

    # For multivariate node
    if x.ndim>1:
        # Transpose first, then sort, then transpose back
        sx = np.sort(x.T).T
    else:
        # Sort univariate node
        sx = np.sort(x)

    try:
        # Generate specified quantiles
        quants = [sx[int(len(sx)*q/100.0)] for q in qlist]

        return dict(zip(qlist, quants))

    except IndexError:
        print("Too few elements for quantile calculation")<|endoftext|>


<commit_before>"""
handler to flush stats to [Netuitive](http://www.netuitive.com)
# Dependencies

# Configuration
Enable handler

  * handlers = diamond.handler.netuitive_cloud.NetuitiveHandler

  * url = https://api.app.netuitive.com/ingest/infrastructure
  * api_key = NETUITIVE_API_KEY
  * tags = tag1:tag1val, tag2:tag2val

"""

from Handler import Handler
import logging
import re
import platform
import datetime
import time
import os
import json
import urllib2
from diamond.util import get_diamond_version

try:
    import psutil
except ImportError:
    psutil = None

try:
    import netuitive
except ImportError:
    netuitive = None

try:
    import docker
except ImportError:
    docker = None

def get_human_readable_size(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return(("%3.2f %s%s" % (num, unit, suffix)).strip())
        num /= 1024.0
    return(("%.2f %s%s" % (num, 'Y', suffix)).strip())

def check_lsb():

    if os.path.isfile("/etc/lsb-release"):
        try:
            _distributor_id_file_re = re.compile(
                "(?:DISTRIB_ID\s*=)\s*(.*)", re.I)
            _release_file_re = re.compile(
                "(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I)
            _codename_file_re = re.compile(
                "(?:DISTRIB_CODENAME\s*=)\s*(.*)", re.I)
            with open("/etc/lsb-release", "rU") as etclsbrel:
                for line in etclsbrel:
                    m = _distributor_id_file_re.search(line)
                    if m:
                        _u_distname = m.group(1).strip()
                    m = _release_file_re.search(line)
                    if m:
                        _u_version = m.group(1).strip()
                    m = _codename_file_re.search(line)
                    if m:
                        _u_id = m.group(1).strip()
                if _u_distname and _u_version:
                    return (_u_distname, _u_version, _u_id)
        except Exception as e:
            logging.debug(e)
            return(None)

    else:
        return(None)

class NetuitiveHandler(Handler):

    def __init__(self, config=None):
        """
        initialize Netuitive api and populate agent host metadata
        """

        if not netuitive:
            self.log.error('netuitive import failed. Handler disabled')
            self.enabled = False
            return

        try:
            Handler.__init__(self, config)

            logging.debug("initialize Netuitive handler")

            self.version = self._get_version()
            self.api = netuitive.Client(self.config['url'], self.config[
                                        'api_key'], self.version)

            self.element = netuitive.Element(
                location=self.config.get('location'))

            self.batch_size = int(self.config['batch'])

            self.max_backlog_multiplier = int(
                self.config['max_backlog_multiplier'])

            self.trim_backlog_multiplier = int(
                self.config['trim_backlog_multiplier'])

            self._add_sys_meta()
            self._add_aws_meta()
            self._add_docker_meta()
            self._add_config_tags()
            self._add_config_relations()

            self.flush_time = 0

            logging.debug(self.config)

        except Exception as e:
            logging.exception('NetuitiveHandler: init - %s', str(e))

    def get_default_config_help(self):
        """
        Returns the help text for the configuration options for this handler
        """
        config = super(NetuitiveHandler, self).get_default_config_help()

        config.update({
            'url': 'NetuitiveCloud url to send data to',
            'api_key': 'Datasource api key',
            'tags': 'Netuitive Tags',
            'relations': 'Netuitive child Element',
            'location': 'the location of this Element',
            'batch': 'How many to store before sending to the graphite server',
            'max_backlog_multiplier': 'how many batches to store before trimming',
            'trim_backlog_multiplier': 'Trim down how many batches',
        })
        return config

    def get_default_config(self):
        """
        Return the default config for the handler
        """

        config = super(NetuitiveHandler, self).get_default_config()

        config.update({
            'url': 'https://api.app.netuitive.com/ingest/infrastructure',
            'api_key': 'apikey',
            'tags': None,
            'relations': None,
            'location': None,
            'batch': 100,
            'max_backlog_multiplier': 5,
            'trim_backlog_multiplier': 4,
        })
        return config

    def __del__(self):
        pass

    def _get_version(self):
        """
        Return version string
        """

        ret = 'Diamond_' + get_diamond_version().rstrip()
        if os.path.isfile('/opt/netuitive-agent/version-manifest.txt'):
            with open('/opt/netuitive-agent/version-manifest.txt', 'r') as f:
                v = f.readline()

            f.close()

            ret = v.replace(' ', '_').lower().rstrip()

        return(ret)

    def _add_sys_meta(self):
        try:

            self.element.add_attribute('platform', platform.system())
            self.element.add_attribute('agent', self.version)

            if psutil:
                self.element.add_attribute('cpus', psutil.cpu_count())
                mem = psutil.virtual_memory()
                self.element.add_attribute(
                    'ram', get_human_readable_size(mem.total))
                self.element.add_attribute('ram bytes', mem.total)
                self.element.add_attribute(
                    'boottime', str(datetime.datetime.fromtimestamp(psutil.boot_time())))

            if platform.system().startswith('Linux'):
                if check_lsb() is None:
                    supported_dists = platform._supported_dists + ('system',)
                    dist = platform.linux_distribution(
                        supported_dists=supported_dists)

                else:
                    dist = check_lsb()

                self.element.add_attribute('distribution_name', str(dist[0]))
                self.element.add_attribute(
                    'distribution_version', str(dist[1]))
                if dist[2] != '':
                    self.element.add_attribute('distribution_id', str(dist[2]))

        except Exception as e:
            logging.debug(e)
            pass

    def _add_docker_meta(self):
        if docker:
            try:

                cc = docker.Client(
                    base_url='unix://var/run/docker.sock', version='auto')
                dockerver = cc.version()

                for k, v in dockerver.items():
                    logging.debug('docker_' + k + ' = ' + v)

                    if type(v) is list:
                        vl = ', '.join(v)
                        v = vl
                    self.element.add_attribute('docker_' + k, v)

            except Exception as e:
                logging.debug(e)
                pass

    def _add_aws_meta(self):
        url = 'http://169.254.169.254/latest/dynamic/instance-identity/document'

        try:
            request = urllib2.Request(url)
            resp = urllib2.urlopen(request, timeout=1).read()
            j = json.loads(resp)

            for k, v in j.items():
                if type(v) is list:
                    vl = ', '.join(v)
                    v = vl
                self.element.add_attribute(k, v)

                if k.lower() == 'instanceid':
                    instanceid = v

                if k.lower() == 'region':
                    region = v

            try:
                child = '{0}:{1}'.format(region, instanceid)
                self.element.add_relation(child)

            except Exception as e:
                pass

        except Exception as e:
            logging.debug(e)
            pass

    def _add_config_tags(self):
        tags = self.config.get('tags')

        if tags is not None:
            if type(tags) is list:
                for k, v in dict(tag.split(":") for tag in tags).iteritems():
                    self.element.add_tag(k.strip(), v.strip())

            if type(tags) is str:
                self.element.add_tag(tags.split(":")[0], tags.split(":")[1])

    def _add_config_relations(self):
        relations = self.config.get('relations')

        if relations is not None:
            if type(relations) is list:
                for r in relations:
                    self.element.add_relation(r.strip())

            if type(relations) is str:
                self.element.add_relation(relations.strip())

    def process(self, metric):
        metricId = metric.getCollectorPath() + '.' + metric.getMetricPath()

        self.element.add_sample(
            metricId, metric.timestamp, metric.value, metric.metric_type, host=metric.host)

        logging.debug(
            'length of self.element.samples: ' + str(len(self.element.samples)))

        if len(self.element.samples) >= self.batch_size:
            logging.debug('flushing data due to exceeding batch_size')
            self.flush()

    def flush(self):
        logging.debug('sending data')

        try:
            toff = self.api.check_time_offset()
            if toff not in range(-300, 300):
                logging.error('local time is {0} seconds out '
                              'of sync with the server'.format(toff))

            # Don't let too many metrics back up
            if len(self.element.metrics) >= (
                    self.batch_size * self.max_backlog_multiplier):
                trim_offset = (self.batch_size *
                               self.trim_backlog_multiplier * -1)
                logging.warn('NetuitiveHandler: Trimming backlog. Removing' +
                             ' oldest %d and keeping newest %d metrics',
                             len(self.element.metrics) - abs(trim_offset),
                             abs(trim_offset))
                self.element.metrics = self.element.metrics[trim_offset:]

            self.api.post(self.element)
            self.element.clear_samples()

            elapsed = int(time.time()) - self.flush_time
            if elapsed > 900 or self.flush_time == 0:
                self.flush_time = int(time.time())
                logging.info('NetuitiveHandler: Data posted successfully. ' +
                             'Next log message in 15 minutes.')

        except Exception as e:
            logging.exception('NetuitiveHandler: flush - %s', str(e))
<commit_msg>reduce the frequency of the time offset check in the linux agent<commit_after>"""
handler to flush stats to [Netuitive](http://www.netuitive.com)
# Dependencies

# Configuration
Enable handler

  * handlers = diamond.handler.netuitive_cloud.NetuitiveHandler

  * url = https://api.app.netuitive.com/ingest/infrastructure
  * api_key = NETUITIVE_API_KEY
  * tags = tag1:tag1val, tag2:tag2val

"""

from Handler import Handler
import logging
import re
import platform
import datetime
import time
import os
import json
import urllib2
from diamond.util import get_diamond_version

try:
    import psutil
except ImportError:
    psutil = None

try:
    import netuitive
except ImportError:
    netuitive = None

try:
    import docker
except ImportError:
    docker = None

def get_human_readable_size(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return(("%3.2f %s%s" % (num, unit, suffix)).strip())
        num /= 1024.0
    return(("%.2f %s%s" % (num, 'Y', suffix)).strip())

def check_lsb():

    if os.path.isfile("/etc/lsb-release"):
        try:
            _distributor_id_file_re = re.compile(
                "(?:DISTRIB_ID\s*=)\s*(.*)", re.I)
            _release_file_re = re.compile(
                "(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I)
            _codename_file_re = re.compile(
                "(?:DISTRIB_CODENAME\s*=)\s*(.*)", re.I)
            with open("/etc/lsb-release", "rU") as etclsbrel:
                for line in etclsbrel:
                    m = _distributor_id_file_re.search(line)
                    if m:
                        _u_distname = m.group(1).strip()
                    m = _release_file_re.search(line)
                    if m:
                        _u_version = m.group(1).strip()
                    m = _codename_file_re.search(line)
                    if m:
                        _u_id = m.group(1).strip()
                if _u_distname and _u_version:
                    return (_u_distname, _u_version, _u_id)
        except Exception as e:
            logging.debug(e)
            return(None)

    else:
        return(None)

class NetuitiveHandler(Handler):

    def __init__(self, config=None):
        """
        initialize Netuitive api and populate agent host metadata
        """

        if not netuitive:
            self.log.error('netuitive import failed. Handler disabled')
            self.enabled = False
            return

        try:
            Handler.__init__(self, config)

            logging.debug("initialize Netuitive handler")

            self.version = self._get_version()
            self.api = netuitive.Client(self.config['url'], self.config[
                                        'api_key'], self.version)

            self.element = netuitive.Element(
                location=self.config.get('location'))

            self.batch_size = int(self.config['batch'])

            self.max_backlog_multiplier = int(
                self.config['max_backlog_multiplier'])

            self.trim_backlog_multiplier = int(
                self.config['trim_backlog_multiplier'])

            self._add_sys_meta()
            self._add_aws_meta()
            self._add_docker_meta()
            self._add_config_tags()
            self._add_config_relations()

            self.flush_time = 0

            logging.debug(self.config)

        except Exception as e:
            logging.exception('NetuitiveHandler: init - %s', str(e))

    def get_default_config_help(self):
        """
        Returns the help text for the configuration options for this handler
        """
        config = super(NetuitiveHandler, self).get_default_config_help()

        config.update({
            'url': 'NetuitiveCloud url to send data to',
            'api_key': 'Datasource api key',
            'tags': 'Netuitive Tags',
            'relations': 'Netuitive child Element',
            'location': 'the location of this Element',
            'batch': 'How many to store before sending to the graphite server',
            'max_backlog_multiplier': 'how many batches to store before trimming',
            'trim_backlog_multiplier': 'Trim down how many batches',
        })
        return config

    def get_default_config(self):
        """
        Return the default config for the handler
        """

        config = super(NetuitiveHandler, self).get_default_config()

        config.update({
            'url': 'https://api.app.netuitive.com/ingest/infrastructure',
            'api_key': 'apikey',
            'tags': None,
            'relations': None,
            'location': None,
            'batch': 100,
            'max_backlog_multiplier': 5,
            'trim_backlog_multiplier': 4,
        })
        return config

    def __del__(self):
        pass

    def _get_version(self):
        """
        Return version string
        """

        ret = 'Diamond_' + get_diamond_version().rstrip()
        if os.path.isfile('/opt/netuitive-agent/version-manifest.txt'):
            with open('/opt/netuitive-agent/version-manifest.txt', 'r') as f:
                v = f.readline()

            f.close()

            ret = v.replace(' ', '_').lower().rstrip()

        return(ret)

    def _add_sys_meta(self):
        try:

            self.element.add_attribute('platform', platform.system())
            self.element.add_attribute('agent', self.version)

            if psutil:
                self.element.add_attribute('cpus', psutil.cpu_count())
                mem = psutil.virtual_memory()
                self.element.add_attribute(
                    'ram', get_human_readable_size(mem.total))
                self.element.add_attribute('ram bytes', mem.total)
                self.element.add_attribute(
                    'boottime', str(datetime.datetime.fromtimestamp(psutil.boot_time())))

            if platform.system().startswith('Linux'):
                if check_lsb() is None:
                    supported_dists = platform._supported_dists + ('system',)
                    dist = platform.linux_distribution(
                        supported_dists=supported_dists)

                else:
                    dist = check_lsb()

                self.element.add_attribute('distribution_name', str(dist[0]))
                self.element.add_attribute(
                    'distribution_version', str(dist[1]))
                if dist[2] != '':
                    self.element.add_attribute('distribution_id', str(dist[2]))

        except Exception as e:
            logging.debug(e)
            pass

    def _add_docker_meta(self):
        if docker:
            try:

                cc = docker.Client(
                    base_url='unix://var/run/docker.sock', version='auto')
                dockerver = cc.version()

                for k, v in dockerver.items():
                    logging.debug('docker_' + k + ' = ' + v)

                    if type(v) is list:
                        vl = ', '.join(v)
                        v = vl
                    self.element.add_attribute('docker_' + k, v)

            except Exception as e:
                logging.debug(e)
                pass

    def _add_aws_meta(self):
        url = 'http://169.254.169.254/latest/dynamic/instance-identity/document'

        try:
            request = urllib2.Request(url)
            resp = urllib2.urlopen(request, timeout=1).read()
            j = json.loads(resp)

            for k, v in j.items():
                if type(v) is list:
                    vl = ', '.join(v)
                    v = vl
                self.element.add_attribute(k, v)

                if k.lower() == 'instanceid':
                    instanceid = v

                if k.lower() == 'region':
                    region = v

            try:
                child = '{0}:{1}'.format(region, instanceid)
                self.element.add_relation(child)

            except Exception as e:
                pass

        except Exception as e:
            logging.debug(e)
            pass

    def _add_config_tags(self):
        tags = self.config.get('tags')

        if tags is not None:
            if type(tags) is list:
                for k, v in dict(tag.split(":") for tag in tags).iteritems():
                    self.element.add_tag(k.strip(), v.strip())

            if type(tags) is str:
                self.element.add_tag(tags.split(":")[0], tags.split(":")[1])

    def _add_config_relations(self):
        relations = self.config.get('relations')

        if relations is not None:
            if type(relations) is list:
                for r in relations:
                    self.element.add_relation(r.strip())

            if type(relations) is str:
                self.element.add_relation(relations.strip())

    def process(self, metric):
        metricId = metric.getCollectorPath() + '.' + metric.getMetricPath()

        self.element.add_sample(
            metricId, metric.timestamp, metric.value, metric.metric_type, host=metric.host)

        logging.debug(
            'length of self.element.samples: ' + str(len(self.element.samples)))

        if len(self.element.samples) >= self.batch_size:
            logging.debug('flushing data due to exceeding batch_size')
            self.flush()

    def flush(self):
        logging.debug('sending data')

        try:

            # Don't let too many metrics back up
            if len(self.element.metrics) >= (
                    self.batch_size * self.max_backlog_multiplier):
                trim_offset = (self.batch_size *
                               self.trim_backlog_multiplier * -1)
                logging.warn('NetuitiveHandler: Trimming backlog. Removing' +
                             ' oldest %d and keeping newest %d metrics',
                             len(self.element.metrics) - abs(trim_offset),
                             abs(trim_offset))
                self.element.metrics = self.element.metrics[trim_offset:]

            self.api.post(self.element)
            self.element.clear_samples()

            elapsed = int(time.time()) - self.flush_time
            if elapsed > 900 or self.flush_time == 0:

                toff = self.api.check_time_offset()
                if toff not in range(-300, 300):
                    logging.error('local time is {0} seconds out '
                                  'of sync with the server'.format(toff))

                self.flush_time = int(time.time())
                logging.info('NetuitiveHandler: Data posted successfully. ' +
                             'Next log message in 15 minutes.')

        except Exception as e:
            logging.exception('NetuitiveHandler: flush - %s', str(e))
<|endoftext|>


<commit_before># The MIT License (MIT)
# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os

from ..geoextent import GeoExtent
from ..image import AbstractTiledImage, ImagePyramid
from ..tilingscheme import TilingScheme

__author__ = "Norman Fomferra (Brockmann Consult GmbH)"

class NaturalEarth2Image(AbstractTiledImage):
    """
    A `TiledImage` implementation which provides 'Natural Earth v2' image tiles.

    @author Norman Fomferra
    """

    NUM_LEVEL_0_TILES = 2, 1
    TILE_SIZE = 256

    @staticmethod
    def get_pyramid():
        """
        Return an instance of a 'Natural Earth v2' image pyramid:
        * global coverage
        * JPEG RGB format
        * 3 levels of detail: 0 to 2
        * tile size: 256 pixels
        * 2 x 1 tiles on level zero
        """
        dir_path = os.path.join(os.path.dirname(__file__), 'NaturalEarth2')
        return ImagePyramid(TilingScheme(3, 2, 1, 256, 256, GeoExtent()),
                            [NaturalEarth2Image(dir_path, level) for level in (0, 1, 2)])

    def __init__(self, dir_path, z_index):
        factor = 1 << z_index
        num_tiles_x = factor * NaturalEarth2Image.NUM_LEVEL_0_TILES[0]
        num_tiles_y = factor * NaturalEarth2Image.NUM_LEVEL_0_TILES[1]
        tile_size = NaturalEarth2Image.TILE_SIZE
        self._z_index = z_index
        self._base_path = '%s/%d' % (dir_path, z_index)
        super().__init__((num_tiles_x * tile_size, num_tiles_y * tile_size),
                         tile_size=(tile_size, tile_size),
                         num_tiles=(num_tiles_x, num_tiles_y), format='JPEG', mode='RGB')

    def get_tile(self, tile_x, tile_y):
        num_tiles_y = self.num_tiles[1]
        path = '%s/%d/%d.jpg' % (self._base_path, tile_x, num_tiles_y - 1 - tile_y)
        with open(path, 'rb') as fp:
            return fp.read()
<commit_msg>using constants<commit_after># The MIT License (MIT)
# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os

from ..geoextent import GeoExtent
from ..image import AbstractTiledImage, ImagePyramid
from ..tilingscheme import TilingScheme

__author__ = "Norman Fomferra (Brockmann Consult GmbH)"

class NaturalEarth2Image(AbstractTiledImage):
    """
    A `TiledImage` implementation which provides 'Natural Earth v2' image tiles.

    @author Norman Fomferra
    """

    NUM_LEVELS = 3
    NUM_LEVEL_0_TILES_X = 2
    NUM_LEVEL_0_TILES_Y = 1
    TILE_SIZE = 256

    @staticmethod
    def get_pyramid():
        """
        Return an instance of a 'Natural Earth v2' image pyramid:
        * global coverage
        * JPEG RGB format
        * 3 levels of detail: 0 to 2
        * tile size: 256 pixels
        * 2 x 1 tiles on level zero
        """
        dir_path = os.path.join(os.path.dirname(__file__), 'NaturalEarth2')
        return ImagePyramid(TilingScheme(NaturalEarth2Image.NUM_LEVELS,
                                         NaturalEarth2Image.NUM_LEVEL_0_TILES_X,
                                         NaturalEarth2Image.NUM_LEVEL_0_TILES_Y,
                                         NaturalEarth2Image.TILE_SIZE,
                                         NaturalEarth2Image.TILE_SIZE,
                                         GeoExtent()),
                            [NaturalEarth2Image(dir_path, level) for level in range(NaturalEarth2Image.NUM_LEVELS)])

    def __init__(self, dir_path, z_index):
        factor = 1 << z_index
        num_tiles_x = factor * NaturalEarth2Image.NUM_LEVEL_0_TILES_X
        num_tiles_y = factor * NaturalEarth2Image.NUM_LEVEL_0_TILES_Y
        tile_size = NaturalEarth2Image.TILE_SIZE
        self._z_index = z_index
        self._base_path = '%s/%d' % (dir_path, z_index)
        super().__init__((num_tiles_x * tile_size, num_tiles_y * tile_size),
                         tile_size=(tile_size, tile_size),
                         num_tiles=(num_tiles_x, num_tiles_y), format='JPEG', mode='RGB')

    def get_tile(self, tile_x, tile_y):
        num_tiles_y = self.num_tiles[1]
        path = '%s/%d/%d.jpg' % (self._base_path, tile_x, num_tiles_y - 1 - tile_y)
        with open(path, 'rb') as fp:
            return fp.read()
<|endoftext|>


<commit_before># stdlib
import glob
import os.path
import sys
import tempfile

def public(f):
    """"Use a decorator to avoid retyping function/class names.

    * Grabbed from recipe by Sam Denton
    http://code.activestate.com/recipes/576993-public-decorator-adds-an-item-to-__all__/
    * Based on an idea by Duncan Booth:
    http://groups.google.com/group/comp.lang.python/msg/11cbb03e09611b8a
    * Improved via a suggestion by Dave Angel:
    http://groups.google.com/group/comp.lang.python/msg/3d400fb22d8a42e1
    """
    all = sys.modules[f.__module__].__dict__.setdefault('__all__', [])
    if f.__name__ not in all:  # Prevent duplicates if run from an IDE.
        all.append(f.__name__)
    return f
public(public)

def is_path_under_directory(path, directory):
    """Return True if ``path`` is in ``dir``.

    This operates only on paths and does not actually access the filesystem.
    """
    # TODO(brownhead): This might not handle unicode correctly...
    directory = os.path.abspath(directory) + "/"
    path = os.path.abspath(path) + "/"
    return path.startswith(directory)

def glob_foreach_list(foreach):
    """Globs a path/paths and return the flattened result.

    :param foreach: The path(s) to glob.
    :type foreach: list, str, or unicode
    :return: The sorted list of paths.

    :Example:

    >>> glob_foreach_list(["b/*", "a"])
    ["a", "b/1", "b/2"]
    >>> glob_foreach_list("b/*")
    ["b/1", "b/2"]
    """
    listified = foreach
    if isinstance(foreach, basestring):
        listified = [foreach]

    # Glob any strings in the list.
    globbed_list = []
    for i in listified:
        if isinstance(i, basestring):
            globbed_list += glob.glob(i)
        else:
            globbed_list.append(i)

    return sorted(globbed_list)

@public
class file(tempfile.SpooledTemporaryFile):
    DEFAULT_SPOOL_SIZE = 10 * 1024 * 1024  # 1024 * 1024 is one mebibyte

    def __init__(self, name=None, spool_size=DEFAULT_SPOOL_SIZE, content=None):
        print name, content
        self.name = name
        self.spool_size = spool_size
        tempfile.SpooledTemporaryFile.__init__(self, max_size=spool_size)
        if content is not None:
            self.write(content)
            self.seek(0)

# HACK(brownhead): I should stabilize on a single name.
TemporaryFile = file

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        # This is a little racey, but does a good job at reducing massive log spam, so definitely
        # a net win.
        if e.errno != 17 and os.path.isdir(path):
            raise

@public
def basename_noext(path):
    return os.path.splitext(os.path.basename(path))[0]
<commit_msg>Removed basename_noext function.<commit_after># stdlib
import glob
import os.path
import sys
import tempfile

def public(f):
    """"Use a decorator to avoid retyping function/class names.

    * Grabbed from recipe by Sam Denton
    http://code.activestate.com/recipes/576993-public-decorator-adds-an-item-to-__all__/
    * Based on an idea by Duncan Booth:
    http://groups.google.com/group/comp.lang.python/msg/11cbb03e09611b8a
    * Improved via a suggestion by Dave Angel:
    http://groups.google.com/group/comp.lang.python/msg/3d400fb22d8a42e1
    """
    all = sys.modules[f.__module__].__dict__.setdefault('__all__', [])
    if f.__name__ not in all:  # Prevent duplicates if run from an IDE.
        all.append(f.__name__)
    return f
public(public)

def is_path_under_directory(path, directory):
    """Return True if ``path`` is in ``dir``.

    This operates only on paths and does not actually access the filesystem.
    """
    # TODO(brownhead): This might not handle unicode correctly...
    directory = os.path.abspath(directory) + "/"
    path = os.path.abspath(path) + "/"
    return path.startswith(directory)

def glob_foreach_list(foreach):
    """Globs a path/paths and return the flattened result.

    :param foreach: The path(s) to glob.
    :type foreach: list, str, or unicode
    :return: The sorted list of paths.

    :Example:

    >>> glob_foreach_list(["b/*", "a"])
    ["a", "b/1", "b/2"]
    >>> glob_foreach_list("b/*")
    ["b/1", "b/2"]
    """
    listified = foreach
    if isinstance(foreach, basestring):
        listified = [foreach]

    # Glob any strings in the list.
    globbed_list = []
    for i in listified:
        if isinstance(i, basestring):
            globbed_list += glob.glob(i)
        else:
            globbed_list.append(i)

    return sorted(globbed_list)

@public
class file(tempfile.SpooledTemporaryFile):
    DEFAULT_SPOOL_SIZE = 10 * 1024 * 1024  # 1024 * 1024 is one mebibyte

    def __init__(self, name=None, spool_size=DEFAULT_SPOOL_SIZE, content=None):
        print name, content
        self.name = name
        self.spool_size = spool_size
        tempfile.SpooledTemporaryFile.__init__(self, max_size=spool_size)
        if content is not None:
            self.write(content)
            self.seek(0)

# HACK(brownhead): I should stabilize on a single name.
TemporaryFile = file

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        # This is a little racey, but does a good job at reducing massive log spam, so definitely
        # a net win.
        if e.errno != 17 and os.path.isdir(path):
            raise
<|endoftext|>