<commit_before>"""
Cosmos automatic deployment Fabric file -

causes Fabric to deploy each Cosmos component at a configured host.
"""
import os
import json
from StringIO import StringIO

from fabric.api import run, execute, sudo, put, cd, env
from fabric.contrib import files
import fabric.context_managers as ctx
from fabric.colors import green, red, yellow
from fabric.decorators import roles, task, parallel
from fabric.utils import puts
from mako.template import Template

import common
import hadoop_install
import hue_deployment

BASEPATH = os.path.dirname(os.path.realpath(__file__))
CONFIG = json.loads(open(env.config, 'r').read())
env.roledefs = CONFIG['hosts']

@task
def deploy(dependenciespath, thrift_tar, jdk_rpm):
    """
    Deploys all the necessary components to get a running Cosmos cluster
    """
    puts(yellow("Opening FTP ports"))
    execute(open_ftp_port)
    process_start_msg = "DEPLOY: stating %s deployment"
    puts(yellow(process_start_msg % "JDK"))
    execute(deploy_jdk, os.path.join(dependenciespath, jdk_rpm))
    puts(yellow(process_start_msg % "CDH"))
    deploy_cdh()
    puts(yellow(process_start_msg % "HUE"))
    execute(deploy_hue, os.path.join(dependenciespath, thrift_tar))
    puts(yellow(process_start_msg % "SFTP"))
    execute(deploy_sftp)
    puts(yellow(process_start_msg % "Ganglia"))
    execute(deploy_ganglia)
    puts(yellow(process_start_msg % "Mongo"))
    execute(deploy_mongo)

@task
@roles('namenode', 'frontend', 'jobtracker', 'mongo', 'datanodes', 'tasktrackers')
def open_ftp_port():
    """
    Opening the FTP port is necessary in the staging cluster because  the
    internal yum repositories are FTP-based instead of the standard HTTP
    repos
    """
    sudo("iptables -A OUTPUT -p tcp --dport 21 -j ACCEPT")
    sudo("iptables -A OUTPUT -p tcp --dport 22 -j ACCEPT")
    sudo("service iptables restart")

@task
@roles('frontend')
def add_test_setup():
    """
    Installs any test-specific setup components
    """
    files_to_delete = put(os.path.join(BASEPATH,
                          '../cosmos/tests/testUser.json'), 'testUser.json')
    with cd('/usr/share/hue'):
        run('build/env/bin/hue loaddata ~/testUser.json')
    for f in files_to_delete:
        run('rm %s' % f)

@task
@parallel
@roles('namenode', 'jobtracker', 'frontend', 'datanodes', 'tasktrackers')
def deploy_jdk(jdkpath):
    """
    Deployes basic packages that somehow are not present in Joyent templates.
    """
    if not common.has_package('wget'):
        run('yum -y install wget')
    if not common.has_package('sudo'):
        run('yum -y install sudo')
    if not common.has_package('jdk'):
        put(jdkpath, 'jdk.rpm')
        run('rpm -ihv jdk.rpm')

@task
@roles('frontend')
def deploy_hue(thrift_tarpath):
    """
    Deploys the HUE frontend from Cloudera, plus our fixes and our app
    """
    common.install_dependencies(['mysql', 'wget'])
    hue_deployment.install_git()
    hue_deployment.install_and_patch_hue(CONFIG)
    execute(hue_deployment.install_hue_plugins)
    hue_deployment.install_thrift(thrift_tarpath)
    hue_deployment.install_cosmos_app()
    hue_deployment.start_daemons()
    hue_deployment.cleanup()

@task
@roles('frontend')
def deploy_sftp():
    """
    Deploys the SFTP server as a Java JAR and starts it
    """
    injection_exec = 'injection-server-{0}.jar'.format(CONFIG['version'])
    injection_jar = os.path.join(CONFIG['injection_path'], 'target',
                                 injection_exec)
    if not os.path.exists(injection_jar):
        puts(red('ERROR: injection server file not found: %s' % injection_jar))
        return
    exec_path = os.path.join('~', 'injection', injection_exec)
    if not files.exists(exec_path):
        run("mkdir -p {0}".format(os.path.join('~', 'injection')))
    put(injection_jar, exec_path)

    injection_conf = StringIO()
    template = Template(filename = os.path.join(BASEPATH,
                                              'templates/injection.conf.mako'))
    injection_conf.write(template.render(
            namenode = CONFIG['hosts']['namenode'][0]))
    put(injection_conf, "/etc/injection.cfg")
    symlink = "/usr/local/injection-server"
    if not files.exists(symlink):
        run("ln -s {0} {1}".format(exec_path, symlink))
    logfile = "/var/log/injection/server.log"
    if not files.exists(logfile):
        run("mkdir -p /var/log/injection")
        run("echo '' >> {0}".format(logfile))
    pidfile = "/var/run/injection/server.pid"
    if not files.exists(pidfile):
        run("mkdir -p /var/run/injection")
        run("echo '' >> {0}".format(pidfile))
    put(os.path.join(BASEPATH, "templates/injection.init.d"),
                               "/etc/init.d/injection")
    with ctx.settings(warn_only=True):
        start = run("/etc/init.d/injection start", pty=False)
        if start.failed:
            restart = run("/etc/init.d/injection restart", pty=False)
            if restart.failed and 'Permission denied' in restart.stdout:
                sudo('chmod +x /etc/init.d/injection')
                run("service injection start")

@task
def deploy_cdh():
    """Deploys the Cloudera Distribution for Hadoop"""
    execute(hadoop_install.install_cdh, CONFIG)
    execute(hadoop_install.create_hadoop_dirs, CONFIG)
    execute(hadoop_install.configure_hadoop, CONFIG)
    execute(hadoop_install.deploy_namenode_daemon)
    execute(hadoop_install.deploy_jobtracker_daemon)
    execute(hadoop_install.deploy_datanode_daemon)
    execute(hadoop_install.deploy_tasktracker_daemon)

@task
@roles('mongo')
def deploy_mongo():
    """Install the latest MongoDB distribution"""
    with cd('/etc/yum.repos.d'):
        if not files.exists('10gen.repo'):
            put(os.path.join(BASEPATH, 'templates/10gen.repo'), '10gen.repo')
    run('yum -y install mongo-10gen mongo-10gen-server')

    mongo_conf = StringIO()
    template = Template(filename = os.path.join(BASEPATH,
                                              'templates/mongod.conf.mako'))
    mongo_conf.write(template.render(
            dbpath = CONFIG['mongo_db_path']))
    put(mongo_conf, '/etc/mongod.conf')

    run('service mongod start')
    run('chkconfig mongod on')
    run("mongo --eval \"db.adminCommand('listDatabases').databases.forEach("
        "function (d) { "
        "  if (d.name != 'local' && d.name != 'admin' && d.name != 'config')"
        "    db.getSiblingDB(d.name).dropDatabase();"
        "})\"")

@task
def deploy_models():
    """
    Deploys the preconfigured statistical models

    This function is not ready for production. It is only here as a general
    idea on what is needed to deploy a model, but we currently have no models
    to deploy
    """
    ## TODO: define in configuration
    model_paths = []
    for model in model_paths:
        put(model)
        sudo('hadoop dfs -put {0} /models/{0}'.format(model))
        run('rm %s' % model)

@task
def deploy_ganglia():
    execute(configure_hadoop_metrics)
    execute(configure_ntp)
    execute(install_gmetad)
    execute(install_ganglia_frontend)
    execute(install_gmond)

@parallel
@roles('namenode', 'namenode', 'jobtracker', 'frontend', 'datanodes', 'mongo',
       'tasktrackers')
def configure_ntp():
    common.install_dependencies(['ntp'])
    ntp_conf = StringIO()
    template = Template(filename='templates/ntp.conf.mako')
    content = template.render()
    ntp_conf.write(content)
    ntp_conf_path = "/etc/ntp.conf"
    put(ntp_conf, ntp_conf_path)
    sudo(("iptables -A OUTPUT -p udp -d 0.rhel.pool.ntp.org "
          "--dport 123 -j ACCEPT"))
    sudo("service iptables restart")
    run("chkconfig --level 2 ntpd on")
    run("service ntpd start")

@roles('frontend')
def install_gmetad():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-gmetad")
        run("yum -y install ganglia-gmetad")
    gmetad_conf = StringIO()
    template = Template(filename='templates/gmetad.conf.mako')
    content = template.render(
            monitored_hosts = common.clean_host_list(CONFIG['hosts'].values()))
    gmetad_conf.write(content)
    gmetad_cfg_path = "/etc/gmetad.conf"
    if not files.exists(gmetad_cfg_path):
        run("mkdir -p /etc/ganglia")
        run("echo '' >> {0}".format(gmetad_cfg_path))
    put(gmetad_conf, gmetad_cfg_path)
    run("service gmetad start")
    run("chkconfig --level 2 gmetad on")

@roles('frontend')
def install_ganglia_frontend():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-web")
        run("yum -y install ganglia-web")
    ganglia_web_conf = StringIO()
    template = Template(filename='templates/conf.php.mako')
    content = template.render(gmetad_port = 8651)
    ganglia_web_conf.write(content)
    ganglia_web_conf_path = "/usr/share/ganglia/conf.php"
    put(ganglia_web_conf, ganglia_web_conf_path)
    start = run("apachectl start")
    if start.endswith("already running"):
        run("apachectl restart")

@parallel
@roles('namenode', 'jobtracker', 'frontend', 'datanodes', 'mongo',
       'tasktrackers')
def install_gmond():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-gmond")
        run("yum -y install ganglia-gmond")
    gmond_conf = StringIO()
    template = Template(filename='templates/gmond.conf.mako')
    content = template.render(
            gmetad_host = common.clean_host_list(CONFIG['hosts']['namenode']))
    gmond_conf.write(content)
    gmond_conf_path = "/etc/gmond.conf"
    if not files.exists(gmond_conf_path):
        run("mkdir -p /etc/ganglia")
        run("echo '' >> {0}".format(gmond_conf_path))
    put(gmond_conf, gmond_conf_path)
    run("service gmond start")
    run("chkconfig --level 2 gmond on")

@parallel
@roles('namenode', 'jobtracker', 'datanodes', 'tasktrackers')
def configure_hadoop_metrics():
    hadoop_metrics_conf = StringIO()
    template = Template(filename='templates/hadoop-metrics.properties.mako')
    content = template.render(
            namenode = common.clean_host_list(CONFIG['hosts']['namenode']))
    hadoop_metrics_conf.write(content)
    ## TODO: parametize Hadoop version
    hadoop_metrics_path = "/etc/hadoop-0.20/conf/hadoop-metrics.properties"
    if not files.exists(hadoop_metrics_path):
        run("mkdir -p /etc/hadoop-0.20/conf/")
        run("echo '' >> {0}".format(hadoop_metrics_path))
    put(hadoop_metrics_conf, hadoop_metrics_path)
    if files.exists("/etc/init.d/hadoop-0.20-tasktracker"):
        with ctx.hide('stdout'):
            run("service hadoop-0.20-tasktracker restart")
<commit_msg>Open ports for Ganglia<commit_after>"""
Cosmos automatic deployment Fabric file -

causes Fabric to deploy each Cosmos component at a configured host.
"""
import os
import json
from StringIO import StringIO

from fabric.api import run, execute, sudo, put, cd, env
from fabric.contrib import files
import fabric.context_managers as ctx
from fabric.colors import green, red, yellow
from fabric.decorators import roles, task, parallel
from fabric.utils import puts
from mako.template import Template

import common
import hadoop_install
import hue_deployment

BASEPATH = os.path.dirname(os.path.realpath(__file__))
CONFIG = json.loads(open(env.config, 'r').read())
env.roledefs = CONFIG['hosts']

@task
def deploy(dependenciespath, thrift_tar, jdk_rpm):
    """
    Deploys all the necessary components to get a running Cosmos cluster
    """
    puts(yellow("Opening FTP ports"))
    execute(open_ftp_port)
    process_start_msg = "DEPLOY: stating %s deployment"
    puts(yellow(process_start_msg % "JDK"))
    execute(deploy_jdk, os.path.join(dependenciespath, jdk_rpm))
    puts(yellow(process_start_msg % "CDH"))
    deploy_cdh()
    puts(yellow(process_start_msg % "HUE"))
    execute(deploy_hue, os.path.join(dependenciespath, thrift_tar))
    puts(yellow(process_start_msg % "SFTP"))
    execute(deploy_sftp)
    puts(yellow(process_start_msg % "Ganglia"))
    execute(deploy_ganglia)
    puts(yellow(process_start_msg % "Mongo"))
    execute(deploy_mongo)

@task
@roles('namenode', 'frontend', 'jobtracker', 'mongo', 'datanodes', 'tasktrackers')
def open_ftp_port():
    """
    Opening the FTP port is necessary in the staging cluster because  the
    internal yum repositories are FTP-based instead of the standard HTTP
    repos
    """
    sudo("iptables -A OUTPUT -p tcp --dport 21 -j ACCEPT")
    sudo("iptables -A OUTPUT -p tcp --dport 22 -j ACCEPT")
    sudo("service iptables restart")

@task
@roles('frontend')
def add_test_setup():
    """
    Installs any test-specific setup components
    """
    files_to_delete = put(os.path.join(BASEPATH,
                          '../cosmos/tests/testUser.json'), 'testUser.json')
    with cd('/usr/share/hue'):
        run('build/env/bin/hue loaddata ~/testUser.json')
    for f in files_to_delete:
        run('rm %s' % f)

@task
@parallel
@roles('namenode', 'jobtracker', 'frontend', 'datanodes', 'tasktrackers')
def deploy_jdk(jdkpath):
    """
    Deployes basic packages that somehow are not present in Joyent templates.
    """
    if not common.has_package('wget'):
        run('yum -y install wget')
    if not common.has_package('sudo'):
        run('yum -y install sudo')
    if not common.has_package('jdk'):
        put(jdkpath, 'jdk.rpm')
        run('rpm -ihv jdk.rpm')

@task
@roles('frontend')
def deploy_hue(thrift_tarpath):
    """
    Deploys the HUE frontend from Cloudera, plus our fixes and our app
    """
    common.install_dependencies(['mysql', 'wget'])
    hue_deployment.install_git()
    hue_deployment.install_and_patch_hue(CONFIG)
    execute(hue_deployment.install_hue_plugins)
    hue_deployment.install_thrift(thrift_tarpath)
    hue_deployment.install_cosmos_app()
    hue_deployment.start_daemons()
    hue_deployment.cleanup()

@task
@roles('frontend')
def deploy_sftp():
    """
    Deploys the SFTP server as a Java JAR and starts it
    """
    injection_exec = 'injection-server-{0}.jar'.format(CONFIG['version'])
    injection_jar = os.path.join(CONFIG['injection_path'], 'target',
                                 injection_exec)
    if not os.path.exists(injection_jar):
        puts(red('ERROR: injection server file not found: %s' % injection_jar))
        return
    exec_path = os.path.join('~', 'injection', injection_exec)
    if not files.exists(exec_path):
        run("mkdir -p {0}".format(os.path.join('~', 'injection')))
    put(injection_jar, exec_path)

    injection_conf = StringIO()
    template = Template(filename = os.path.join(BASEPATH,
                                              'templates/injection.conf.mako'))
    injection_conf.write(template.render(
            namenode = CONFIG['hosts']['namenode'][0]))
    put(injection_conf, "/etc/injection.cfg")
    symlink = "/usr/local/injection-server"
    if not files.exists(symlink):
        run("ln -s {0} {1}".format(exec_path, symlink))
    logfile = "/var/log/injection/server.log"
    if not files.exists(logfile):
        run("mkdir -p /var/log/injection")
        run("echo '' >> {0}".format(logfile))
    pidfile = "/var/run/injection/server.pid"
    if not files.exists(pidfile):
        run("mkdir -p /var/run/injection")
        run("echo '' >> {0}".format(pidfile))
    put(os.path.join(BASEPATH, "templates/injection.init.d"),
                               "/etc/init.d/injection")
    with ctx.settings(warn_only=True):
        start = run("/etc/init.d/injection start", pty=False)
        if start.failed:
            restart = run("/etc/init.d/injection restart", pty=False)
            if restart.failed and 'Permission denied' in restart.stdout:
                sudo('chmod +x /etc/init.d/injection')
                run("service injection start")

@task
def deploy_cdh():
    """Deploys the Cloudera Distribution for Hadoop"""
    execute(hadoop_install.install_cdh, CONFIG)
    execute(hadoop_install.create_hadoop_dirs, CONFIG)
    execute(hadoop_install.configure_hadoop, CONFIG)
    execute(hadoop_install.deploy_namenode_daemon)
    execute(hadoop_install.deploy_jobtracker_daemon)
    execute(hadoop_install.deploy_datanode_daemon)
    execute(hadoop_install.deploy_tasktracker_daemon)

@task
@roles('mongo')
def deploy_mongo():
    """Install the latest MongoDB distribution"""
    with cd('/etc/yum.repos.d'):
        if not files.exists('10gen.repo'):
            put(os.path.join(BASEPATH, 'templates/10gen.repo'), '10gen.repo')
    run('yum -y install mongo-10gen mongo-10gen-server')

    mongo_conf = StringIO()
    template = Template(filename = os.path.join(BASEPATH,
                                              'templates/mongod.conf.mako'))
    mongo_conf.write(template.render(
            dbpath = CONFIG['mongo_db_path']))
    put(mongo_conf, '/etc/mongod.conf')

    run('service mongod start')
    run('chkconfig mongod on')
    run("mongo --eval \"db.adminCommand('listDatabases').databases.forEach("
        "function (d) { "
        "  if (d.name != 'local' && d.name != 'admin' && d.name != 'config')"
        "    db.getSiblingDB(d.name).dropDatabase();"
        "})\"")

@task
def deploy_models():
    """
    Deploys the preconfigured statistical models

    This function is not ready for production. It is only here as a general
    idea on what is needed to deploy a model, but we currently have no models
    to deploy
    """
    ## TODO: define in configuration
    model_paths = []
    for model in model_paths:
        put(model)
        sudo('hadoop dfs -put {0} /models/{0}'.format(model))
        run('rm %s' % model)

@task
def deploy_ganglia():
    execute(configure_hadoop_metrics)
    execute(configure_ntp)
    execute(install_gmetad)
    execute(install_ganglia_frontend)
    execute(install_gmond)

@parallel
@roles('namenode', 'namenode', 'jobtracker', 'frontend', 'datanodes', 'mongo',
       'tasktrackers')
def configure_ntp():
    common.install_dependencies(['ntp'])
    ntp_conf = StringIO()
    template = Template(filename='templates/ntp.conf.mako')
    content = template.render()
    ntp_conf.write(content)
    ntp_conf_path = "/etc/ntp.conf"
    put(ntp_conf, ntp_conf_path)
    sudo(("iptables -A OUTPUT -p udp -d 0.rhel.pool.ntp.org "
          "--dport 123 -j ACCEPT"))
    sudo("service iptables restart")
    run("chkconfig --level 2 ntpd on")
    run("service ntpd start")

@roles('frontend')
def install_gmetad():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-gmetad")
        run("yum -y install ganglia-gmetad")
    gmetad_conf = StringIO()
    template = Template(filename='templates/gmetad.conf.mako')
    content = template.render(
            monitored_hosts = common.clean_host_list(CONFIG['hosts'].values()))
    gmetad_conf.write(content)
    gmetad_cfg_path = "/etc/gmetad.conf"
    if not files.exists(gmetad_cfg_path):
        run("mkdir -p /etc/ganglia")
        run("echo '' >> {0}".format(gmetad_cfg_path))
    put(gmetad_conf, gmetad_cfg_path)
    sudo("iptables -A INPUT -p tcp -d {0} --dport 8649 -j ACCEPT"
            .format(common.clean_host_list(CONFIG['hosts']['frontend'])))
    sudo("service iptables restart")
    run("service gmetad start")
    run("chkconfig --level 2 gmetad on")

@roles('frontend')
def install_ganglia_frontend():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-web")
        run("yum -y install ganglia-web")
    ganglia_web_conf = StringIO()
    template = Template(filename='templates/conf.php.mako')
    content = template.render(gmetad_port = 8651)
    ganglia_web_conf.write(content)
    ganglia_web_conf_path = "/usr/share/ganglia/conf.php"
    put(ganglia_web_conf, ganglia_web_conf_path)
    start = run("apachectl start")
    if start.endswith("already running"):
        run("apachectl restart")

@parallel
@roles('namenode', 'jobtracker', 'frontend', 'datanodes', 'mongo',
       'tasktrackers')
def install_gmond():
    with ctx.hide('stdout'):
        repolist = run("yum repolist")
        epel_installed = any([line.split()[0] == 'epel'
                              for line in repolist.splitlines()])
        if not epel_installed:
            major_version = 5
            minor_version = 4
            repo_rpm = 'epel-release-{0}-{1}.noarch.rpm'.format(
                    major_version, minor_version)
            base_url = ('http://download.fedoraproject.org/pub/epel/{0}/i386/'
                            .format(major_version))
            repo_url = base_url + repo_rpm
            run('wget %s' % repo_url)
            run('rpm -Uvh %s' % repo_rpm)
        run("yum -y erase ganglia-gmond")
        run("yum -y install ganglia-gmond")
    gmond_conf = StringIO()
    template = Template(filename='templates/gmond.conf.mako')
    content = template.render(
            gmetad_host = common.clean_host_list(CONFIG['hosts']['namenode']))
    gmond_conf.write(content)
    gmond_conf_path = "/etc/gmond.conf"
    if not files.exists(gmond_conf_path):
        run("mkdir -p /etc/ganglia")
        run("echo '' >> {0}".format(gmond_conf_path))
    put(gmond_conf, gmond_conf_path)
    sudo("iptables -A OUTPUT -p udp -d {0} --dport 8649 -j ACCEPT"
            .format(common.clean_host_list(CONFIG['hosts']['frontend'])))
    sudo("service iptables restart")
    run("service gmond start")
    run("chkconfig --level 2 gmond on")

@parallel
@roles('namenode', 'jobtracker', 'datanodes', 'tasktrackers')
def configure_hadoop_metrics():
    hadoop_metrics_conf = StringIO()
    template = Template(filename='templates/hadoop-metrics.properties.mako')
    content = template.render(
            namenode = common.clean_host_list(CONFIG['hosts']['namenode']))
    hadoop_metrics_conf.write(content)
    ## TODO: parametize Hadoop version
    hadoop_metrics_path = "/etc/hadoop-0.20/conf/hadoop-metrics.properties"
    if not files.exists(hadoop_metrics_path):
        run("mkdir -p /etc/hadoop-0.20/conf/")
        run("echo '' >> {0}".format(hadoop_metrics_path))
    put(hadoop_metrics_conf, hadoop_metrics_path)
    if files.exists("/etc/init.d/hadoop-0.20-tasktracker"):
        with ctx.hide('stdout'):
            run("service hadoop-0.20-tasktracker restart")
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''checkplotserver.py - Waqas Bhatti (wbhatti@astro.princeton.edu) - Nov 2016

This is the Tornado web-server for serving checkplots.

'''

####################
## SYSTEM IMPORTS ##
####################

import os
import os.path
import gzip
try:
    import cPickle as pickle
except:
    import pickle
import base64
import hashlib
import signal
import logging
from datetime import time
import json

# setup signal trapping on SIGINT
def recv_sigint(signum, stack):
    '''
    handler function to receive and process a SIGINT

    '''
    raise KeyboardInterrupt

# register the signal callback
signal.signal(signal.SIGINT,recv_sigint)
signal.signal(signal.SIGTERM,recv_sigint)

#####################
## TORNADO IMPORTS ##
#####################

import tornado.ioloop
import tornado.httpserver
import tornado.web
import tornado.options
from tornado.options import define, options

###########################
## DEFINING URL HANDLERS ##
###########################

import astrobase.checkplotserver_handlers as cphandlers

###############################
### APPLICATION SETUP BELOW ###
###############################

modpath = os.path.abspath(os.path.dirname(__file__))

# define our commandline options
define('port',
       default=5225,
       help='run on the given port.',
       type=int)
define('serve',
       default='127.0.0.1',
       help='bind to given address and serve content.',
       type=str)
define('assetpath',
       default=os.path.join(modpath,'data','cps-assets'),
       help=('sets the asset (server images, css, js, DB) path for '
             'checkplotserver.'),
       type=str)
define('checkplotlist',
       default=None,
       help=('the path to the checkplot-filelist.json file '
             'listing checkplots to load and serve. if this is not provided, '
             'checkplotserver will start up in global mode, '
             'showing all checkplot lists '
             'it knows about, and ask which one should be used'),
       type=str)
define('debugmode',
       default=0,
       help='start up in debug mode if set to 1.',
       type=int)

############
### MAIN ###
############

def main():
    # parse the command line
    tornado.options.parse_command_line()

    DEBUG = True if options.debugmode == 1 else False

    # get a logger
    LOGGER = logging.getLogger('checkplotserver')
    if DEBUG:
        LOGGER.setLevel(logging.DEBUG)
    else:
        LOGGER.setLevel(logging.INFO)

    ###################
    ## SET UP CONFIG ##
    ###################

    ASSETPATH = options.assetpath

    # this is the directory checkplotserver.py was executed from. used to figure
    # out checkplot locations
    CURRENTDIR = os.getcwd()

    # load the checkplot project list in the {ASSETPATH}/cps-projects.json file
    try:

        projectlistf = os.path.join(ASSETPATH, 'cps-projects.json')

        with open(projectlistf,'r') as infd:
            ALLPROJECTS = json.load(infd)

        LOGGER.info('using project database: %s' % projectlistf)

    # if it doesn't exist, make one
    except Exception as e:

        LOGGER.warning('no existing project database. '
                       'creating a new one at %s' % projectlistf)

        projectdict = {
            'nprojects':1,
            'sampleproject':{
                'checkplotlist':'checkplot-pickle-flist.json',
                'ncheckplots':1,
            }
        }
        with open(projectlistf,'w') as outfd:
            json.dump(projectdict, outfd)

        ALLPROJECTS = projectdict

    # if a checkplotlist is provided, then load it. all paths in this file are
    # relative to the path of the checkplotlist file itself.
    cplistfile = options.checkplotlist
    LOGGER.info('provided checkplotlist = %s' % cplistfile)

    if cplistfile and os.path.exists(cplistfile):

        with open(cplistfile,'r') as infd:
            CHECKPLOTLIST = json.load(infd)
        LOGGER.info('using checkplot list file %s' % cplistfile)

    else:
        LOGGER.warning('could not find checkplotlist %s' %
                       cplistfile)
        CHECKPLOTLIST = None

    ##################
    ## URL HANDLERS ##
    ##################

    HANDLERS = [
        # index page
        (r'/',
         cphandlers.IndexHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
        (r'/cp/?(.*)',
         cphandlers.CheckplotHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
        (r'/op',
         cphandlers.OperationsHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
    ]

    #######################
    ## APPLICATION SETUP ##
    #######################

    app = tornado.web.Application(
        handlers=HANDLERS,
        static_path=ASSETPATH,
        template_path=ASSETPATH,
        static_url_prefix='/static/',
        debug=DEBUG,
    )

    # start up the HTTP server and our application. xheaders = True turns on
    # X-Forwarded-For support so we can see the remote IP in the logs
    http_server = tornado.httpserver.HTTPServer(app, xheaders=True)
    http_server.listen(options.port, options.serve)

    LOGGER.info('starting event loop. listening on http://localhost:%s' %
                options.port)

    # start the IOLoop and begin serving requests
    try:
        tornado.ioloop.IOLoop.instance().start()

    except KeyboardInterrupt:
        LOGGER.info('received Ctrl-C: shutting down...')

# run the server
if __name__ == '__main__':
    main()
<commit_msg>minor fix<commit_after>#!/usr/bin/env python

'''checkplotserver.py - Waqas Bhatti (wbhatti@astro.princeton.edu) - Nov 2016

This is the Tornado web-server for serving checkplots.

'''

####################
## SYSTEM IMPORTS ##
####################

import os
import os.path
import gzip
try:
    import cPickle as pickle
except:
    import pickle
import base64
import hashlib
import signal
import logging
from datetime import time
import json

# setup signal trapping on SIGINT
def recv_sigint(signum, stack):
    '''
    handler function to receive and process a SIGINT

    '''
    raise KeyboardInterrupt

# register the signal callback
signal.signal(signal.SIGINT,recv_sigint)
signal.signal(signal.SIGTERM,recv_sigint)

#####################
## TORNADO IMPORTS ##
#####################

import tornado.ioloop
import tornado.httpserver
import tornado.web
import tornado.options
from tornado.options import define, options

###########################
## DEFINING URL HANDLERS ##
###########################

import astrobase.checkplotserver_handlers as cphandlers

###############################
### APPLICATION SETUP BELOW ###
###############################

modpath = os.path.abspath(os.path.dirname(__file__))

# define our commandline options
define('port',
       default=5225,
       help='run on the given port.',
       type=int)
define('serve',
       default='127.0.0.1',
       help='bind to given address and serve content.',
       type=str)
define('assetpath',
       default=os.path.join(modpath,'data','cps-assets'),
       help=('sets the asset (server images, css, js, DB) path for '
             'checkplotserver.'),
       type=str)
define('checkplotlist',
       default=None,
       help=('the path to the checkplot-filelist.json file '
             'listing checkplots to load and serve. if this is not provided, '
             'checkplotserver will start up in global mode, '
             'showing all checkplot lists '
             'it knows about, and ask which one should be used'),
       type=str)
define('debugmode',
       default=0,
       help='start up in debug mode if set to 1.',
       type=int)

############
### MAIN ###
############

def main():
    # parse the command line
    tornado.options.parse_command_line()

    DEBUG = True if options.debugmode == 1 else False

    # get a logger
    LOGGER = logging.getLogger('checkplotserver')
    if DEBUG:
        LOGGER.setLevel(logging.DEBUG)
    else:
        LOGGER.setLevel(logging.INFO)

    ###################
    ## SET UP CONFIG ##
    ###################

    ASSETPATH = options.assetpath

    # this is the directory checkplotserver.py was executed from. used to figure
    # out checkplot locations
    CURRENTDIR = os.getcwd()

    # load the checkplot project list in the {ASSETPATH}/cps-projects.json file
    try:

        projectlistf = os.path.join(modpath,'data', 'cps-projects.json')

        with open(projectlistf,'r') as infd:
            ALLPROJECTS = json.load(infd)

        LOGGER.info('using project database: %s' % projectlistf)

    # if it doesn't exist, make one
    except Exception as e:

        LOGGER.warning('no existing project database. '
                       'creating a new one at %s' % projectlistf)

        projectdict = {
            'nprojects':1,
            'sampleproject':{
                'checkplotlist':'checkplot-pickle-flist.json',
                'ncheckplots':1,
            }
        }
        with open(projectlistf,'w') as outfd:
            json.dump(projectdict, outfd)

        ALLPROJECTS = projectdict

    # if a checkplotlist is provided, then load it. all paths in this file are
    # relative to the path of the checkplotlist file itself.
    cplistfile = options.checkplotlist
    LOGGER.info('provided checkplotlist = %s' % cplistfile)

    if cplistfile and os.path.exists(cplistfile):

        with open(cplistfile,'r') as infd:
            CHECKPLOTLIST = json.load(infd)
        LOGGER.info('using checkplot list file %s' % cplistfile)

    else:
        LOGGER.warning('could not find checkplotlist %s' %
                       cplistfile)
        CHECKPLOTLIST = None

    ##################
    ## URL HANDLERS ##
    ##################

    HANDLERS = [
        # index page
        (r'/',
         cphandlers.IndexHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
        (r'/cp/?(.*)',
         cphandlers.CheckplotHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
        (r'/op',
         cphandlers.OperationsHandler,
         {'currentdir':CURRENTDIR,
          'assetpath':ASSETPATH,
          'allprojects':ALLPROJECTS,
          'cplist':CHECKPLOTLIST,
          'cplistfile':cplistfile}),
    ]

    #######################
    ## APPLICATION SETUP ##
    #######################

    app = tornado.web.Application(
        handlers=HANDLERS,
        static_path=ASSETPATH,
        template_path=ASSETPATH,
        static_url_prefix='/static/',
        debug=DEBUG,
    )

    # start up the HTTP server and our application. xheaders = True turns on
    # X-Forwarded-For support so we can see the remote IP in the logs
    http_server = tornado.httpserver.HTTPServer(app, xheaders=True)
    http_server.listen(options.port, options.serve)

    LOGGER.info('starting event loop. listening on http://localhost:%s' %
                options.port)

    # start the IOLoop and begin serving requests
    try:
        tornado.ioloop.IOLoop.instance().start()

    except KeyboardInterrupt:
        LOGGER.info('received Ctrl-C: shutting down...')

# run the server
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2015-2016 Claudiu Popa <pcmanticore@gmail.com>

# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER

"""Astroid hooks for numpy."""

import astroid

# TODO(cpopa): drop when understanding augmented assignments

def numpy_core_transform():
    return astroid.parse('''
    from numpy.core import numeric
    from numpy.core import fromnumeric
    from numpy.core import defchararray
    from numpy.core import records
    from numpy.core import function_base
    from numpy.core import machar
    from numpy.core import getlimits
    from numpy.core import shape_base
    __all__ = (['char', 'rec', 'memmap', 'chararray'] + numeric.__all__ +
               fromnumeric.__all__ +
               records.__all__ +
               function_base.__all__ +
               machar.__all__ +
               getlimits.__all__ +
               shape_base.__all__)
    ''')

def numpy_transform():
    return astroid.parse('''
    from numpy import core
    from numpy import matrixlib as _mat
    from numpy import lib
    __all__ = ['add_newdocs',
               'ModuleDeprecationWarning',
               'VisibleDeprecationWarning', 'linalg', 'fft', 'random',
               'ctypeslib', 'ma',
               '__version__', 'pkgload', 'PackageLoader',
               'show_config'] + core.__all__ + _mat.__all__ + lib.__all__

    ''')

astroid.register_module_extender(astroid.MANAGER, 'numpy.core', numpy_core_transform)
astroid.register_module_extender(astroid.MANAGER, 'numpy', numpy_transform)
<commit_msg>Expose more objects in numpy brain tip<commit_after># Copyright (c) 2015-2016 Claudiu Popa <pcmanticore@gmail.com>

# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER

"""Astroid hooks for numpy."""

import astroid

# TODO(cpopa): drop when understanding augmented assignments

def numpy_core_transform():
    return astroid.parse('''
    from numpy.core import numeric
    from numpy.core import fromnumeric
    from numpy.core import defchararray
    from numpy.core import records
    from numpy.core import function_base
    from numpy.core import machar
    from numpy.core import getlimits
    from numpy.core import shape_base
    from numpy.core.umath import (
        absolute, add, arccos, arccosh, arcsin, arcsinh,
        arctan, arctan2, arctanh, bitwise_and, bitwise_or,
        bitwise_xor, cbrt, ceil, conj, conjugate, copysign,
        cos, cosh, deg2rad, degrees, divide, e, equal,
        euler_gamma, exp, exp2, expm1, fabs, float_power,
        floor, floor_divide, fmax, fmin, fmod, frexp,
        frompyfunc, geterrobj, greater, greater_equal,
        hypot, invert, isfinite, isinf, isnan, ldexp,
        left_shift, less, less_equal, log, log10, log1p,
        log2, logaddexp, logaddexp2, logical_and, logical_not,
        logical_or, logical_xor, maximum, minimum, mod, modf,
        multiply, negative, nextafter, not_equal, pi, power,
        rad2deg, radians, reciprocal, remainder, right_shift,
        rint, seterrobj, sign, signbit, sin, sinh, spacing,
        sqrt, square, subtract, tan, tanh, true_divide, trunc)
    __all__ = (['char', 'rec', 'memmap', 'chararray'] + numeric.__all__ +
               fromnumeric.__all__ +
               records.__all__ +
               function_base.__all__ +
               machar.__all__ +
               getlimits.__all__ +
               shape_base.__all__)
    ''')

def numpy_transform():
    return astroid.parse('''
    from numpy import core
    from numpy import matrixlib as _mat
    from numpy import lib
    __all__ = ['add_newdocs',
               'ModuleDeprecationWarning',
               'VisibleDeprecationWarning', 'linalg', 'fft', 'random',
               'ctypeslib', 'ma',
               '__version__', 'pkgload', 'PackageLoader',
               'show_config'] + core.__all__ + _mat.__all__ + lib.__all__

    ''')

astroid.register_module_extender(astroid.MANAGER, 'numpy.core', numpy_core_transform)
astroid.register_module_extender(astroid.MANAGER, 'numpy', numpy_transform)
<|endoftext|>


<commit_before># Copyright 2012 Managed I.T.
#
# Author: Kiall Mac Innes <kiall@managedit.ie>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os

# Eventlet's GreenDNS Patching will prevent the resolution of names in
# the /etc/hosts file, causing problems for installs.
os.environ['EVENTLET_NO_GREENDNS'] = 'yes'

import socket

from oslo_config import cfg
from oslo_log import log
from oslo_concurrency import lockutils
import oslo_messaging as messaging

designate_opts = [
    cfg.StrOpt('host', default=socket.gethostname(),
               sample_default='current_hostname',
               help='Name of this node'),
    cfg.StrOpt(
        'pybasedir',
        sample_default='<Path>',
        default=os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             '../')),
        help='Directory where the designate python module is installed'
    ),
    cfg.StrOpt('state-path', default='/var/lib/designate',
               help='Top-level directory for maintaining designate\'s state'),

    cfg.StrOpt('central-topic', default='central', help='Central Topic'),
    cfg.StrOpt('mdns-topic', default='mdns', help='mDNS Topic'),
    cfg.StrOpt('pool-manager-topic', default='pool_manager',
               help='Pool Manager Topic'),
    cfg.StrOpt('worker-topic', default='worker', help='Worker Topic'),

    # Default TTL
    cfg.IntOpt('default-ttl', default=3600, help='TTL Value'),

    # Default SOA Values
    cfg.IntOpt('default-soa-refresh-min', default=3500,
               deprecated_name='default-soa-refresh',
               help='SOA refresh-min value'),
    cfg.IntOpt('default-soa-refresh-max', default=3600,
               help='SOA max value'),
    cfg.IntOpt('default-soa-retry', default=600, help='SOA retry'),
    cfg.IntOpt('default-soa-expire', default=86400, help='SOA expire'),
    cfg.IntOpt('default-soa-minimum', default=3600, help='SOA minimum value'),

    # Supported record types
    cfg.ListOpt('supported-record-type', help='Supported record types',
                default=['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS',
                         'PTR', 'SSHFP', 'SOA']),
]

# Set some Oslo Log defaults
log.set_defaults(default_log_levels=[
    'amqplib=WARN',
    'amqp=WARN',
    'boto=WARN',
    'eventlet.wsgi.server=WARN',
    'iso8601=WARN',
    'kazoo.client=WARN',
    'keystone=INFO',
    'keystonemiddleware.auth_token=INFO',
    'oslo_messaging=WARN',
    'oslo.messaging=INFO',
    'oslo_service.loopingcall=WARN',
    'sqlalchemy=WARN',
    'stevedore=WARN',
    'suds=INFO',
])

# Set some Oslo RPC defaults
messaging.set_transport_defaults('designate')

# Set some Oslo Oslo Concurrency defaults
lockutils.set_defaults(lock_path='$state_path')

cfg.CONF.register_opts(designate_opts)
<commit_msg>Simplify olso.log setup<commit_after># Copyright 2012 Managed I.T.
#
# Author: Kiall Mac Innes <kiall@managedit.ie>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os

# Eventlet's GreenDNS Patching will prevent the resolution of names in
# the /etc/hosts file, causing problems for installs.
os.environ['EVENTLET_NO_GREENDNS'] = 'yes'

import socket

from oslo_config import cfg
from oslo_log import log
from oslo_concurrency import lockutils
import oslo_messaging as messaging

_EXTRA_DEFAULT_LOG_LEVELS = [
    'eventlet.wsgi.server=WARN',
    'kazoo.client=WARN',
    'keystone=INFO',
    'oslo_service.loopingcall=WARN',
]

designate_opts = [
    cfg.StrOpt('host', default=socket.gethostname(),
               sample_default='current_hostname',
               help='Name of this node'),
    cfg.StrOpt(
        'pybasedir',
        sample_default='<Path>',
        default=os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             '../')),
        help='Directory where the designate python module is installed'
    ),
    cfg.StrOpt('state-path', default='/var/lib/designate',
               help='Top-level directory for maintaining designate\'s state'),

    cfg.StrOpt('central-topic', default='central', help='Central Topic'),
    cfg.StrOpt('mdns-topic', default='mdns', help='mDNS Topic'),
    cfg.StrOpt('pool-manager-topic', default='pool_manager',
               help='Pool Manager Topic'),
    cfg.StrOpt('worker-topic', default='worker', help='Worker Topic'),

    # Default TTL
    cfg.IntOpt('default-ttl', default=3600, help='TTL Value'),

    # Default SOA Values
    cfg.IntOpt('default-soa-refresh-min', default=3500,
               deprecated_name='default-soa-refresh',
               help='SOA refresh-min value'),
    cfg.IntOpt('default-soa-refresh-max', default=3600,
               help='SOA max value'),
    cfg.IntOpt('default-soa-retry', default=600, help='SOA retry'),
    cfg.IntOpt('default-soa-expire', default=86400, help='SOA expire'),
    cfg.IntOpt('default-soa-minimum', default=3600, help='SOA minimum value'),

    # Supported record types
    cfg.ListOpt('supported-record-type', help='Supported record types',
                default=['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS',
                         'PTR', 'SSHFP', 'SOA']),
]

# Set some Oslo Log defaults
log.set_defaults(default_log_levels=log.get_default_log_levels() +
                 _EXTRA_DEFAULT_LOG_LEVELS)

# Set some Oslo RPC defaults
messaging.set_transport_defaults('designate')

# Set some Oslo Oslo Concurrency defaults
lockutils.set_defaults(lock_path='$state_path')

cfg.CONF.register_opts(designate_opts)
<|endoftext|>


<commit_before>'''
The MIT License (MIT)

Copyright (c) 2014 Ilya Volkov

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

__all__ = ["make",]

def _store_index(a, x, v):
    lo = 0
    hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if a[mid][0] < x: lo = mid+1
        else: hi = mid
    if lo < len(a) and a[lo][0] == x:
        a[lo][1].append(v)
    else:
        a.insert(lo, (x, [v]))

def _take_index(a, x):
    lo = 0
    hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if a[mid][0] < x: lo = mid+1
        else: hi = mid
    if lo < len(a) and a[lo][0] == x:
        if a[lo][1]:
            return a[lo][1].pop()
    return None

class _compare_info(object):
    def __init__(self):
        self.removed = []
        self.added   = []
        self.__root = root = []
        root[:] = [root, root, None]

    def insert(self, op):
        root = self.__root
        last = root[0]
        last[1] = root[0] = [last, root, op]
        return root[0]

    def remove(self, index):
        link_prev, link_next, _ = index
        link_prev[1] = link_next
        link_next[0] = link_prev
        index[:] = []

    def iter_from(self, start):
        root = self.__root
        curr = start[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def __iter__(self):
        root = self.__root
        curr = root[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def execute(self):
        root = self.__root
        curr = root[1]
        while curr is not root:
            if curr[1] is not root:
                op_first, op_second = curr[2], curr[1][2]
                if op_first.key == op_second.key and \
                        op_first.path == op_second.path and \
                        type(op_first) == _op_remove and \
                        type(op_second) == _op_add:
                    yield _op_replace(op_second.path, op_second.key, op_second.value).get()
                    curr = curr[1][1]
                    continue
            yield curr[2].get()
            curr = curr[1]

class _op_base(object):
    def __init__(self, path, key, value):
        self.path  = path
        self.key   = key
        self.value = value

    def __repr__(self):
        return str(self.get())

class _op_add(_op_base):
    def _on_undo_remove(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key += 1
            else:
                key += 1
        return key

    def _on_undo_add(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key += 1
        return key

    def get(self):
        return {'op': 'add', 'path': _path_join(self.path, self.key), 'value': self.value}

class _op_remove(_op_base):
    def _on_undo_remove(self, path, key):
        if self.path == path:
            if self.key >= key:
                self.key += 1
            else:
                key -= 1
        return key

    def _on_undo_add(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key -= 1
        return key

    def get(self):
        return {'op': 'remove', 'path': _path_join(self.path, self.key)}

class _op_replace(_op_base):
    def _on_undo_remove(self, path, key):
        return key

    def _on_undo_add(self, path, key):
        return key

    def get(self):
        return {'op': 'replace', 'path': _path_join(self.path, self.key), 'value': self.value}

class _op_move(object):
    def __init__(self, oldpath, oldkey, path, key):
        self.oldpath = oldpath
        self.oldkey  = oldkey
        self.path    = path
        self.key     = key

    def _on_undo_remove(self, path, key):
        if self.oldpath == path:
            if self.oldkey >= key:
                self.oldkey += 1
            else:
                key -= 1
        if self.path == path:
            if self.key > key:
                self.key += 1
            else:
                key += 1
        return key

    def _on_undo_add(self, path, key):
        if self.oldpath == path:
            if self.oldkey > key:
                self.oldkey -= 1
            else:
                key -= 1
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key += 1
        return key

    def get(self):
        return {'op': 'move', 'path': _path_join(self.path, self.key), 'from': _path_join(self.oldpath, self.oldkey)}

    def __repr__(self):
        return str(self.get())

def _path_join(path, key):
    if key != None:
        return path + '/' + str(key).replace('~', '~0').replace('/', '~1')
    return path

def _item_added(path, key, info, item):
    index = _take_index(info.removed, item)
    if index != None:
        op = index[2]
        if type(op.key) == int:
            for v in info.iter_from(index):
                op.key = v._on_undo_remove(op.path, op.key)
        info.remove(index)
        if op.path != path or op.key != key:
            new_op = _op_move(op.path, op.key, path, key)
            info.insert(new_op)
    else:
        new_op = _op_add(path, key, item)
        new_index = info.insert(new_op)
        _store_index(info.added, item, new_index)

def _item_removed(path, key, info, item):
    new_op = _op_remove(path, key, item)
    index = _take_index(info.added, item)
    new_index = info.insert(new_op)
    if index != None:
        op = index[2]
        if type(op.key) == int:
            for v in info.iter_from(index):
                op.key = v._on_undo_add(op.path, op.key)
        info.remove(index)
        if new_op.path != op.path or new_op.key != op.key:
            new_op = _op_move(new_op.path, new_op.key, op.path, op.key)
            new_index[2] = new_op
        else:
            info.remove(new_index)
    else:
        _store_index(info.removed, item, new_index)

def _item_replaced(path, key, info, item):
    info.insert(_op_replace(path, key, item))

def _compare_dicts(path, info, src, dst):
    added_keys = dst.viewkeys() - src.viewkeys()
    removed_keys = src.viewkeys() - dst.viewkeys()
    for key in removed_keys:
        _item_removed(path, str(key), info, src[key])
    for key in added_keys:
        _item_added(path, str(key), info, dst[key])
    for key in src.viewkeys() & dst.viewkeys():
        _compare_values(path, key, info, src[key], dst[key])

def _compare_lists(path, info, src, dst):
    len_src, len_dst = len(src), len(dst)
    max_len = max(len_src, len_dst)
    min_len = min(len_src, len_dst)
    for key in xrange(max_len):
        if key < min_len:
            old, new = src[key], dst[key]
            if old == new:
                continue
            _item_removed(path, key, info, old)
            _item_added(path, key, info, new)
        elif len_src > len_dst:
            _item_removed(path, len_dst, info, src[key])
        else:
            _item_added(path, key, info, dst[key])

def _compare_values(path, key, info, src, dst):
    if src == dst:
        return
    elif isinstance(src, dict) and \
            isinstance(dst, dict):
        _compare_dicts(_path_join(path, key), info, src, dst)
    elif isinstance(src, list) and \
            isinstance(dst, list):
        _compare_lists(_path_join(path, key), info, src, dst)
    else:
        _item_replaced(path, key, info, dst)

def make(src, dst, **kwargs):
    info = _compare_info()
    _compare_values('', None, info, src, dst)
    return [op for op in info.execute()]
<commit_msg>Remove trailing whitespace<commit_after>'''
The MIT License (MIT)

Copyright (c) 2014 Ilya Volkov

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

__all__ = ["make",]

def _store_index(a, x, v):
    lo = 0
    hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if a[mid][0] < x: lo = mid+1
        else: hi = mid
    if lo < len(a) and a[lo][0] == x:
        a[lo][1].append(v)
    else:
        a.insert(lo, (x, [v]))

def _take_index(a, x):
    lo = 0
    hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if a[mid][0] < x: lo = mid+1
        else: hi = mid
    if lo < len(a) and a[lo][0] == x:
        if a[lo][1]:
            return a[lo][1].pop()
    return None

class _compare_info(object):
    def __init__(self):
        self.removed = []
        self.added   = []
        self.__root = root = []
        root[:] = [root, root, None]

    def insert(self, op):
        root = self.__root
        last = root[0]
        last[1] = root[0] = [last, root, op]
        return root[0]

    def remove(self, index):
        link_prev, link_next, _ = index
        link_prev[1] = link_next
        link_next[0] = link_prev
        index[:] = []

    def iter_from(self, start):
        root = self.__root
        curr = start[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def __iter__(self):
        root = self.__root
        curr = root[1]
        while curr is not root:
            yield curr[2]
            curr = curr[1]

    def execute(self):
        root = self.__root
        curr = root[1]
        while curr is not root:
            if curr[1] is not root:
                op_first, op_second = curr[2], curr[1][2]
                if op_first.key == op_second.key and \
                        op_first.path == op_second.path and \
                        type(op_first) == _op_remove and \
                        type(op_second) == _op_add:
                    yield _op_replace(op_second.path, op_second.key, op_second.value).get()
                    curr = curr[1][1]
                    continue
            yield curr[2].get()
            curr = curr[1]

class _op_base(object):
    def __init__(self, path, key, value):
        self.path  = path
        self.key   = key
        self.value = value

    def __repr__(self):
        return str(self.get())

class _op_add(_op_base):
    def _on_undo_remove(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key += 1
            else:
                key += 1
        return key

    def _on_undo_add(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key += 1
        return key

    def get(self):
        return {'op': 'add', 'path': _path_join(self.path, self.key), 'value': self.value}

class _op_remove(_op_base):
    def _on_undo_remove(self, path, key):
        if self.path == path:
            if self.key >= key:
                self.key += 1
            else:
                key -= 1
        return key

    def _on_undo_add(self, path, key):
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key -= 1
        return key

    def get(self):
        return {'op': 'remove', 'path': _path_join(self.path, self.key)}

class _op_replace(_op_base):
    def _on_undo_remove(self, path, key):
        return key

    def _on_undo_add(self, path, key):
        return key

    def get(self):
        return {'op': 'replace', 'path': _path_join(self.path, self.key), 'value': self.value}

class _op_move(object):
    def __init__(self, oldpath, oldkey, path, key):
        self.oldpath = oldpath
        self.oldkey  = oldkey
        self.path    = path
        self.key     = key

    def _on_undo_remove(self, path, key):
        if self.oldpath == path:
            if self.oldkey >= key:
                self.oldkey += 1
            else:
                key -= 1
        if self.path == path:
            if self.key > key:
                self.key += 1
            else:
                key += 1
        return key

    def _on_undo_add(self, path, key):
        if self.oldpath == path:
            if self.oldkey > key:
                self.oldkey -= 1
            else:
                key -= 1
        if self.path == path:
            if self.key > key:
                self.key -= 1
            else:
                key += 1
        return key

    def get(self):
        return {'op': 'move', 'path': _path_join(self.path, self.key), 'from': _path_join(self.oldpath, self.oldkey)}

    def __repr__(self):
        return str(self.get())

def _path_join(path, key):
    if key != None:
        return path + '/' + str(key).replace('~', '~0').replace('/', '~1')
    return path

def _item_added(path, key, info, item):
    index = _take_index(info.removed, item)
    if index != None:
        op = index[2]
        if type(op.key) == int:
            for v in info.iter_from(index):
                op.key = v._on_undo_remove(op.path, op.key)
        info.remove(index)
        if op.path != path or op.key != key:
            new_op = _op_move(op.path, op.key, path, key)
            info.insert(new_op)
    else:
        new_op = _op_add(path, key, item)
        new_index = info.insert(new_op)
        _store_index(info.added, item, new_index)

def _item_removed(path, key, info, item):
    new_op = _op_remove(path, key, item)
    index = _take_index(info.added, item)
    new_index = info.insert(new_op)
    if index != None:
        op = index[2]
        if type(op.key) == int:
            for v in info.iter_from(index):
                op.key = v._on_undo_add(op.path, op.key)
        info.remove(index)
        if new_op.path != op.path or new_op.key != op.key:
            new_op = _op_move(new_op.path, new_op.key, op.path, op.key)
            new_index[2] = new_op
        else:
            info.remove(new_index)
    else:
        _store_index(info.removed, item, new_index)

def _item_replaced(path, key, info, item):
    info.insert(_op_replace(path, key, item))

def _compare_dicts(path, info, src, dst):
    added_keys = dst.viewkeys() - src.viewkeys()
    removed_keys = src.viewkeys() - dst.viewkeys()
    for key in removed_keys:
        _item_removed(path, str(key), info, src[key])
    for key in added_keys:
        _item_added(path, str(key), info, dst[key])
    for key in src.viewkeys() & dst.viewkeys():
        _compare_values(path, key, info, src[key], dst[key])

def _compare_lists(path, info, src, dst):
    len_src, len_dst = len(src), len(dst)
    max_len = max(len_src, len_dst)
    min_len = min(len_src, len_dst)
    for key in xrange(max_len):
        if key < min_len:
            old, new = src[key], dst[key]
            if old == new:
                continue
            _item_removed(path, key, info, old)
            _item_added(path, key, info, new)
        elif len_src > len_dst:
            _item_removed(path, len_dst, info, src[key])
        else:
            _item_added(path, key, info, dst[key])

def _compare_values(path, key, info, src, dst):
    if src == dst:
        return
    elif isinstance(src, dict) and \
            isinstance(dst, dict):
        _compare_dicts(_path_join(path, key), info, src, dst)
    elif isinstance(src, list) and \
            isinstance(dst, list):
        _compare_lists(_path_join(path, key), info, src, dst)
    else:
        _item_replaced(path, key, info, dst)

def make(src, dst, **kwargs):
    info = _compare_info()
    _compare_values('', None, info, src, dst)
    return [op for op in info.execute()]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Jan 26 16:42:47 2015

@author: htelg
"""
import numpy as np
import pandas as pd
from atmPy import sizedistribution
from atmPy.instruments.tools import dia

def bincenters2BinStuff(bincenters):
    if type(bincenters) != np.ndarray:
        raise TypeError('Parameter "bincenters" has to be numpy.ndarray. Given object is %s'%type(bincenters))
    noEnds = (bincenters[:-1]+bincenters[1:])/2.
    firstEdge = bincenters[0] - (noEnds[0]-bincenters[0])
    lastEdge = bincenters[-1] + (noEnds[-1]-bincenters[-1])
    binedges = np.append(firstEdge,noEnds)
    binedges = np.append(binedges,lastEdge)
    minusses = binedges[:-1].copy().astype(str)
    minusses[:] = ' - '
    a = binedges[:-1].astype(str)
    b = binedges[1:].astype(str)
    newColnames = np.core.defchararray.add(a,minusses)
    newColnames = np.core.defchararray.add(newColnames,b)
    return binedges,newColnames

def load_PMEL_APS(fname):
    na_values = [u'StartDateTime', u'Dp_1', u'Dp_2', u'Dp_3', u'Dp_4', u'Dp_5', u'Dp_6', u'Dp_7', u'Dp_8', u'Dp_9', u'Dp_10', u'Dp_11', u'Dp_12', u'Dp_13', u'Dp_14', u'Dp_15', u'Dp_16', u'Dp_17', u'Dp_18', u'Dp_19', u'Dp_20', u'Dp_21', u'Dp_22', u'Dp_23', u'Dp_24', u'Dp_25', u'Dp_26', u'Dp_27', u'Dp_28', u'Dp_29', u'Dp_30', u'Dp_31', u'Dp_32', u'Dp_33', u'Dp_34', u'Dp_35', u'Dp_36', u'Dp_37', u'Dp_38', u'Dp_39', u'Dp_40', u'Dp_41', u'Dp_42', u'Dp_43', u'Dp_44', u'Dp_45', u'Dp_46', u'Dp_47', u'Dp_48', u'Dp_49', u'Dp_50', u'Dp_51', u'Dp_52',u'dNdlogDp_1', u'dNdlogDp_2', u'dNdlogDp_3', u'dNdlogDp_4', u'dNdlogDp_5', u'dNdlogDp_6', u'dNdlogDp_7', u'dNdlogDp_8', u'dNdlogDp_9', u'dNdlogDp_10', u'dNdlogDp_11', u'dNdlogDp_12', u'dNdlogDp_13', u'dNdlogDp_14', u'dNdlogDp_15', u'dNdlogDp_16', u'dNdlogDp_17', u'dNdlogDp_18', u'dNdlogDp_19', u'dNdlogDp_20', u'dNdlogDp_21', u'dNdlogDp_22', u'dNdlogDp_23', u'dNdlogDp_24', u'dNdlogDp_25', u'dNdlogDp_26', u'dNdlogDp_27', u'dNdlogDp_28', u'dNdlogDp_29', u'dNdlogDp_30', u'dNdlogDp_31', u'dNdlogDp_32', u'dNdlogDp_33', u'dNdlogDp_34', u'dNdlogDp_35', u'dNdlogDp_36', u'dNdlogDp_37', u'dNdlogDp_38', u'dNdlogDp_39', u'dNdlogDp_40', u'dNdlogDp_41', u'dNdlogDp_42', u'dNdlogDp_43', u'dNdlogDp_44', u'dNdlogDp_45', u'dNdlogDp_46', u'dNdlogDp_47', u'dNdlogDp_48', u'dNdlogDp_49', u'dNdlogDp_50', u'dNdlogDp_51', u'dNdlogDp_52']
    tab = pd.read_csv(fname, sep = '\t', na_values=na_values)
    tab = tab.dropna()
    newIndex = pd.to_datetime(tab.StartDateTime.values)
    tab.index = newIndex
    reducedTab = tab.iloc[:,53:]
    bincenters = tab.iloc[0,1:53].values*1000
    binedges,newColnames = bincenters2BinStuff(bincenters)
    reducedTab.columns = newColnames
    dist = sizedistribution.aerosolSizeDistribution(reducedTab,binedges, 'dNdlogDp' )
    return dist<commit_msg>out sourced the binning<commit_after># -*- coding: utf-8 -*-
"""
This module is out of date!

@author: htelg
"""
import numpy as np
import pandas as pd
from atmPy import sizedistribution
from atmPy.instruments.tools import diameter_binning

def load_PMEL_APS(fname):
    na_values = [u'StartDateTime', u'Dp_1', u'Dp_2', u'Dp_3', u'Dp_4', u'Dp_5', u'Dp_6', u'Dp_7', u'Dp_8', u'Dp_9', u'Dp_10', u'Dp_11', u'Dp_12', u'Dp_13', u'Dp_14', u'Dp_15', u'Dp_16', u'Dp_17', u'Dp_18', u'Dp_19', u'Dp_20', u'Dp_21', u'Dp_22', u'Dp_23', u'Dp_24', u'Dp_25', u'Dp_26', u'Dp_27', u'Dp_28', u'Dp_29', u'Dp_30', u'Dp_31', u'Dp_32', u'Dp_33', u'Dp_34', u'Dp_35', u'Dp_36', u'Dp_37', u'Dp_38', u'Dp_39', u'Dp_40', u'Dp_41', u'Dp_42', u'Dp_43', u'Dp_44', u'Dp_45', u'Dp_46', u'Dp_47', u'Dp_48', u'Dp_49', u'Dp_50', u'Dp_51', u'Dp_52',u'dNdlogDp_1', u'dNdlogDp_2', u'dNdlogDp_3', u'dNdlogDp_4', u'dNdlogDp_5', u'dNdlogDp_6', u'dNdlogDp_7', u'dNdlogDp_8', u'dNdlogDp_9', u'dNdlogDp_10', u'dNdlogDp_11', u'dNdlogDp_12', u'dNdlogDp_13', u'dNdlogDp_14', u'dNdlogDp_15', u'dNdlogDp_16', u'dNdlogDp_17', u'dNdlogDp_18', u'dNdlogDp_19', u'dNdlogDp_20', u'dNdlogDp_21', u'dNdlogDp_22', u'dNdlogDp_23', u'dNdlogDp_24', u'dNdlogDp_25', u'dNdlogDp_26', u'dNdlogDp_27', u'dNdlogDp_28', u'dNdlogDp_29', u'dNdlogDp_30', u'dNdlogDp_31', u'dNdlogDp_32', u'dNdlogDp_33', u'dNdlogDp_34', u'dNdlogDp_35', u'dNdlogDp_36', u'dNdlogDp_37', u'dNdlogDp_38', u'dNdlogDp_39', u'dNdlogDp_40', u'dNdlogDp_41', u'dNdlogDp_42', u'dNdlogDp_43', u'dNdlogDp_44', u'dNdlogDp_45', u'dNdlogDp_46', u'dNdlogDp_47', u'dNdlogDp_48', u'dNdlogDp_49', u'dNdlogDp_50', u'dNdlogDp_51', u'dNdlogDp_52']
    tab = pd.read_csv(fname, sep = '\t', na_values=na_values)
    tab = tab.dropna()
    newIndex = pd.to_datetime(tab.StartDateTime.values)
    tab.index = newIndex
    reducedTab = tab.iloc[:,53:]
    bincenters = tab.iloc[0,1:53].values*1000
    binedges,newColnames = diameter_binning.bincenters2binsANDnames(bincenters)
    reducedTab.columns = newColnames
    dist = sizedistribution.aerosolSizeDistribution(reducedTab,binedges, 'dNdlogDp' )
    return dist<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""This file is part of DINGO, the DIstribution Network GeneratOr.
DINGO is a tool to generate synthetic medium and low voltage power
distribution grids based on open data.

It is developed in the project open_eGo: https://openegoproject.wordpress.com

DINGO lives at github: https://github.com/openego/dingo/
The documentation is available on RTD: http://dingo.readthedocs.io"""

__copyright__  = "Reiner Lemoine Institut gGmbH"
__license__    = "GNU Affero General Public License Version 3 (AGPL-3.0)"
__url__        = "https://github.com/openego/dingo/blob/master/LICENSE"
__author__     = "nesnoj, gplssm"

import matplotlib.pyplot as plt
# import oemof.db as db
import time
import os
import pandas as pd

from dingo.core import NetworkDingo
from dingo.tools import config as cfg_dingo, results, db

plt.close('all')
cfg_dingo.load_config('config_db_tables.cfg')
cfg_dingo.load_config('config_calc.cfg')
cfg_dingo.load_config('config_files.cfg')
cfg_dingo.load_config('config_misc.cfg')

BASEPATH = os.path.join(os.path.expanduser('~'), '.dingo')

def create_results_dirs(base_path):
    """Create base path dir and subdirectories

    Parameters
    ----------
    base_path : str
        The base path has subdirectories for raw and processed results
    """

    if not os.path.exists(base_path):
        print("Creating directory {} for results data.".format(base_path))
        os.mkdir(base_path)
    if not os.path.exists(os.path.join(base_path, 'results')):
        os.mkdir(os.path.join(base_path, 'results'))
    if not os.path.exists(os.path.join(base_path, 'plots')):
        os.mkdir(os.path.join(base_path, 'plots'))
    if not os.path.exists(os.path.join(base_path, 'info')):
        os.mkdir(os.path.join(base_path, 'info'))
    if not os.path.exists(os.path.join(base_path, 'log')):
        os.mkdir(os.path.join(base_path, 'log'))

def run_multiple_grid_districts(mv_grid_districts, failsafe=False, base_path=None):
    """
    Perform dingo run on given grid districts

    Parameters
    ----------
    mv_grid_districs : list
        Integers describing grid districts
    failsafe : bool
        Setting to True enables failsafe mode where corrupt grid districts
        (mostly due to data issues) are reported and skipped. Report is to be
         found in the log dir under :code:`~/.dingo` . Default is False.
    base_path : str
        Base path for dingo data (input, results and logs).
        Default is `None` which sets it to :code:`~/.dingo` (may deviate on
        windows systems).
        Specify your own but keep in mind that it a required a particular
        structure of subdirectories.

    Returns
    -------
    msg : str
        Traceback of error computing corrupt MV grid district
        .. TODO: this is only true if try-except environment is moved into this
            fundion and traceback return is implemented

    Notes
    -----
    Consider that a large amount of MV grid districts may take hours or up to
    days to compute. A computational run for a single grid district may consume
    around 30 secs.
    """
    start = time.time()

    # define base path
    if base_path is None:
        base_path = BASEPATH

    # database connection
    conn = db.connection(section='oedb')

    corrupt_grid_districts = pd.DataFrame(columns=['id', 'message'])

    for mvgd in mv_grid_districts:
        # instantiate dingo  network object
        nd = NetworkDingo(name='network')

        if not failsafe:
            # run DINGO on selected MV Grid District
            msg = nd.run_dingo(conn=conn,
                         mv_grid_districts_no=[mvgd])

            # save results
            results.save_nd_to_pickle(nd, os.path.join(base_path, 'results'))
        else:
            # try to perform dingo run on grid district
            try:
                msg = nd.run_dingo(conn=conn,
                         mv_grid_districts_no=[mvgd])
                # if not successful, put grid district to report
                if msg:
                    corrupt_grid_districts = corrupt_grid_districts.append(
                        pd.Series({'id': mvgd,
                                   'message': msg[0]}),
                        ignore_index=True)
                # if successful, save results
                else:
                    results.save_nd_to_pickle(nd, os.path.join(base_path,
                                                               'results'))
            except Exception as e:
                corrupt_grid_districts = corrupt_grid_districts.append(
                    pd.Series({'id': mvgd,
                               'message': e}),
                    ignore_index=True)

                continue

        # Merge metadata of multiple runs
        if 'metadata' not in locals():
            metadata = nd.metadata
        else:
            metadata['mv_grid_districts'].extend(mvgd)

    # report on unsuccessful runs
    corrupt_grid_districts.to_csv(
        os.path.join(
            base_path,
            'info',
            'corrupt_mv_grid_districts.txt'),
        index=False,
        float_format='%.0f')

    conn.close()
    print('Elapsed time for', str(len(mv_grid_districts)),
          'MV grid districts (seconds): {}'.format(time.time() - start))

    return msg

if __name__ == '__main__':

    # create directories for local results data
    create_results_dirs(BASEPATH)

    # define grid district by its id (int)
    mv_grid_districts = list(range(1, 3608))

    # run grid districts
    run_multiple_grid_districts(mv_grid_districts, failsafe=True, base_path='/home/guido/git-repos/dingo/tmp_dingo')
<commit_msg>Save results to run_id subdirectory<commit_after>#!/usr/bin/env python3

"""This file is part of DINGO, the DIstribution Network GeneratOr.
DINGO is a tool to generate synthetic medium and low voltage power
distribution grids based on open data.

It is developed in the project open_eGo: https://openegoproject.wordpress.com

DINGO lives at github: https://github.com/openego/dingo/
The documentation is available on RTD: http://dingo.readthedocs.io"""

__copyright__  = "Reiner Lemoine Institut gGmbH"
__license__    = "GNU Affero General Public License Version 3 (AGPL-3.0)"
__url__        = "https://github.com/openego/dingo/blob/master/LICENSE"
__author__     = "nesnoj, gplssm"

import matplotlib.pyplot as plt
# import oemof.db as db
import time
import os
import pandas as pd

from dingo.core import NetworkDingo
from dingo.tools import config as cfg_dingo, results, db
import json

plt.close('all')
cfg_dingo.load_config('config_db_tables.cfg')
cfg_dingo.load_config('config_calc.cfg')
cfg_dingo.load_config('config_files.cfg')
cfg_dingo.load_config('config_misc.cfg')

BASEPATH = os.path.join(os.path.expanduser('~'), '.dingo')

def create_results_dirs(base_path):
    """Create base path dir and subdirectories

    Parameters
    ----------
    base_path : str
        The base path has subdirectories for raw and processed results
    """

    if not os.path.exists(base_path):
        print("Creating directory {} for results data.".format(base_path))
        os.mkdir(base_path)
    if not os.path.exists(os.path.join(base_path, 'results')):
        os.mkdir(os.path.join(base_path, 'results'))
    if not os.path.exists(os.path.join(base_path, 'plots')):
        os.mkdir(os.path.join(base_path, 'plots'))
    if not os.path.exists(os.path.join(base_path, 'info')):
        os.mkdir(os.path.join(base_path, 'info'))
    if not os.path.exists(os.path.join(base_path, 'log')):
        os.mkdir(os.path.join(base_path, 'log'))

def run_multiple_grid_districts(mv_grid_districts, failsafe=False, base_path=None):
    """
    Perform dingo run on given grid districts

    Parameters
    ----------
    mv_grid_districs : list
        Integers describing grid districts
    failsafe : bool
        Setting to True enables failsafe mode where corrupt grid districts
        (mostly due to data issues) are reported and skipped. Report is to be
         found in the log dir under :code:`~/.dingo` . Default is False.
    base_path : str
        Base path for dingo data (input, results and logs).
        Default is `None` which sets it to :code:`~/.dingo` (may deviate on
        windows systems).
        Specify your own but keep in mind that it a required a particular
        structure of subdirectories.

    Returns
    -------
    msg : str
        Traceback of error computing corrupt MV grid district
        .. TODO: this is only true if try-except environment is moved into this
            fundion and traceback return is implemented

    Notes
    -----
    Consider that a large amount of MV grid districts may take hours or up to
    days to compute. A computational run for a single grid district may consume
    around 30 secs.
    """
    start = time.time()

    # define base path
    if base_path is None:
        base_path = BASEPATH

    # database connection
    conn = db.connection(section='oedb')

    corrupt_grid_districts = pd.DataFrame(columns=['id', 'message'])

    for mvgd in mv_grid_districts:
        # instantiate dingo  network object
        nd = NetworkDingo(name='network')

        run_id = nd.metadata['run_id']

        if not os.path.exists(os.path.join(base_path, run_id)):
            os.mkdir(os.path.join(base_path, run_id))

        if not failsafe:
            # run DINGO on selected MV Grid District
            msg = nd.run_dingo(conn=conn,
                         mv_grid_districts_no=[mvgd])

            # save results
            results.save_nd_to_pickle(nd, os.path.join(base_path, run_id))
        else:
            # try to perform dingo run on grid district
            try:
                msg = nd.run_dingo(conn=conn,
                         mv_grid_districts_no=[mvgd])
                # if not successful, put grid district to report
                if msg:
                    corrupt_grid_districts = corrupt_grid_districts.append(
                        pd.Series({'id': mvgd,
                                   'message': msg[0]}),
                        ignore_index=True)
                # if successful, save results
                else:
                    results.save_nd_to_pickle(nd, os.path.join(base_path,
                                                               run_id))
            except Exception as e:
                corrupt_grid_districts = corrupt_grid_districts.append(
                    pd.Series({'id': mvgd,
                               'message': e}),
                    ignore_index=True)

                continue

        # Merge metadata of multiple runs
        if 'metadata' not in locals():
            metadata = nd.metadata
        else:
            if isinstance(mvgd, list):
                metadata['mv_grid_districts'].extend(mvgd)
            else:
                metadata['mv_grid_districts'].append(mvgd)

    # Save metadata to disk
    with open(os.path.join(base_path, run_id), 'w') as f:
        json.dump(metadata, f)

    # report on unsuccessful runs
    corrupt_grid_districts.to_csv(
        os.path.join(
            base_path,
            run_id,
            'corrupt_mv_grid_districts.txt'),
        index=False,
        float_format='%.0f')

    conn.close()
    print('Elapsed time for', str(len(mv_grid_districts)),
          'MV grid districts (seconds): {}'.format(time.time() - start))

    return msg

if __name__ == '__main__':
    base_path='/home/guido/mnt/rli-daten/Dingo/'

    # create directories for local results data
    create_results_dirs(base_path)

    # define grid district by its id (int)
    mv_grid_districts = list(range(1729, 1732))

    # run grid districts
    run_multiple_grid_districts(mv_grid_districts, failsafe=True, base_path=base_path)
<|endoftext|>