<commit_before>#
# See top-level LICENSE.rst file for Copyright information
#
# -*- coding: utf-8 -*-
"""
desispec.pipeline.run
=====================

Tools for running the pipeline.
"""

from __future__ import absolute_import, division, print_function

import os
import errno
import sys
import subprocess as sp

from desispec.log import get_logger

def pid_exists( pid ):
    """Check whether pid exists in the current process table.

    **UNIX only.**  Should work the same as psutil.pid_exists().

    Args:
        pid (int): A process ID.

    Returns:
        pid_exists (bool): ``True`` if the process exists in the current process table.
    """
    if pid < 0:
        return False
    if pid == 0:
        # According to "man 2 kill" PID 0 refers to every process
        # in the process group of the calling process.
        # On certain systems 0 is a valid PID but we have no way
        # to know that in a portable fashion.
        raise ValueError('invalid PID 0')
    try:
        os.kill(pid, 0)
    except OSError as err:
        if err.errno == errno.ESRCH:
            # ESRCH == No such process
            return False
        elif err.errno == errno.EPERM:
            # EPERM clearly means there's a process to deny access to
            return True
        else:
            # According to "man 2 kill" possible error values are
            # (EINVAL, EPERM, ESRCH)
            raise
    else:
        return True

def subprocess_list(tasks, rank=0):
    log = get_logger()
    #pids = []
    for tsk in tasks:
        runcom = True
        newest_in = 0
        for dep in tsk['inputs']:
            if not os.path.isfile(dep):
                err = "dependency {} missing, cannot run task {}".format(dep, " ".join(tsk['command']))
                log.error(err)
                runcom = False
            else:
                t = os.path.getmtime(dep)
                if t > newest_in:
                    newest_in = t
        alldone = True
        if len(tsk['outputs']) == 0:
            alldone = False
        for outf in tsk['outputs']:
            if not os.path.isfile(outf):
                alldone = False
            else:
                t = os.path.getmtime(outf)
                if t < newest_in:
                    alldone = False
        if alldone:
            runcom = False
        proc = None
        if runcom:
            # proc = sp.Popen(tsk['command'], stdout=sp.PIPE, stderr=sp.STDOUT)
            # log.info("subproc[{}]: {}".format(proc.pid, " ".join(tsk['command'])))
            # outs, errs = proc.communicate()
            # for line in outs:
            #     log.debug("subproc[{}]:   {}".format(proc.pid, line.rstrip()))
            # proc.wait()
            log.info("subproc: {}".format(" ".join(tsk['command'])))
            ret = sp.call(tsk['command'])
            for outf in tsk['outputs']:
                ret = sp.call(['mv', '{}.part'.format(outf), outf])

    return

def shell_job(path, logroot, envsetup, desisetup, commands):
    with open(path, 'w') as f:
        f.write("#!/bin/bash\n\n")
        f.write("now=`date +%Y%m%d-%H:%M:%S`\n")
        f.write("log={}_${{now}}.log\n\n".format(logroot))
        for com in envsetup:
            f.write("{}\n".format(com))
        f.write("\n")
        f.write("source {}\n\n".format(desisetup))
        for com in commands:
            executable = com.split(' ')[0]
            f.write("which {}\n".format(executable))
            f.write("time {} >>${{log}} 2>&1\n\n".format(com))
    return

def nersc_job(path, logroot, envsetup, desisetup, commands, nodes=1, nodeproc=1, minutes=10, openmp=False, multiproc=False):
    hours = int(minutes/60)
    fullmin = int(minutes - 60*hours)
    timestr = "{:02d}:{:02d}:00".format(hours, fullmin)
    with open(path, 'w') as f:
        f.write("#!/bin/bash -l\n\n")
        f.write("#SBATCH --partition=debug\n")
        f.write("#SBATCH --account=desi\n")
        f.write("#SBATCH --nodes={}\n".format(nodes))
        f.write("#SBATCH --time={}\n".format(timestr))
        f.write("#SBATCH --job-name=desipipe\n")
        f.write("#SBATCH --output={}_slurm_%j.log\n".format(logroot))
        f.write("#SBATCH --export=NONE\n\n")
        for com in envsetup:
            f.write("{}\n".format(com))
        f.write("\n")
        f.write("source {}\n\n".format(desisetup))
        f.write("node_cores=0\n")
        f.write("if [ ${NERSC_HOST} = edison ]; then\n")
        f.write("  node_cores=24\n")
        f.write("else\n")
        f.write("  node_cores=32\n")
        f.write("fi\n")
        f.write("\n")
        f.write("nodes={}\n".format(nodes))
        f.write("node_proc={}\n".format(nodeproc))
        f.write("node_thread=$(( node_cores / node_proc ))\n")
        f.write("procs=$(( nodes * node_proc ))\n\n")
        if openmp:
            f.write("export OMP_NUM_THREADS=${node_thread}\n")
            f.write("\n")
        runstr = "srun --export=ALL"
        if multiproc:
            runstr = "{} --cpu_bind=no".format(runstr)
        f.write("run=\"{} -n ${{procs}} -N ${{nodes}} -c ${{node_thread}}\"\n\n".format(runstr))
        f.write("now=`date +%Y%m%d-%H:%M:%S`\n")
        f.write("echo \"job datestamp = ${now}\"\n")
        f.write("log={}_${{now}}.log\n\n".format(logroot))
        for com in commands:
            executable = com.split(' ')[0]
            f.write("which {}\n".format(executable))
            f.write("time ${{run}} {} >>${{log}} 2>&1\n\n".format(com))
    return

<commit_msg>Fix typo in NERSC slurm script generation.  All commands should be backgrounded and a wait placed at the end.<commit_after>#
# See top-level LICENSE.rst file for Copyright information
#
# -*- coding: utf-8 -*-
"""
desispec.pipeline.run
=====================

Tools for running the pipeline.
"""

from __future__ import absolute_import, division, print_function

import os
import errno
import sys
import subprocess as sp

from desispec.log import get_logger

def pid_exists( pid ):
    """Check whether pid exists in the current process table.

    **UNIX only.**  Should work the same as psutil.pid_exists().

    Args:
        pid (int): A process ID.

    Returns:
        pid_exists (bool): ``True`` if the process exists in the current process table.
    """
    if pid < 0:
        return False
    if pid == 0:
        # According to "man 2 kill" PID 0 refers to every process
        # in the process group of the calling process.
        # On certain systems 0 is a valid PID but we have no way
        # to know that in a portable fashion.
        raise ValueError('invalid PID 0')
    try:
        os.kill(pid, 0)
    except OSError as err:
        if err.errno == errno.ESRCH:
            # ESRCH == No such process
            return False
        elif err.errno == errno.EPERM:
            # EPERM clearly means there's a process to deny access to
            return True
        else:
            # According to "man 2 kill" possible error values are
            # (EINVAL, EPERM, ESRCH)
            raise
    else:
        return True

def subprocess_list(tasks, rank=0):
    log = get_logger()
    #pids = []
    for tsk in tasks:
        runcom = True
        newest_in = 0
        for dep in tsk['inputs']:
            if not os.path.isfile(dep):
                err = "dependency {} missing, cannot run task {}".format(dep, " ".join(tsk['command']))
                log.error(err)
                runcom = False
            else:
                t = os.path.getmtime(dep)
                if t > newest_in:
                    newest_in = t
        alldone = True
        if len(tsk['outputs']) == 0:
            alldone = False
        for outf in tsk['outputs']:
            if not os.path.isfile(outf):
                alldone = False
            else:
                t = os.path.getmtime(outf)
                if t < newest_in:
                    alldone = False
        if alldone:
            runcom = False
        proc = None
        if runcom:
            # proc = sp.Popen(tsk['command'], stdout=sp.PIPE, stderr=sp.STDOUT)
            # log.info("subproc[{}]: {}".format(proc.pid, " ".join(tsk['command'])))
            # outs, errs = proc.communicate()
            # for line in outs:
            #     log.debug("subproc[{}]:   {}".format(proc.pid, line.rstrip()))
            # proc.wait()
            log.info("subproc: {}".format(" ".join(tsk['command'])))
            ret = sp.call(tsk['command'])
            for outf in tsk['outputs']:
                ret = sp.call(['mv', '{}.part'.format(outf), outf])

    return

def shell_job(path, logroot, envsetup, desisetup, commands):
    with open(path, 'w') as f:
        f.write("#!/bin/bash\n\n")
        f.write("now=`date +%Y%m%d-%H:%M:%S`\n")
        f.write("log={}_${{now}}.log\n\n".format(logroot))
        for com in envsetup:
            f.write("{}\n".format(com))
        f.write("\n")
        f.write("source {}\n\n".format(desisetup))
        for com in commands:
            executable = com.split(' ')[0]
            f.write("which {}\n".format(executable))
            f.write("time {} >>${{log}} 2>&1\n\n".format(com))
    return

def nersc_job(path, logroot, envsetup, desisetup, commands, nodes=1, nodeproc=1, minutes=10, openmp=False, multiproc=False):
    hours = int(minutes/60)
    fullmin = int(minutes - 60*hours)
    timestr = "{:02d}:{:02d}:00".format(hours, fullmin)
    with open(path, 'w') as f:
        f.write("#!/bin/bash -l\n\n")
        f.write("#SBATCH --partition=debug\n")
        f.write("#SBATCH --account=desi\n")
        f.write("#SBATCH --nodes={}\n".format(nodes))
        f.write("#SBATCH --time={}\n".format(timestr))
        f.write("#SBATCH --job-name=desipipe\n")
        f.write("#SBATCH --output={}_slurm_%j.log\n".format(logroot))
        f.write("#SBATCH --export=NONE\n\n")
        for com in envsetup:
            f.write("{}\n".format(com))
        f.write("\n")
        f.write("source {}\n\n".format(desisetup))
        f.write("node_cores=0\n")
        f.write("if [ ${NERSC_HOST} = edison ]; then\n")
        f.write("  node_cores=24\n")
        f.write("else\n")
        f.write("  node_cores=32\n")
        f.write("fi\n")
        f.write("\n")
        f.write("nodes={}\n".format(nodes))
        f.write("node_proc={}\n".format(nodeproc))
        f.write("node_thread=$(( node_cores / node_proc ))\n")
        f.write("procs=$(( nodes * node_proc ))\n\n")
        if openmp:
            f.write("export OMP_NUM_THREADS=${node_thread}\n")
            f.write("\n")
        runstr = "srun --export=ALL"
        if multiproc:
            runstr = "{} --cpu_bind=no".format(runstr)
        f.write("run=\"{} -n ${{procs}} -N ${{nodes}} -c ${{node_thread}}\"\n\n".format(runstr))
        f.write("now=`date +%Y%m%d-%H:%M:%S`\n")
        f.write("echo \"job datestamp = ${now}\"\n")
        f.write("log={}_${{now}}.log\n\n".format(logroot))
        for com in commands:
            executable = com.split(' ')[0]
            f.write("which {}\n".format(executable))
            f.write("time ${{run}} {} >>${{log}} 2>&1 &\n\n".format(com))
        f.write("wait\n\n")
    return

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from numpy.testing import assert_approx_equal, assert_allclose, assert_array_equal
from nose.tools import assert_equal, assert_true, raises, assert_less, assert_is_none, assert_raises
from UliEngineering.SignalProcessing.DateTime import *
from nose_parameterized import parameterized
import concurrent.futures
import numpy as np
import datetime

class TestSpliceDate(object):
    def test_simple(self):
        d1 = datetime.datetime(2016, 1, 1, 12, 32, 15, microsecond=123456)
        d2 = datetime.datetime(1905, 1, 1, 14, 11, 25, microsecond=52)
        dres = datetime.datetime(2016, 1, 1, 14, 11, 25, microsecond=52)
        assert_equal(dres, splice_date(d1, d2))

class TestAutoStrptime(object):
    def test_formats(self):
        #%Y-%m-%d %H:%M:%S.%f
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime("2016-02-01 15:02:11.000050"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime(" 2016-02-01 15:02:11.000050"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime("2016-02-01 15:02:11.000050 "))
        #%H:%M:%S.%f
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11, 50), auto_strptime("15:02:11.000050"))
        #%Y-%m-%d
        assert_equal(datetime.datetime(2016, 2, 1), auto_strptime("2016-02-01"))
        assert_equal(datetime.datetime(2016, 2, 1), auto_strptime("2016-02-01 "))
        #%H:%M:%S
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime("15:02:11"))
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime("15:02:11 "))
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime(" 15:02:11"))
        #%Y-%m-%d %H
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime("2016-02-01 15"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime(" 2016-02-01 15"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime("2016-02-01 15"))
        #%Y-%m-%d %H
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime("2016-02-01 15:02"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime("2016-02-01 15:02 "))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime(" 2016-02-01 15:02"))
        #%Y-%m-%d %H:%M:%S
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime("2016-02-01 15:02:11"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime("2016-02-01 15:02:11 "))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime(" 2016-02-01 15:02:11"))

    def test_examples(self):
        assert_equal(datetime.datetime(2016, 7, 21, 00, 00, 00), auto_strptime("2016-07-21 00:00:00"))
        assert_equal(datetime.datetime(2016, 7, 21, 3, 00, 00), auto_strptime("2016-07-21 03:00:00"))
<commit_msg>Added more unit tests to auto_strptime()<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from numpy.testing import assert_approx_equal, assert_allclose, assert_array_equal
from nose.tools import assert_equal, assert_true, raises, assert_less, assert_is_none, assert_raises
from UliEngineering.SignalProcessing.DateTime import *
from nose_parameterized import parameterized
import concurrent.futures
import numpy as np
import datetime

class TestSpliceDate(object):
    def test_simple(self):
        d1 = datetime.datetime(2016, 1, 1, 12, 32, 15, microsecond=123456)
        d2 = datetime.datetime(1905, 1, 1, 14, 11, 25, microsecond=52)
        dres = datetime.datetime(2016, 1, 1, 14, 11, 25, microsecond=52)
        assert_equal(dres, splice_date(d1, d2))

class TestAutoStrptime(object):
    def test_formats(self):
        #%Y-%m-%d %H:%M:%S.%f
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime("2016-02-01 15:02:11.000050"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime(" 2016-02-01 15:02:11.000050"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11, 50), auto_strptime("2016-02-01 15:02:11.000050 "))
        #%H:%M:%S.%f
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11, 50), auto_strptime("15:02:11.000050"))
        #%Y-%m-%d
        assert_equal(datetime.datetime(2016, 2, 1), auto_strptime("2016-02-01"))
        assert_equal(datetime.datetime(2016, 2, 1), auto_strptime("2016-02-01 "))
        #%H:%M:%S
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime("15:02:11"))
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime("15:02:11 "))
        assert_equal(datetime.datetime(1900, 1, 1, 15, 2, 11), auto_strptime(" 15:02:11"))
        #%Y-%m-%d %H
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime("2016-02-01 15"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime(" 2016-02-01 15"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 0, 0), auto_strptime("2016-02-01 15"))
        #%Y-%m-%d %H
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime("2016-02-01 15:02"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime("2016-02-01 15:02 "))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 0), auto_strptime(" 2016-02-01 15:02"))
        #%Y-%m-%d %H:%M:%S
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime("2016-02-01 15:02:11"))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime("2016-02-01 15:02:11 "))
        assert_equal(datetime.datetime(2016, 2, 1, 15, 2, 11), auto_strptime(" 2016-02-01 15:02:11"))

    def test_examples(self):
        assert_equal(datetime.datetime(2016, 7, 21, 00, 00, 00), auto_strptime("2016-07-21 00:00:00"))
        assert_equal(datetime.datetime(2016, 7, 21, 3, 00, 00), auto_strptime("2016-07-21 03:00:00"))
        assert_equal(datetime.datetime(2016, 9, 1, 10, 00, 00), auto_strptime("2016-09-01 10"))
<|endoftext|>


<commit_before>'''
 Description
 -----------

 This python script is used to submit a single request to a web service
 or collection of web services. This script will capture and parse the
 web service response and write an entry to a text file. Intended to be
 included in a scheduled job with a regular interval to monitor a
 web service.

 Specifically, the script will log the date and time, executionTime
 and response code for the BC Physical Address Geocoder and the
 BC Route Planner.

 Learn more about these APIs using API consoles linked from the BC
 Data Catalogue:

 BC Physical Address Geocoder:
 https://catalogue.data.gov.bc.ca/dataset/8f4a016f-14db-4def-8ef9-7c797de1cdd9

 BC Route Planner:
 https://catalogue.data.gov.bc.ca/dataset/3dad0c30-ef32-4f4c-82fa-33787d5f85f8

 Two of the APIs below require the use of API keys. To acquire an API
 key for your application please contact DataBC.

 Contact Us
 -----------
 https://forms.gov.bc.ca/databc-contact-us/
'''

# Import modules
import sys
import time
import urllib2
import json

# Clear variables
logfile_1 = ''
logfile_2 = ''
logfile_3 = ''
geocoder_secure_url = ''
geocoder_public_url = ''
router_url = ''
json_response = ''
web_request = ''
service_url = []
log_files = []

dbc_api_key = str(sys.argv[1])

# Define variables

# Request URL to the gated geocoder
# Please contact DataBC to acquire an API key for your application
geocoder_secure_url = 'https://geocoder.api.gov.bc.ca/addresses.json?' \
                      'addressString=%20525%20Superior%20Street%2C%20' \
                      'Victoria%2C%20BC&locationDescriptor=any&maxResults=1' \
                      '&interpolation=adaptive&echo=true&setBack=0&' \
                      'outputSRS=4326&minScore=1&provinceCode=BC' \
                      '&apikey=' + dbc_api_key

# Request URL to the public geocoder
geocoder_public_url = 'https://apps.gov.bc.ca/pub/geocoder/addresses.json?' \
                      'addressString=%20525%20Superior%20Street%2C%20' \
                      'Victoria%2C%20BC&locationDescriptor=any&maxResults=1' \
                      '&interpolation=adaptive&echo=true&setBack=0&' \
                      'outputSRS=4326&minScore=1&provinceCode=BC&' \
                      'apikey=nokeyprovided'

# Request URL to the BC Route Planner
# Please contact DataBC to acquire an API key for your application
router_url = 'https://router.api.gov.bc.ca/route.json?routeDescription=' \
             'spansProvinceFastest&points=-126.844567%2C49.978503%2C-' \
             '122.799997%2C58.925305&outputSRS=4326&criteria=fastest' \
             '&distanceUnit=km&apikey=' + dbc_api_key

# Location of the text files
logfile_1 = 'https://raw.githubusercontent.com/bcgov/dbcrss/master/heartbeat/src/geocoder-secure-heartbeat.txt'
logfile_2 = 'https://raw.githubusercontent.com/bcgov/dbcrss/master/heartbeat/src/geocoder-public-heartbeat.txt'
logfile_3 = 'https://raw.githubusercontent.com/bcgov/dbcrss/master/heartbeat/src/router-heartbeat.txt'

# Counter used to iterate through the 'log_file' list.
log_count = 0

# Lists to iterate through the variables above
service_url = [geocoder_secure_url, geocoder_public_url, router_url]
log_files = [logfile_1, logfile_2, logfile_3]

# Loop through and write results to the respective log files
for _URL_ in service_url:
    # Get the date and time
    timestamp = time.strftime('%Y/%m/%d|%H:%M:%S|')
    # Get the HTTP response code
    response_code = urllib2.urlopen(_URL_).getcode()
    # Get the JSON response and parse out the execution time (executionTime)
    web_request = urllib2.urlopen(_URL_)
    json_response = web_request.read()
    # Load the JSON response for parsing
    data = json.loads(json_response)
    # Grab the executionTime parameter
    exec_time = data['executionTime']
    # Write the results to log file
    target_gated_log = open(log_files[log_count], 'a')
    target_gated_log.write(str(timestamp))
    target_gated_log.write(str(exec_time) + '|')
    target_gated_log.write(str(response_code))
    target_gated_log.write('\n')
    target_gated_log.close()
    log_count += 1
<commit_msg>Update Heartbeat.py<commit_after>'''
 Description
 -----------

 This python script is used to submit a single request to a web service
 or collection of web services. This script will capture and parse the
 web service response and write an entry to a text file. Intended to be
 included in a scheduled job with a regular interval to monitor a
 web service.

 Specifically, the script will log the date and time, executionTime
 and response code for the BC Physical Address Geocoder and the
 BC Route Planner.

 Learn more about these APIs using API consoles linked from the BC
 Data Catalogue:

 BC Physical Address Geocoder:
 https://catalogue.data.gov.bc.ca/dataset/8f4a016f-14db-4def-8ef9-7c797de1cdd9

 BC Route Planner:
 https://catalogue.data.gov.bc.ca/dataset/3dad0c30-ef32-4f4c-82fa-33787d5f85f8

 Two of the APIs below require the use of API keys. To acquire an API
 key for your application please contact DataBC.

 Contact Us
 -----------
 https://forms.gov.bc.ca/databc-contact-us/
'''

# Import modules
import sys
import time
import urllib2
import json

# Clear variables
logfile_1 = ''
logfile_2 = ''
logfile_3 = ''
geocoder_secure_url = ''
geocoder_public_url = ''
router_url = ''
json_response = ''
web_request = ''
service_url = []
log_files = []

dbc_api_key = str(sys.argv[1])

# Define variables

# Request URL to the gated geocoder
# Please contact DataBC to acquire an API key for your application
geocoder_secure_url = 'https://geocoder.api.gov.bc.ca/addresses.json?' \
                      'addressString=%20525%20Superior%20Street%2C%20' \
                      'Victoria%2C%20BC&locationDescriptor=any&maxResults=1' \
                      '&interpolation=adaptive&echo=true&setBack=0&' \
                      'outputSRS=4326&minScore=1&provinceCode=BC' \
                      '&apikey=' + dbc_api_key

# Request URL to the public geocoder
geocoder_public_url = 'https://apps.gov.bc.ca/pub/geocoder/addresses.json?' \
                      'addressString=%20525%20Superior%20Street%2C%20' \
                      'Victoria%2C%20BC&locationDescriptor=any&maxResults=1' \
                      '&interpolation=adaptive&echo=true&setBack=0&' \
                      'outputSRS=4326&minScore=1&provinceCode=BC&' \
                      'apikey=nokeyprovided'

# Request URL to the BC Route Planner
# Please contact DataBC to acquire an API key for your application
router_url = 'https://router.api.gov.bc.ca/route.json?routeDescription=' \
             'spansProvinceFastest&points=-126.844567%2C49.978503%2C-' \
             '122.799997%2C58.925305&outputSRS=4326&criteria=fastest' \
             '&distanceUnit=km&apikey=' + dbc_api_key

# Location of the text files
logfile_1 = 'geocoder-secure-heartbeat.txt'
logfile_2 = 'geocoder-public-heartbeat.txt'
logfile_3 = 'router-heartbeat.txt'

# Counter used to iterate through the 'log_file' list.
log_count = 0

# Lists to iterate through the variables above
service_url = [geocoder_secure_url, geocoder_public_url, router_url]
log_files = [logfile_1, logfile_2, logfile_3]

# Loop through and write results to the respective log files
for _URL_ in service_url:
    # Get the date and time
    timestamp = time.strftime('%Y/%m/%d|%H:%M:%S|')
    # Get the HTTP response code
    response_code = urllib2.urlopen(_URL_).getcode()
    # Get the JSON response and parse out the execution time (executionTime)
    web_request = urllib2.urlopen(_URL_)
    json_response = web_request.read()
    # Load the JSON response for parsing
    data = json.loads(json_response)
    # Grab the executionTime parameter
    exec_time = data['executionTime']
    # Write the results to log file
    target_gated_log = open(log_files[log_count], 'a')
    target_gated_log.write(str(timestamp))
    target_gated_log.write(str(exec_time) + '|')
    target_gated_log.write(str(response_code))
    target_gated_log.write('\n')
    target_gated_log.close()
    log_count += 1
<|endoftext|>


<commit_before>from django.db import models
import datetime

# Create your models here.
class Theater(models.Model):
    name = models.CharField(max_length=100)
    street_address = models.CharField(max_length=40, blank=True)
    city = models.CharField(max_length=40)
    state_or_province = models.CharField(max_length=50, blank=True)
    country = models.CharField(max_length=50)
    def __unicode__(self):
        return "{0}: {1}, {2}".format(self.name, self.city, self.state_or_province)

class Play(models.Model):
    title = models.CharField(max_length=100)
    synopsis = models.TextField(blank=True)
    def __unicode__(self):
        return self.title

class People(models.Model):
    name = models.CharField(max_length=40)
    short_bio = models.TextField(blank=True)
    headshot = models.ImageField(blank=True, upload_to='photos')
    def __unicode__(self):
        return self.name

# A production is defined as a Play that ran at a particular Theater between given Dates
class Production(models.Model):
    start_date = models.DateField()
    end_date = models.DateField()
    production_company = models.CharField(max_length=100)
    play = models.ForeignKey(Play)
    theater = models.ForeignKey(Theater)
    notes = models.TextField(blank=True)
    def display_year(self):
        start = self.start_date.year
        end = self.end_date.year
        if end == start:
            return "{0}".format(start)
        else:
            return "{0} - {1}".format(start, end)
    def __unicode__(self):
        return "{0} at The {1} ({2}) : {3}".format(self.play.title, self.theater.name, self.theater.city, self.start_date.year)

# A cast is defined as the list of People playing characters in a particular Production
class Cast(models.Model):
    person = models.ForeignKey(People)
    character = models.CharField(max_length=40)
    production = models.ForeignKey(Production)
    billed_as = models.CharField(max_length = 40, blank=True)
    def __unicode__(self):
        return "{0} in {1} at The {2} ({3}), played by {4}".format(self.character, self.production.play.title, self.production.theater.name, self.production.theater.city, self.person)

# A crew is defined as the list of off-stage People working on a particular Production
class Crew(models.Model):
    person = models.ForeignKey(People)
    job = models.CharField(max_length=40)
    production = models.ForeignKey(Production)
    is_director = models.BooleanField(default=False)
    is_writer = models.BooleanField(default=False)
    is_producer = models.BooleanField(default=False)
    def __unicode__(self):
        return "{0}: {4} for {1) at The {2} ({3})".format(self.job, self.production.play.title, self.production.theater.name, self.production.theater.city, self.person)
    # Comment to trigger a new commit
    # Another comment to trigger another commit

class Theater_pictures(models.Model):
    image = models.ImageField(blank=True, upload_to='photos')
    theater = models.ForeignKey(Theater)

# kinds of photos:
    # headshots (person)
    # scenes (several persons)
    # theaters
    # playbills

# each person, theater, play, or production can have multiple pictures, but one primary picture
# people will use headshots and scenes
# plays will use ?
# productions will use scenes and playbills
# theaters will use only theater pictures

# Want to target images at average of 20KB each

#TODO: Add class for production companies
#TODO: Add class for musical numbers, connect them to plays<commit_msg>Yet another trivial change to force a commit - still cannot get production environment to update files from git.<commit_after>from django.db import models
import datetime

# Create your models here.
class Theater(models.Model):
    name = models.CharField(max_length=100)
    street_address = models.CharField(max_length=40, blank=True)
    city = models.CharField(max_length=40)
    state_or_province = models.CharField(max_length=50, blank=True)
    country = models.CharField(max_length=50)
    def __unicode__(self):
        return "{0}: {1}, {2}".format(self.name, self.city, self.state_or_province)

class Play(models.Model):
    title = models.CharField(max_length=100)
    synopsis = models.TextField(blank=True)
    def __unicode__(self):
        return self.title

class People(models.Model):
    name = models.CharField(max_length=40)
    short_bio = models.TextField(blank=True)
    headshot = models.ImageField(blank=True, upload_to='photos')
    def __unicode__(self):
        return self.name

# A production is defined as a Play that ran at a particular Theater between given Dates
class Production(models.Model):
    start_date = models.DateField()
    end_date = models.DateField()
    production_company = models.CharField(max_length=100)
    play = models.ForeignKey(Play)
    theater = models.ForeignKey(Theater)
    notes = models.TextField(blank=True)
    def display_year(self):
        start = self.start_date.year
        end = self.end_date.year
        if end == start:
            return "{0}".format(start)
        else:
            return "{0} - {1}".format(start, end)
    def __unicode__(self):
        return "{0} at The {1} ({2}) : {3}".format(self.play.title, self.theater.name, self.theater.city, self.start_date.year)

# A cast is defined as the list of People playing characters in a particular Production
class Cast(models.Model):
    person = models.ForeignKey(People)
    character = models.CharField(max_length=40)
    production = models.ForeignKey(Production)
    billed_as = models.CharField(max_length = 40, blank=True)
    def __unicode__(self):
        return "{0} in {1} at The {2} ({3}), played by {4}".format(self.character, self.production.play.title, self.production.theater.name, self.production.theater.city, self.person)

# A crew is defined as the list of off-stage People working on a particular Production
class Crew(models.Model):
    person = models.ForeignKey(People)
    job = models.CharField(max_length=40)
    production = models.ForeignKey(Production)
    is_director = models.BooleanField(default=False)
    is_writer = models.BooleanField(default=False)
    is_producer = models.BooleanField(default=False)
    def __unicode__(self):
        return "{0}: {4} for {1) at The {2} ({3})".format(self.job, self.production.play.title, self.production.theater.name, self.production.theater.city, self.person)
    # Comment to trigger a new commit
    # Another comment to trigger another commit
    # Yet another comment to trigger another commit (trouble getting production to update files from git)

class Theater_pictures(models.Model):
    image = models.ImageField(blank=True, upload_to='photos')
    theater = models.ForeignKey(Theater)

# kinds of photos:
    # headshots (person)
    # scenes (several persons)
    # theaters
    # playbills

# each person, theater, play, or production can have multiple pictures, but one primary picture
# people will use headshots and scenes
# plays will use ?
# productions will use scenes and playbills
# theaters will use only theater pictures

# Want to target images at average of 20KB each

#TODO: Add class for production companies
#TODO: Add class for musical numbers, connect them to plays<|endoftext|>


<commit_before>import logging
import psycopg2

from socorro.external.postgresql.base import PostgreSQLBase
from socorro.external.postgresql.util import Util
from socorro.lib import search_common, util

import socorro.database.database as db

logger = logging.getLogger("webapi")

class Search(PostgreSQLBase):

    """
    Implement the /search service with PostgreSQL.
    """

    def __init__(self, *args, **kwargs):
        super(Search, self).__init__(*args, **kwargs)

    def search(self, **kwargs):
        """
        Search for crashes and return them.

        See http://socorro.readthedocs.org/en/latest/middleware.html#search

        Optional arguments: see SearchCommon.get_parameters()

        """
        # Creating the connection to the DB
        self.connection = self.database.connection()
        cur = self.connection.cursor()

        params = search_common.get_parameters(kwargs)

        # Default mode falls back to starts_with for postgres
        if params["search_mode"] == "default":
            params["search_mode"] = "starts_with"
        if params["plugin_search_mode"] == "default":
            params["plugin_search_mode"] = "starts_with"

        # For Postgres, we never search for a list of terms
        if params["terms"]:
            params["terms"] = " ".join(params["terms"])
            params["terms"] = Search.prepare_terms(params["terms"],
                                                   params["search_mode"])

        # Searching for terms in plugins
        if params["report_process"] == "plugin" and params["plugin_terms"]:
            params["plugin_terms"] = " ".join(params["plugin_terms"])
            params["plugin_terms"] = Search.prepare_terms(
                                                params["plugin_terms"],
                                                params["plugin_search_mode"])

        # Get information about the versions
        util_service = Util(config=self.context)
        params["versions_info"] = util_service.versions_info(**params)

        # Parsing the versions
        params["versions_string"] = params["versions"]
        (params["versions"], params["products"]) = Search.parse_versions(
                                                            params["versions"],
                                                            params["products"])

        # Changing the OS ids to OS names
        for i, elem in enumerate(params["os"]):
            for platform in self.context.platforms:
                if platform["id"] == elem:
                    params["os"][i] = platform["name"]

        # Creating the parameters for the sql query
        sql_params = {
        }

        # Preparing the different parts of the sql query
        sql_select = self.generate_sql_select(params)

        # Adding count for each OS
        for i in self.context.platforms:
            sql_params["os_%s" % i["id"]] = i["name"]

        sql_from = self.build_reports_sql_from(params)

        (sql_where, sql_params) = self.build_reports_sql_where(params,
                                                               sql_params,
                                                               self.context)

        sql_group = self.generate_sql_group(params)

        sql_order = """
            ORDER BY total DESC
        """

        (sql_limit, sql_params) = self.build_reports_sql_limit(params,
                                                               sql_params)

        # Assembling the query
        sql_query = " ".join(("/* socorro.search.Search search */",
                              sql_select, sql_from, sql_where, sql_group,
                              sql_order, sql_limit))

        # Query for counting the results
        sql_count_query = " ".join((
                "/* socorro.external.postgresql.search.Search search.count */",
                "SELECT count(DISTINCT r.signature)", sql_from, sql_where))

        # Debug
        logger.debug(cur.mogrify(sql_query, sql_params))

        # Querying the DB
        try:
            total = db.singleValueSql(cur, sql_count_query, sql_params)
        except db.SQLDidNotReturnSingleValue:
            total = 0
            util.reportExceptionAndContinue(logger)

        results = []

        # No need to call Postgres if we know there will be no results
        if total != 0:
            try:
                results = db.execute(cur, sql_query, sql_params)
            except psycopg2.Error:
                util.reportExceptionAndContinue(logger)

        json_result = {
            "total": total,
            "hits": []
        }

        # Transforming the results into what we want
        for crash in results:
            if params["report_process"] == "plugin":
                row = dict(zip(("signature", "count", "is_windows", "is_mac",
                                "is_linux", "numhang", "numplugin",
                                "numcontent", "pluginname", "pluginversion",
                                "pluginfilename"), crash))
            else:
                row = dict(zip(("signature", "count", "is_windows", "is_mac",
                                "is_linux", "numhang", "numplugin",
                                "numcontent"), crash))
            json_result["hits"].append(row)

        self.connection.close()

        return json_result

    def generate_sql_select(self, params):
        """
        Generate and return the SELECT part of the final SQL query.
        """
        sql_select = ["SELECT r.signature, count(r.id) as total"]

        ## Adding count for each OS
        for i in self.context.platforms:
            sql_select.append("".join(("count(CASE WHEN (r.os_name = %(os_",
                                       i["id"], ")s) THEN 1 END) AS is_",
                                       i["id"])))

        sql_select.append(("SUM (CASE WHEN r.hangid IS NULL THEN 0  ELSE 1 "
                           "END) AS numhang"))
        sql_select.append(("SUM (CASE WHEN r.process_type='plugin' THEN 1"
                           "ELSE 0 END) as numplugin"))
        sql_select.append(("SUM (CASE WHEN r.process_type='content' THEN 1"
                           "ELSE 0 END) as numcontent"))

        ## Searching through plugins
        if params["report_process"] == "plugin":
            sql_select.append(("plugins.name AS pluginName, "
                               "plugins_reports.version AS pluginVersion, "
                               "plugins.filename AS pluginFilename"))

        return ", ".join(sql_select)

    def generate_sql_group(self, params):
        """
        Generate and return the GROUP BY part of the final SQL query.
        """
        sql_group = ["GROUP BY r.signature"]

        # Searching through plugins
        if params["report_process"] == "plugin":
            sql_group.append("pluginName, pluginVersion, pluginFilename ")

        return ", ".join(sql_group)
<commit_msg>Fixes bug 737421 - Order search results by total and signature.<commit_after>import logging
import psycopg2

from socorro.external.postgresql.base import PostgreSQLBase
from socorro.external.postgresql.util import Util
from socorro.lib import search_common, util

import socorro.database.database as db

logger = logging.getLogger("webapi")

class Search(PostgreSQLBase):

    """
    Implement the /search service with PostgreSQL.
    """

    def __init__(self, *args, **kwargs):
        super(Search, self).__init__(*args, **kwargs)

    def search(self, **kwargs):
        """
        Search for crashes and return them.

        See http://socorro.readthedocs.org/en/latest/middleware.html#search

        Optional arguments: see SearchCommon.get_parameters()

        """
        # Creating the connection to the DB
        self.connection = self.database.connection()
        cur = self.connection.cursor()

        params = search_common.get_parameters(kwargs)

        # Default mode falls back to starts_with for postgres
        if params["search_mode"] == "default":
            params["search_mode"] = "starts_with"
        if params["plugin_search_mode"] == "default":
            params["plugin_search_mode"] = "starts_with"

        # For Postgres, we never search for a list of terms
        if params["terms"]:
            params["terms"] = " ".join(params["terms"])
            params["terms"] = Search.prepare_terms(params["terms"],
                                                   params["search_mode"])

        # Searching for terms in plugins
        if params["report_process"] == "plugin" and params["plugin_terms"]:
            params["plugin_terms"] = " ".join(params["plugin_terms"])
            params["plugin_terms"] = Search.prepare_terms(
                                                params["plugin_terms"],
                                                params["plugin_search_mode"])

        # Get information about the versions
        util_service = Util(config=self.context)
        params["versions_info"] = util_service.versions_info(**params)

        # Parsing the versions
        params["versions_string"] = params["versions"]
        (params["versions"], params["products"]) = Search.parse_versions(
                                                            params["versions"],
                                                            params["products"])

        # Changing the OS ids to OS names
        for i, elem in enumerate(params["os"]):
            for platform in self.context.platforms:
                if platform["id"] == elem:
                    params["os"][i] = platform["name"]

        # Creating the parameters for the sql query
        sql_params = {
        }

        # Preparing the different parts of the sql query
        sql_select = self.generate_sql_select(params)

        # Adding count for each OS
        for i in self.context.platforms:
            sql_params["os_%s" % i["id"]] = i["name"]

        sql_from = self.build_reports_sql_from(params)

        (sql_where, sql_params) = self.build_reports_sql_where(params,
                                                               sql_params,
                                                               self.context)

        sql_group = self.generate_sql_group(params)

        sql_order = """
            ORDER BY total DESC, signature
        """

        (sql_limit, sql_params) = self.build_reports_sql_limit(params,
                                                               sql_params)

        # Assembling the query
        sql_query = " ".join(("/* socorro.search.Search search */",
                              sql_select, sql_from, sql_where, sql_group,
                              sql_order, sql_limit))

        # Query for counting the results
        sql_count_query = " ".join((
                "/* socorro.external.postgresql.search.Search search.count */",
                "SELECT count(DISTINCT r.signature)", sql_from, sql_where))

        # Debug
        logger.debug(cur.mogrify(sql_query, sql_params))

        # Querying the DB
        try:
            total = db.singleValueSql(cur, sql_count_query, sql_params)
        except db.SQLDidNotReturnSingleValue:
            total = 0
            util.reportExceptionAndContinue(logger)

        results = []

        # No need to call Postgres if we know there will be no results
        if total != 0:
            try:
                results = db.execute(cur, sql_query, sql_params)
            except psycopg2.Error:
                util.reportExceptionAndContinue(logger)

        json_result = {
            "total": total,
            "hits": []
        }

        # Transforming the results into what we want
        for crash in results:
            if params["report_process"] == "plugin":
                row = dict(zip(("signature", "count", "is_windows", "is_mac",
                                "is_linux", "numhang", "numplugin",
                                "numcontent", "pluginname", "pluginversion",
                                "pluginfilename"), crash))
            else:
                row = dict(zip(("signature", "count", "is_windows", "is_mac",
                                "is_linux", "numhang", "numplugin",
                                "numcontent"), crash))
            json_result["hits"].append(row)

        self.connection.close()

        return json_result

    def generate_sql_select(self, params):
        """
        Generate and return the SELECT part of the final SQL query.
        """
        sql_select = ["SELECT r.signature, count(r.id) as total"]

        ## Adding count for each OS
        for i in self.context.platforms:
            sql_select.append("".join(("count(CASE WHEN (r.os_name = %(os_",
                                       i["id"], ")s) THEN 1 END) AS is_",
                                       i["id"])))

        sql_select.append(("SUM (CASE WHEN r.hangid IS NULL THEN 0  ELSE 1 "
                           "END) AS numhang"))
        sql_select.append(("SUM (CASE WHEN r.process_type='plugin' THEN 1"
                           "ELSE 0 END) as numplugin"))
        sql_select.append(("SUM (CASE WHEN r.process_type='content' THEN 1"
                           "ELSE 0 END) as numcontent"))

        ## Searching through plugins
        if params["report_process"] == "plugin":
            sql_select.append(("plugins.name AS pluginName, "
                               "plugins_reports.version AS pluginVersion, "
                               "plugins.filename AS pluginFilename"))

        return ", ".join(sql_select)

    def generate_sql_group(self, params):
        """
        Generate and return the GROUP BY part of the final SQL query.
        """
        sql_group = ["GROUP BY r.signature"]

        # Searching through plugins
        if params["report_process"] == "plugin":
            sql_group.append("pluginName, pluginVersion, pluginFilename ")

        return ", ".join(sql_group)
<|endoftext|>


<commit_before>import threading
import os
import sys
import subprocess
import time

globalconfig = {
    'benchmark_dir': None,
    'sender_ip'    : None,
    'receiver_ip'  : None,
    'num_messages' : None,
    'message_len'  : None
  }

globalresults = {}

# mode should be a string: "sender" or "receiver"
def buildCommand(mode, config):
  # copy config for safe mutation
  conf = dict(config)
  ip_key = mode + "_ip"
  conf['ssh_ip'] = config[ip_key]
  conf['mode'] = mode
  command = "ssh %(user)s@%(ssh_ip)s %(benchmark_dir)s/benchmark %(num_messages)s %(message_len)s %(mode)s %(receiver_ip)s %(sender_ip)s" % conf
  return command

class ReceiverThread(threading.Thread):
  def run(self):
    command = buildCommand("receiver", globalconfig)
    print(command)
    subprocess.call(command, shell=True)

class SenderThread(threading.Thread):
  def run(self):
    command = buildCommand("sender", globalconfig)
    print(command)
    subprocess.call(command, shell=True)
    # fetch results file
    fetch = "scp %(user)s@%(sender_ip)s:results.txt ." % globalconfig
    print(fetch)
    subprocess.call(fetch, shell=True)
    # read results
    f = open("results.txt","r")
    run_time = f.readline()
    f.close()
    ## stash results
    stashResult(run_time)

def stashResult(run_time):
  num_messages = globalconfig['num_messages']
  message_len = globalconfig['message_len']
  k = (num_messages, message_len)
  if  k not in globalresults:
    globalresults[k] = []

  print(run_time)
  globalresults[k].append(run_time)

def sleep():
  time.sleep(2)

def runExperiment(num_messages, message_len):
  # config
  globalconfig['num_messages'] = num_messages
  globalconfig['message_len'] = message_len

  t1 = ReceiverThread()
  t2 = SenderThread()
  t1.start()
  sleep()
  t2.start()
  t1.join()
  t2.join()

if __name__ == "__main__":
  # Args
  if len(sys.argv) < 5:
    print("usage:", sys.argv[0],": ssh_user receiver_ip sender_ip benchmark_dir")

  globalconfig["user"] = sys.argv[1]
  globalconfig["receiver_ip"] = sys.argv[2]
  globalconfig["sender_ip"] = sys.argv[3]
  globalconfig["benchmark_dir"] = sys.argv[4]

  # Config
  nums = [100000]
  lens = [16, 1024, 32*1024, 64*1024 , 128*1024, 256*1024, 512*1024, 1024*1024]
  trials = range(3)
  # Run
  for num_messages in nums:
    for message_len in lens:
      for t in trials:
        runExperiment(num_messages, message_len)
        sleep()
  # Print
  print(globalresults)

  exit(0)
<commit_msg>Move script to benchmark repo<commit_after><|endoftext|>


<commit_before>"""
Hearthstone Strings file

File format: TSV. Lines starting with `#` are ignored.
Key is always `TAG`
"""
import csv
from typing import Dict

import hearthstone_data

StringsRow = Dict[str, str]
StringsDict = Dict[str, StringsRow]

_cache: Dict[str, StringsDict] = {}

def load(fp) -> StringsDict:
	reader = csv.DictReader(
		filter(lambda row: row.strip() and not row.startswith("#"), fp),
		delimiter="\t"
	)
	stripped_rows = [{k: v for k, v in row.items() if v} for row in reader]
	return {stripped_row.pop("TAG"): stripped_row for stripped_row in stripped_rows}

def load_globalstrings(locale="enUS") -> StringsDict:
	path: str = hearthstone_data.get_strings_file(locale, filename="GLOBAL.txt")
	if path not in _cache:
		with open(path, "r", encoding="utf-8-sig") as f:
			_cache[path] = load(f)

	return _cache[path]

if __name__ == "__main__":
	import json
	import sys

	for path in sys.argv[1:]:
		with open(path, "r") as f:
			print(json.dumps(load(f)))
<commit_msg>Skip blank lines in strings files<commit_after>"""
Hearthstone Strings file

File format: TSV. Lines starting with `#` are ignored.
Key is always `TAG`
"""
import csv
from typing import Dict

import hearthstone_data

StringsRow = Dict[str, str]
StringsDict = Dict[str, StringsRow]

_cache: Dict[str, StringsDict] = {}

def load(fp) -> StringsDict:
	reader = csv.DictReader(
		filter(lambda row: row.strip() and not row.startswith("#"), fp),
		delimiter="\t"
	)
	stripped_rows = [{k: v for k, v in row.items() if k and v} for row in reader]
	return {
		stripped_row.pop("TAG"): stripped_row for stripped_row in stripped_rows
		if stripped_row
	}

def load_globalstrings(locale="enUS") -> StringsDict:
	path: str = hearthstone_data.get_strings_file(locale, filename="GLOBAL.txt")
	if path not in _cache:
		with open(path, "r", encoding="utf-8-sig") as f:
			_cache[path] = load(f)

	return _cache[path]

if __name__ == "__main__":
	import json
	import sys

	for path in sys.argv[1:]:
		with open(path, "r") as f:
			print(json.dumps(load(f)))
<|endoftext|>


<commit_before>from cStringIO import StringIO
import boto.exception
import boto.s3.connection
import boto.s3.acl
import boto.utils
import bunch
import nose
import operator
import random
import string
import socket
import ssl

from boto.s3.connection import S3Connection

from nose.tools import eq_ as eq
from nose.plugins.attrib import attr

from .utils import assert_raises
import AnonymousAuth

from email.header import decode_header

from . import (
    nuke_prefixed_buckets,
    get_new_bucket,
    s3,
    config,
    get_prefix,
    )

_orig_conn = {}
_custom_headers = {}
_remove_headers = []

# fill_in_auth does not exist in boto master, so if we update boto, this will
# need to be changed to handle make_request's changes. Likely, at authorize
class HeaderS3Connection(S3Connection):
    def fill_in_auth(self, http_request, **kwargs):
        global _custom_headers, _remove_headers

        # do our header magic
        final_headers = http_request.headers
        final_headers.update(_custom_headers)

        for header in _remove_headers:
            try:
                del final_headers[header]
            except KeyError:
                pass

        S3Connection.fill_in_auth(self, http_request, **kwargs)

        final_headers = http_request.headers
        final_headers.update(_custom_headers)

        for header in _remove_headers:
            try:
                del final_headers[header]
            except KeyError:
                pass

        return http_request

def setup():
    global _orig_conn

    for conn in s3:
        _orig_conn[conn] = s3[conn]
        header_conn = HeaderS3Connection(
            aws_access_key_id=s3[conn].aws_access_key_id,
            aws_secret_access_key=s3[conn].aws_secret_access_key,
            is_secure=s3[conn].is_secure,
            port=s3[conn].port,
            host=s3[conn].host,
            calling_format=s3[conn].calling_format
            )

        s3[conn] = header_conn

def teardown():
    global _orig_auth_handler
    for conn in s3:
        s3[conn] = _orig_conn[conn]

def _clear_custom_headers():
    global _custom_headers, _remove_headers
    _custom_headers = {}
    _remove_headers = []

def _add_custom_headers(headers=None, remove=None):
    global _custom_headers, _remove_headers
    if not _custom_headers:
        _custom_headers = {}

    if headers is not None:
        _custom_headers.update(headers)
    if remove is not None:
        _remove_headers.extend(remove)

def _setup_bad_object(headers=None, remove=None):
    bucket = get_new_bucket()

    _add_custom_headers(headers=headers, remove=remove)
    return bucket.new_key('foo')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_md5_invalid():
    key = _setup_bad_object({'Content-MD5':'AWS HAHAHA'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'InvalidDigest')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_md5_empty():
    key = _setup_bad_object({'Content-MD5': ''})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'InvalidDigest')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_md5_unreadable():
    key = _setup_bad_object({'Content-MD5': '\x07'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 403)
    eq(e.reason, 'Forbidden')
    assert e.error_code in ('AccessDenied', 'SignatureDoesNotMatch')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_md5_none():
    key = _setup_bad_object(remove=('Content-MD5',))
    key.set_contents_from_string('bar')

# strangely, amazon doesn't report an error with a non-expect 100 also, our
# error comes back as html, and not xml as I normally expect
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_mismatch():
    key = _setup_bad_object({'Expect': 200})
    key.set_contents_from_string('bar')

# this is a really long test, and I don't know if it's valid...
# again, accepts this with no troubles
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_empty():
    key = _setup_bad_object({'Expect': ''})
    key.set_contents_from_string('bar')

# this is a really long test..
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_utf8():
    key = _setup_bad_object({'Expect': '\x07'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_empty():
    key = _setup_bad_object({'Content-Length': ''})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, None)

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_zero():
    key = _setup_bad_object({'Content-Length': 0})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'BadDigest')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_mismatch_above():
    content = 'bar'
    length = len(content) + 1

    key = _setup_bad_object({'Content-Length': length})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, content)
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'RequestTimeout')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_mismatch_below():
    content = 'bar'
    length = len(content) - 1
    key = _setup_bad_object({'Content-Length': length})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, content)
    eq(e.status, 400)
    # dho is 'Bad request', which doesn't match the http response code
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'BadDigest')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_invalid():
    key = _setup_bad_object({'Content-Type': 'text/plain'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_empty():
    key = _setup_bad_object({'Content-Type': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_none():
    key = _setup_bad_object(remove=('Content-Type',))
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_unreadable():
    key = _setup_bad_object({'Content-Type': '\x08'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 403)
    eq(e.reason, 'Forbidden')
    assert e.error_code in ('AccessDenied', 'SignatureDoesNotMatch')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_invalid():
    key = _setup_bad_object({'User-Agent': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_empty():
    key = _setup_bad_object({'User-Agent': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_unreadable():
    key = _setup_bad_object({'User-Agent': '\x07'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_none():
    key = _setup_bad_object(remove=('User-Agent',))
    key.set_contents_from_string('bar')
<commit_msg>add another Expect header test<commit_after>from cStringIO import StringIO
import boto.exception
import boto.s3.connection
import boto.s3.acl
import boto.utils
import bunch
import nose
import operator
import random
import string
import socket
import ssl

from boto.s3.connection import S3Connection

from nose.tools import eq_ as eq
from nose.plugins.attrib import attr

from .utils import assert_raises
import AnonymousAuth

from email.header import decode_header

from . import (
    nuke_prefixed_buckets,
    get_new_bucket,
    s3,
    config,
    get_prefix,
    )

_orig_conn = {}
_custom_headers = {}
_remove_headers = []

# fill_in_auth does not exist in boto master, so if we update boto, this will
# need to be changed to handle make_request's changes. Likely, at authorize
class HeaderS3Connection(S3Connection):
    def fill_in_auth(self, http_request, **kwargs):
        global _custom_headers, _remove_headers

        # do our header magic
        final_headers = http_request.headers
        final_headers.update(_custom_headers)

        for header in _remove_headers:
            try:
                del final_headers[header]
            except KeyError:
                pass

        S3Connection.fill_in_auth(self, http_request, **kwargs)

        final_headers = http_request.headers
        final_headers.update(_custom_headers)

        for header in _remove_headers:
            try:
                del final_headers[header]
            except KeyError:
                pass

        return http_request

def setup():
    global _orig_conn

    for conn in s3:
        _orig_conn[conn] = s3[conn]
        header_conn = HeaderS3Connection(
            aws_access_key_id=s3[conn].aws_access_key_id,
            aws_secret_access_key=s3[conn].aws_secret_access_key,
            is_secure=s3[conn].is_secure,
            port=s3[conn].port,
            host=s3[conn].host,
            calling_format=s3[conn].calling_format
            )

        s3[conn] = header_conn

def teardown():
    global _orig_auth_handler
    for conn in s3:
        s3[conn] = _orig_conn[conn]

def _clear_custom_headers():
    global _custom_headers, _remove_headers
    _custom_headers = {}
    _remove_headers = []

def _add_custom_headers(headers=None, remove=None):
    global _custom_headers, _remove_headers
    if not _custom_headers:
        _custom_headers = {}

    if headers is not None:
        _custom_headers.update(headers)
    if remove is not None:
        _remove_headers.extend(remove)

def _setup_bad_object(headers=None, remove=None):
    bucket = get_new_bucket()

    _add_custom_headers(headers=headers, remove=remove)
    return bucket.new_key('foo')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_md5_invalid():
    key = _setup_bad_object({'Content-MD5':'AWS HAHAHA'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'InvalidDigest')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_md5_empty():
    key = _setup_bad_object({'Content-MD5': ''})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'InvalidDigest')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_md5_unreadable():
    key = _setup_bad_object({'Content-MD5': '\x07'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 403)
    eq(e.reason, 'Forbidden')
    assert e.error_code in ('AccessDenied', 'SignatureDoesNotMatch')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_md5_none():
    key = _setup_bad_object(remove=('Content-MD5',))
    key.set_contents_from_string('bar')

# strangely, amazon doesn't report an error with a non-expect 100 also, our
# error comes back as html, and not xml as I normally expect
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_mismatch():
    key = _setup_bad_object({'Expect': 200})
    key.set_contents_from_string('bar')

# this is a really long test, and I don't know if it's valid...
# again, accepts this with no troubles
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_empty():
    key = _setup_bad_object({'Expect': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_expect_none():
    key = _setup_bad_object(remove=('Expect',))
    key.set_contents_from_string('bar')

# this is a really long test..
@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_expect_utf8():
    key = _setup_bad_object({'Expect': '\x07'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_empty():
    key = _setup_bad_object({'Content-Length': ''})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, None)

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_zero():
    key = _setup_bad_object({'Content-Length': 0})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'BadDigest')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_mismatch_above():
    content = 'bar'
    length = len(content) + 1

    key = _setup_bad_object({'Content-Length': length})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, content)
    eq(e.status, 400)
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'RequestTimeout')

@nose.with_setup(teardown=_clear_custom_headers)
@attr('fails_on_dho')
def test_object_create_bad_contentlength_mismatch_below():
    content = 'bar'
    length = len(content) - 1
    key = _setup_bad_object({'Content-Length': length})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, content)
    eq(e.status, 400)
    # dho is 'Bad request', which doesn't match the http response code
    eq(e.reason, 'Bad Request')
    eq(e.error_code, 'BadDigest')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_invalid():
    key = _setup_bad_object({'Content-Type': 'text/plain'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_empty():
    key = _setup_bad_object({'Content-Type': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_none():
    key = _setup_bad_object(remove=('Content-Type',))
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_contenttype_unreadable():
    key = _setup_bad_object({'Content-Type': '\x08'})

    e = assert_raises(boto.exception.S3ResponseError, key.set_contents_from_string, 'bar')
    eq(e.status, 403)
    eq(e.reason, 'Forbidden')
    assert e.error_code in ('AccessDenied', 'SignatureDoesNotMatch')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_invalid():
    key = _setup_bad_object({'User-Agent': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_empty():
    key = _setup_bad_object({'User-Agent': ''})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_unreadable():
    key = _setup_bad_object({'User-Agent': '\x07'})
    key.set_contents_from_string('bar')

@nose.with_setup(teardown=_clear_custom_headers)
def test_object_create_bad_ua_none():
    key = _setup_bad_object(remove=('User-Agent',))
    key.set_contents_from_string('bar')
<|endoftext|>


<commit_before>import shutil
import subprocess
from distutils import dir_util
from os import listdir, makedirs
from os.path import basename, dirname, exists, join
from tempfile import mkdtemp

import f90nml
import pandas as pd

from .config import config

IS_WINDOWS = config['is_windows']

class MAGICC(object):
    """
    Provides access to the MAGICC binary and configuration.

    To enable multiple MAGICC 'packages' to be configured independently,
    the MAGICC directory containing the input files, configuration
    and binary is copied to a new folder. The configuration in this
    MAGICC copy can then be edited without impacting other instances.

    A `MAGICC` instance first has to be setup by calling
    `create_copy`. If many model runs are being performed this step only has
    to be performed once. `run` can be called many times with the
    configuration files being updated between each call.

    Alternatively, an existing MAGICC directory structure can be used by
    setting `root_dir`.
    """

    version = None

    def __init__(self, root_dir=None):
        self.root_dir = root_dir
        self.config = {}
        self.executable = self.get_executable()

        if root_dir is not None:
            self.is_temp = False
        else:
            # Create a temp directory
            self.is_temp = True
            self.root_dir = mkdtemp(prefix="pymagicc-")

    def __enter__(self):
        if self.is_temp and not exists(self.run_dir):
            self.create_copy()
        return self

    def __exit__(self, *args, **kwargs):
        self.remove_temp_copy()

    def create_copy(self):
        """
        Initialises a temporary directory structure and copy of MAGICC
        configuration files and binary.
        """
        if exists(self.run_dir):
            raise Exception("A copy of MAGICC has already been created.")
        if not exists(self.root_dir):
            makedirs(self.root_dir)
        # Copy the MAGICC run directory into the appropriate location
        dir_util.copy_tree(join(self.original_dir, ".."), self.root_dir)

        # Create basic configuration files so magicc can run
        self.set_years()
        self.set_config()

    @property
    def binary_name(self):
        return basename(self.executable)

    @property
    def original_dir(self):
        return dirname(self.executable)

    @property
    def run_dir(self):
        return join(self.root_dir, 'run')

    @property
    def out_dir(self):
        return join(self.root_dir, 'out')

    def run(self, only=None):
        """
        Run MAGICC and parse the output.

        :param only: If not None, only extract variables in this list
        :return: Dict containing DataFrames for each of the extracted variables
        """
        command = [join(self.run_dir, self.binary_name)]

        if not IS_WINDOWS \
                and self.binary_name.endswith(".exe"):  # pragma: no cover
            command.insert(0, 'wine')

        # On Windows shell=True is required.
        subprocess.check_call(command, cwd=self.run_dir, shell=IS_WINDOWS)

        results = {}

        outfiles = [f for f in listdir(self.out_dir)
                    if f.startswith("DAT_") and f.endswith(".OUT")]

        for filename in outfiles:
            name = filename.replace("DAT_", "").replace(".OUT", "")
            if only is None or name in only:
                results[name] = pd.read_csv(
                    join(self.out_dir, filename),
                    delim_whitespace=True,
                    skiprows=19 if self.version == 6 else 21,
                    index_col=0,
                    engine="python"
                )

        with open(join(self.out_dir, "PARAMETERS.OUT")) as nml_file:
            parameters = dict(f90nml.read(nml_file))
            for group in ["nml_years", "nml_allcfgs", "nml_outputcfgs"]:
                parameters[group] = dict(parameters[group])
                for k, v in parameters[group].items():
                    if isinstance(v, str):
                        parameters[group][k] = v.strip()
                    elif isinstance(v, list):
                        if isinstance(v[0], str):
                            parameters[group][k] = [
                                i.strip().replace("\n", "") for i in v]
                parameters[group.replace("nml_", "")] = parameters.pop(group)
            self.config = parameters

        return results

    def remove_temp_copy(self):
        """
        Removes a temporary copy of the MAGICC version shipped with Pymagicc.
        """
        if self.is_temp:
            shutil.rmtree(self.root_dir)

    def set_config(self, filename='MAGTUNE_SIMPLE.CFG',
                   top_level_key='nml_allcfgs', **kwargs):
        """
        Create a configuration file for MAGICC

        Writes a fortran namelist in run_dir.
        :param filename:
        :param top_level_key:
        :param kwargs: Other parameters to pass to the configuration file. No
            validation on the parameters is performed.
        :return: A dict containing the contents of the namelist which was
            written to file
        """
        fname = join(self.run_dir, filename)
        data = {
            top_level_key: kwargs
        }
        f90nml.write(data, fname, force=True)

        return data

    def set_years(self, startyear=1765, endyear=2100):
        """
        Set the start and end dates of the simulations

        :param startyear: Start year of the simulation
        :param endyear: End year of the simulation
        :return: The contents of the namelist
        """
        # stepsperyear is required and should never be overridden
        return self.set_config('MAGCFG_NMLYEARS.CFG', 'nml_years',
                               endyear=endyear, startyear=startyear,
                               stepsperyear=12)

class MAGICC6(MAGICC):
    version = 6

    def get_executable(cls):
        return config['executable']

class MAGICC7(MAGICC):
    version = 7

    def get_executable(cls):
        return config['executable_7']
<commit_msg>Explicitly copy run directory<commit_after>import shutil
import subprocess
from os import listdir, makedirs
from os.path import basename, dirname, exists, join, isfile
from tempfile import mkdtemp

import f90nml
import pandas as pd

from .config import config

IS_WINDOWS = config['is_windows']

def _copy_files(source, target):
    """
    Copy all the files in source directory to target

    Ignores subdirectories
    """
    source_files = listdir(source)
    if not exists(target):
        makedirs(target)
    for filename in source_files:
        full_filename = join(source, filename)
        if isfile(full_filename):
            shutil.copy(full_filename, target)

class MAGICC(object):
    """
    Provides access to the MAGICC binary and configuration.

    To enable multiple MAGICC 'packages' to be configured independently,
    the MAGICC directory containing the input files, configuration
    and binary is copied to a new folder. The configuration in this
    MAGICC copy can then be edited without impacting other instances.

    A `MAGICC` instance first has to be setup by calling
    `create_copy`. If many model runs are being performed this step only has
    to be performed once. `run` can be called many times with the
    configuration files being updated between each call.

    Alternatively, an existing MAGICC directory structure can be used by
    setting `root_dir`.
    """

    version = None

    def __init__(self, root_dir=None):
        self.root_dir = root_dir
        self.config = {}
        self.executable = self.get_executable()

        if root_dir is not None:
            self.is_temp = False
        else:
            # Create a temp directory
            self.is_temp = True
            self.root_dir = mkdtemp(prefix="pymagicc-")

    def __enter__(self):
        if self.is_temp and not exists(self.run_dir):
            self.create_copy()
        return self

    def __exit__(self, *args, **kwargs):
        self.remove_temp_copy()

    def create_copy(self):
        """
        Initialises a temporary directory structure and copy of MAGICC
        configuration files and binary.
        """
        if exists(self.run_dir):
            raise Exception("A copy of MAGICC has already been created.")
        if not exists(self.root_dir):
            makedirs(self.root_dir)

        # Copy a subset of folders from the MAGICC `original_dir`
        # Also copy anything which is in the root of the MAGICC distribution
        # Assumes that the MAGICC binary is in a folder one level below the root
        # of the MAGICC distribution. I.e. /run/magicc.exe or /bin/magicc
        dirs_to_copy = [
            '.',
            'bin',
            'run'
        ]
        for d in dirs_to_copy:
            source_dir = join(self.original_dir, '..', d)
            if exists(source_dir):
                _copy_files(source_dir, join(self.root_dir, d))

        # Create an empty out dir
        # MAGICC assumes that the 'out' directory already exists
        makedirs(join(self.root_dir, 'out'))

        # Create basic configuration files so magicc can run
        self.set_years()
        self.set_config()

    @property
    def binary_name(self):
        return basename(self.executable)

    @property
    def original_dir(self):
        return dirname(self.executable)

    @property
    def run_dir(self):
        return join(self.root_dir, 'run')

    @property
    def out_dir(self):
        return join(self.root_dir, 'out')

    def run(self, only=None):
        """
        Run MAGICC and parse the output.

        :param only: If not None, only extract variables in this list
        :return: Dict containing DataFrames for each of the extracted variables
        """
        command = [join(self.run_dir, self.binary_name)]

        if not IS_WINDOWS \
                and self.binary_name.endswith(".exe"):  # pragma: no cover
            command.insert(0, 'wine')

        # On Windows shell=True is required.
        subprocess.check_call(command, cwd=self.run_dir, shell=IS_WINDOWS)

        results = {}

        outfiles = [f for f in listdir(self.out_dir)
                    if f.startswith("DAT_") and f.endswith(".OUT")]

        for filename in outfiles:
            name = filename.replace("DAT_", "").replace(".OUT", "")
            if only is None or name in only:
                results[name] = pd.read_csv(
                    join(self.out_dir, filename),
                    delim_whitespace=True,
                    skiprows=19 if self.version == 6 else 21,
                    index_col=0,
                    engine="python"
                )

        with open(join(self.out_dir, "PARAMETERS.OUT")) as nml_file:
            parameters = dict(f90nml.read(nml_file))
            for group in ["nml_years", "nml_allcfgs", "nml_outputcfgs"]:
                parameters[group] = dict(parameters[group])
                for k, v in parameters[group].items():
                    if isinstance(v, str):
                        parameters[group][k] = v.strip()
                    elif isinstance(v, list):
                        if isinstance(v[0], str):
                            parameters[group][k] = [
                                i.strip().replace("\n", "") for i in v]
                parameters[group.replace("nml_", "")] = parameters.pop(group)
            self.config = parameters

        return results

    def remove_temp_copy(self):
        """
        Removes a temporary copy of the MAGICC version shipped with Pymagicc.
        """
        if self.is_temp:
            shutil.rmtree(self.root_dir)

    def set_config(self, filename='MAGTUNE_SIMPLE.CFG',
                   top_level_key='nml_allcfgs', **kwargs):
        """
        Create a configuration file for MAGICC

        Writes a fortran namelist in run_dir.
        :param filename:
        :param top_level_key:
        :param kwargs: Other parameters to pass to the configuration file. No
            validation on the parameters is performed.
        :return: A dict containing the contents of the namelist which was
            written to file
        """
        fname = join(self.run_dir, filename)
        data = {
            top_level_key: kwargs
        }
        f90nml.write(data, fname, force=True)

        return data

    def set_years(self, startyear=1765, endyear=2100):
        """
        Set the start and end dates of the simulations

        :param startyear: Start year of the simulation
        :param endyear: End year of the simulation
        :return: The contents of the namelist
        """
        # stepsperyear is required and should never be overridden
        return self.set_config('MAGCFG_NMLYEARS.CFG', 'nml_years',
                               endyear=endyear, startyear=startyear,
                               stepsperyear=12)

class MAGICC6(MAGICC):
    version = 6

    def get_executable(cls):
        return config['executable']

class MAGICC7(MAGICC):
    version = 7

    def get_executable(cls):
        return config['executable_7']
<|endoftext|>