<commit_before>import math
import stibble.py

def readFromFile():
	text_file = open("test.txt", "r")
	lines = text_file.read().split('\n')
	text_file.close()
	pointList = []
	for each in lines:
		if each != "":
			point = each.split(',')
			xval = int(point[0][1:])
			yval = int(point[1][:(len(point[1])-1)])
			pointList += [(xval, yval)]
	return pointList

def distance(p1, p2):
	return math.sqrt((p1[0]-p2[0])*(p1[0]-p2[0]) + (p1[1]-p2[1])*(p1[1]-p2[1]))

def mst(pointList):
	segments = []
	for i in range(len(pointList)):
		for j in range(i+1, len(pointList)):
			#segments is a list of a 3-ple: (point1, point2, distance)
			segments += [(pointList[i], pointList[j], distance(pointList[i], pointList[j]))]
	sortedSegments = sorted(segments, key=lambda each:each[2])

def nearestNeighbor(pointList):
	uList = [pointList[0]]
	vList = pointList[1:]
	while len(vList) != 0:
		u = uList[len(uList)-1]
		minDist = distance(u, vList[0])
		addV = vList[0]
		for each in vList:
			newDist = distance(u, each)
			if newDist < minDist:
				minDist = newDist
				addV = each
		uList += [addV]
		vList.remove(addV)
	return uList

def clarkWright(pointList):
	return pointList

def makeTour(pointList):
	tour = []
	for i in range(len(pointList)-1):
		tour += [(pointList[i], pointList[i+1], distance(pointList[i], pointList[i+1]))]
	tour += [(pointList[len(pointList)-1], pointList[0], distance(pointList[0], pointList[len(pointList)-1]))]
	return tour

def makePoints(tour):
	orderedTour = [tour[0]]
	tour = tour[1:]
	while len(tour) != 0:
		lastPoint = orderedTour[len(orderedTour)-1][1]
		nextAdd = tour[0]
		for each in tour:
			if each[0] == lastPoint:
				orderedTour += [each]
				nextAdd = each
			elif each[1] == lastPoint:
				orderedTour += [(each[1], each[0], each[2])]
				nextAdd = each
		tour.remove(nextAdd)
	pointList = map(lambda x: x[0], orderedTour)
	pointList += [orderedTour[len(orderedTour)-1][1]]
	return pointList

def twoOptSwap(pointList):
	tour = pointList
	currDist = distance(tour[0], tour[len(tour)-1])
	for i in range(len(tour)-1):
		currDist += distance(tour[i], tour[i+1])
	changed = True
	while(changed):
		changed = False
		for i in range(len(tour)-1):
			pointA1 = tour[i]
			pointA2 = tour[i+1]
			for k in range(i+1, len(tour)-1):
				pointB1 = tour[k]
				pointB2 = tour[k+1]
				originalLength = distance(pointA1, pointA2) + distance(pointB1, pointB2)
				testLength = distance(pointA1, pointB2) + distance(pointA2, pointB1)
				if testLength < originalLength:
					currDist = currDist - originalLength + testLength
					print currDist
					changed = True
					break

def main(pointList):
	nearestNeighborOrder = nearestNeighbor(pointList)
	optSwapTour = twoOptSwap(nearestNeighborOrder)
	print optSwapTour

	#clarkWrightOrder = clarkWright(pointList)

if __name__ == "__main__":
	if len(sys.argv) is 3:
		n_points = int(sys.argv[2])
		img = Image.open(sys.argv[1]).convert('LA')
		pointList = stibble(img)
		main(pointList)
	else:
		print "Incorrect number of arguments"
<commit_msg>nearest neighbor<commit_after>import math
import stibble.py

def readFromFile():
	text_file = open("test.txt", "r")
	lines = text_file.read().split('\n')
	text_file.close()
	pointList = []
	for each in lines:
		if each != "":
			point = each.split(',')
			xval = int(point[0][1:])
			yval = int(point[1][:(len(point[1])-1)])
			pointList += [(xval, yval)]
	return pointList

def distance(p1, p2):
	return math.sqrt((p1[0]-p2[0])*(p1[0]-p2[0]) + (p1[1]-p2[1])*(p1[1]-p2[1]))

def mst(pointList):
	segments = []
	for i in range(len(pointList)):
		for j in range(i+1, len(pointList)):
			#segments is a list of a 3-ple: (point1, point2, distance)
			segments += [(pointList[i], pointList[j], distance(pointList[i], pointList[j]))]
	sortedSegments = sorted(segments, key=lambda each:each[2])

def nearestNeighbor(pointList):
	uList = [pointList[0]]
	vList = pointList[1:]
	while len(vList) != 0:
		u = uList[len(uList)-1]
		minDist = distance(u, vList[0])
		addV = vList[0]
		for each in vList:
			newDist = distance(u, each)
			if newDist < minDist:
				minDist = newDist
				addV = each
		uList += [addV]
		vList.remove(addV)
	return uList

def clarkWright(pointList):
	return pointList

def makeTour(pointList):
	tour = []
	for i in range(len(pointList)-1):
		tour += [(pointList[i], pointList[i+1], distance(pointList[i], pointList[i+1]))]
	tour += [(pointList[len(pointList)-1], pointList[0], distance(pointList[0], pointList[len(pointList)-1]))]
	return tour

def makePoints(tour):
	orderedTour = [tour[0]]
	tour = tour[1:]
	while len(tour) != 0:
		lastPoint = orderedTour[len(orderedTour)-1][1]
		nextAdd = tour[0]
		for each in tour:
			if each[0] == lastPoint:
				orderedTour += [each]
				nextAdd = each
			elif each[1] == lastPoint:
				orderedTour += [(each[1], each[0], each[2])]
				nextAdd = each
		tour.remove(nextAdd)
	pointList = map(lambda x: x[0], orderedTour)
	pointList += [orderedTour[len(orderedTour)-1][1]]
	return pointList

def twoOptSwap(pointList):
	tour = pointList
	currDist = distance(tour[0], tour[len(tour)-1])
	for i in range(len(tour)-1):
		currDist += distance(tour[i], tour[i+1])
	changed = True
	while(changed):
		changed = False
		for i in range(len(tour)-1):
			pointA1 = tour[i]
			pointA2 = tour[i+1]
			for k in range(i+1, len(tour)-1):
				pointB1 = tour[k]
				pointB2 = tour[k+1]
				originalLength = distance(pointA1, pointA2) + distance(pointB1, pointB2)
				testLength = distance(pointA1, pointB2) + distance(pointA2, pointB1)
				if testLength < originalLength:
					currDist = currDist - originalLength + testLength
					print currDist
					changed = True
					break

def main(pointList):
	nearestNeighborOrder = nearestNeighbor(pointList)
	#optSwapTour = twoOptSwap(nearestNeighborOrder)
	#print optSwapTour
	#clarkWrightOrder = clarkWright(pointList)

if __name__ == "__main__":
	if len(sys.argv) is 3:
		n_points = int(sys.argv[2])
		img = Image.open(sys.argv[1]).convert('LA')
		pointList = stibble(img)
		main(pointList)
	else:
		print "Incorrect number of arguments"
<|endoftext|>


<commit_before>from boto.ec2.blockdevicemapping import (
    BlockDeviceMapping,
    BlockDeviceType,
)
import boto.ec2
import getpass
import re
import subprocess
import sys
import time

def nameify(s):
    name = ''.join(c if c.isalnum() else '-' for c in s.lower()).strip('-')
    return re.subn(r'\-+', '-', name)[0]

def run(wale_s3_prefix, image_id, instance_type,
        branch=None, name=None, role='demo', profile_name=None):
    if branch is None:
        branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').strip()

    commit = subprocess.check_output(['git', 'rev-parse', '--short', branch]).decode('utf-8').strip()
    if not subprocess.check_output(['git', 'branch', '-r', '--contains', commit]).strip():
        print("Commit %r not in origin. Did you git push?" % commit)
        sys.exit(1)

    username = getpass.getuser()

    if name is None:
        name = nameify('%s-%s-%s' % (branch, commit, username))

    conn = boto.ec2.connect_to_region("us-west-2", profile_name=profile_name)

    domain = 'production' if profile_name == 'production' else 'demo'

    if any(name == i.tags.get('Name')
           for reservation in conn.get_all_instances()
           for i in reservation.instances
           if i.state != 'terminated'):
        print('An instance already exists with name: %s' % name)
        sys.exit(1)

    bdm = BlockDeviceMapping()
    bdm['/dev/sda1'] = BlockDeviceType(volume_type='gp2', delete_on_termination=True, size=40)
    # Don't attach instance storage so we can support auto recovery
    bdm['/dev/sdb'] = BlockDeviceType(no_device=True)
    bdm['/dev/sdc'] = BlockDeviceType(no_device=True)
    user_data = subprocess.check_output(['git', 'show', commit + ':cloud-config.yml']).decode('utf-8')
    user_data = user_data % {
        'WALE_S3_PREFIX': wale_s3_prefix,
        'COMMIT': commit,
        'ROLE': role,
    }

    reservation = conn.run_instances(
        image_id=image_id,
        instance_type=instance_type,
        security_groups=['ssh-http-https'],
        user_data=user_data,
        block_device_map=bdm,
        instance_initiated_shutdown_behavior='terminate',
        instance_profile_name='encoded-instance',
    )

    time.sleep(0.5)  # sleep for a moment to ensure instance exists...
    instance = reservation.instances[0]  # Instance:i-34edd56f
    print('%s.%s.encodedcc.org' % (instance.id, domain))
    instance.add_tags({
        'Name': name,
        'branch': branch,
        'commit': commit,
        'started_by': username,
    })
    print('%s.%s.encodedcc.org' % (name, domain))

    sys.stdout.write(instance.state)
    while instance.state == 'pending':
        sys.stdout.write('.')
        sys.stdout.flush()
        time.sleep(1)
        instance.update()
    print('')
    print(instance.state)

def main():
    import argparse

    def hostname(value):
        if value != nameify(value):
            raise argparse.ArgumentTypeError(
                "%r is an invalid hostname, only [a-z0-9] and hyphen allowed." % value)
        return value

    parser = argparse.ArgumentParser(
        description="Deploy ENCODE on AWS",
    )
    parser.add_argument('-b', '--branch', default=None, help="Git branch or tag")
    parser.add_argument('-n', '--name', type=hostname, help="Instance name")
    parser.add_argument('--wale-s3-prefix', default='s3://encoded-backups-prod/production')
    parser.add_argument(
        '--candidate', action='store_const', default='demo', const='candidate', dest='role',
        help="Deploy candidate instance")
    parser.add_argument(
        '--test', action='store_const', default='demo', const='test', dest='role',
        help="Deploy to production AWS")
    parser.add_argument(
        '--image-id', default='ami-3d50120d',
        help="ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-20140927")
    parser.add_argument(
        '--instance-type', default='t2.medium',
        help="specify 'm3.large' for faster indexing.")
    parser.add_argument('--profile-name', default=None, help="AWS creds profile")
    args = parser.parse_args()

    return run(**vars(args))

if __name__ == '__main__':
    main()
<commit_msg>increased disk space to enable peak search indexing<commit_after>from boto.ec2.blockdevicemapping import (
    BlockDeviceMapping,
    BlockDeviceType,
)
import boto.ec2
import getpass
import re
import subprocess
import sys
import time

def nameify(s):
    name = ''.join(c if c.isalnum() else '-' for c in s.lower()).strip('-')
    return re.subn(r'\-+', '-', name)[0]

def run(wale_s3_prefix, image_id, instance_type,
        branch=None, name=None, role='demo', profile_name=None):
    if branch is None:
        branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').strip()

    commit = subprocess.check_output(['git', 'rev-parse', '--short', branch]).decode('utf-8').strip()
    if not subprocess.check_output(['git', 'branch', '-r', '--contains', commit]).strip():
        print("Commit %r not in origin. Did you git push?" % commit)
        sys.exit(1)

    username = getpass.getuser()

    if name is None:
        name = nameify('%s-%s-%s' % (branch, commit, username))

    conn = boto.ec2.connect_to_region("us-west-2", profile_name=profile_name)

    domain = 'production' if profile_name == 'production' else 'demo'

    if any(name == i.tags.get('Name')
           for reservation in conn.get_all_instances()
           for i in reservation.instances
           if i.state != 'terminated'):
        print('An instance already exists with name: %s' % name)
        sys.exit(1)

    bdm = BlockDeviceMapping()
    bdm['/dev/sda1'] = BlockDeviceType(volume_type='gp2', delete_on_termination=True, size=80)
    # Don't attach instance storage so we can support auto recovery
    bdm['/dev/sdb'] = BlockDeviceType(no_device=True)
    bdm['/dev/sdc'] = BlockDeviceType(no_device=True)
    user_data = subprocess.check_output(['git', 'show', commit + ':cloud-config.yml']).decode('utf-8')
    user_data = user_data % {
        'WALE_S3_PREFIX': wale_s3_prefix,
        'COMMIT': commit,
        'ROLE': role,
    }

    reservation = conn.run_instances(
        image_id=image_id,
        instance_type=instance_type,
        security_groups=['ssh-http-https'],
        user_data=user_data,
        block_device_map=bdm,
        instance_initiated_shutdown_behavior='terminate',
        instance_profile_name='encoded-instance',
    )

    time.sleep(0.5)  # sleep for a moment to ensure instance exists...
    instance = reservation.instances[0]  # Instance:i-34edd56f
    print('%s.%s.encodedcc.org' % (instance.id, domain))
    instance.add_tags({
        'Name': name,
        'branch': branch,
        'commit': commit,
        'started_by': username,
    })
    print('%s.%s.encodedcc.org' % (name, domain))

    sys.stdout.write(instance.state)
    while instance.state == 'pending':
        sys.stdout.write('.')
        sys.stdout.flush()
        time.sleep(1)
        instance.update()
    print('')
    print(instance.state)

def main():
    import argparse

    def hostname(value):
        if value != nameify(value):
            raise argparse.ArgumentTypeError(
                "%r is an invalid hostname, only [a-z0-9] and hyphen allowed." % value)
        return value

    parser = argparse.ArgumentParser(
        description="Deploy ENCODE on AWS",
    )
    parser.add_argument('-b', '--branch', default=None, help="Git branch or tag")
    parser.add_argument('-n', '--name', type=hostname, help="Instance name")
    parser.add_argument('--wale-s3-prefix', default='s3://encoded-backups-prod/production')
    parser.add_argument(
        '--candidate', action='store_const', default='demo', const='candidate', dest='role',
        help="Deploy candidate instance")
    parser.add_argument(
        '--test', action='store_const', default='demo', const='test', dest='role',
        help="Deploy to production AWS")
    parser.add_argument(
        '--image-id', default='ami-3d50120d',
        help="ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-20140927")
    parser.add_argument(
        '--instance-type', default='t2.medium',
        help="specify 'm3.large' for faster indexing.")
    parser.add_argument('--profile-name', default=None, help="AWS creds profile")
    args = parser.parse_args()

    return run(**vars(args))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
r"""
Subsampling (:mod:`skbio.maths.subsample`)
==========================================

.. currentmodule:: skbio.maths.subsample

This module provides functionality for subsampling from vectors of counts.

Functions
---------

.. autosummary::
   :toctree: generated/

   subsample

"""
from __future__ import division

#-----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

import numpy as np

def subsample(counts, n):
    """Subsample from a vector of counts.

    Returns `counts` if `n` is equal to or larger than the number of items in
    `counts`.

    Parameters
    ----------
    counts : 1-D array_like
        Vector of counts (integers) to subsample from.
    n : int
        Number of items to subsample from `counts`.

    Returns
    -------
    subsampled : ndarray
        Subsampled vector of counts where the sum of the elements equals `n`
        (i.e., ``subsampled.sum() == n``).

    Examples
    --------
    Subsample 4 items from a vector of counts:

    >>> import numpy as np
    >>> from skbio.maths.subsample import subsample
    >>> a = np.array([4, 5, 0, 2, 1])
    >>> sub = subsample(a, 4)
    >>> sub.sum()
    4

    Trying to subsample an equal or greater number of items results in the same
    vector as our input:

    >>> subsample([0, 3, 0, 1], 8)
    array([0, 3, 0, 1])

    """
    counts = np.asarray(counts)
    counts = counts.astype(int, casting='safe')

    if counts.ndim != 1:
        raise ValueError("Only 1-D vectors are supported.")

    if counts.sum() <= n:
        return counts

    nz = counts.nonzero()[0]
    unpacked = np.concatenate([np.repeat(np.array(i,), counts[i]) for i in nz])
    permuted = np.random.permutation(unpacked)[:n]

    result = np.zeros(len(counts), dtype=int)
    for p in permuted:
        result[p] += 1

    return result
<commit_msg>added Raises section<commit_after>#!/usr/bin/env python
r"""
Subsampling (:mod:`skbio.maths.subsample`)
==========================================

.. currentmodule:: skbio.maths.subsample

This module provides functionality for subsampling from vectors of counts.

Functions
---------

.. autosummary::
   :toctree: generated/

   subsample

"""
from __future__ import division

#-----------------------------------------------------------------------------
# Copyright (c) 2013--, scikit-bio development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

import numpy as np

def subsample(counts, n):
    """Subsample from a vector of counts.

    Returns `counts` if `n` is equal to or larger than the number of items in
    `counts`.

    Parameters
    ----------
    counts : 1-D array_like
        Vector of counts (integers) to subsample from.
    n : int
        Number of items to subsample from `counts`.

    Returns
    -------
    subsampled : ndarray
        Subsampled vector of counts where the sum of the elements equals `n`
        (i.e., ``subsampled.sum() == n``).

    Raises
    ------
    TypeError
        If `counts` cannot be safely converted to an integer datatype.

    Examples
    --------
    Subsample 4 items from a vector of counts:

    >>> import numpy as np
    >>> from skbio.maths.subsample import subsample
    >>> a = np.array([4, 5, 0, 2, 1])
    >>> sub = subsample(a, 4)
    >>> sub.sum()
    4

    Trying to subsample an equal or greater number of items results in the same
    vector as our input:

    >>> subsample([0, 3, 0, 1], 8)
    array([0, 3, 0, 1])

    """
    counts = np.asarray(counts)
    counts = counts.astype(int, casting='safe')

    if counts.ndim != 1:
        raise ValueError("Only 1-D vectors are supported.")

    if counts.sum() <= n:
        return counts

    nz = counts.nonzero()[0]
    unpacked = np.concatenate([np.repeat(np.array(i,), counts[i]) for i in nz])
    permuted = np.random.permutation(unpacked)[:n]

    result = np.zeros(len(counts), dtype=int)
    for p in permuted:
        result[p] += 1

    return result
<|endoftext|>


<commit_before>"""Standard test images.

For more images, see

 - http://sipi.usc.edu/database/database.php

"""

import os as _os

from ..io import imread
from skimage import data_dir

def load(f):
    """Load an image file located in the data directory.

    Parameters
    ----------
    f : string
        File name.

    Returns
    -------
    img : ndarray
        Image loaded from skimage.data_dir.
    """
    return imread(_os.path.join(data_dir, f))

def camera():
    """Gray-level "camera" image, often used for segmentation
    and denoising examples.

    """
    return load("camera.png")

def lena():
    """Colour "Lena" image.

    The standard, yet sometimes controversial Lena test image was
    scanned from the November 1972 edition of Playboy magazine.  From
    an image processing perspective, this image is useful because it
    contains smooth, textured, shaded as well as detail areas.

    """
    return load("lena.png")

def checkerboard():
    """Checkerboard image.

    Checkerboards are often used in image calibration, since the
    corner-points are easy to locate.  Because of the many parallel
    edges, they also visualise distortions particularly well.

    """
    return load("chessboard_GRAY_U8.png")

def coins():
    """Greek coins from Pompeii.

    This image shows several coins outlined against a gray background.
    It is especially useful in, e.g. segmentation tests, where
    individual objects need to be identified against a background.
    The background shares enough grey levels with the coins that a
    simple segmentation is not sufficient.

    Notes
    -----
    This image was downloaded from the
    `Brooklyn Museum Collection
    <http://www.brooklynmuseum.org/opencollection/archives/image/617/image>`__.

    No known copyright restrictions.

    """
    return load("coins.png")
<commit_msg>BUG: data: Fix chessboard file name.<commit_after>"""Standard test images.

For more images, see

 - http://sipi.usc.edu/database/database.php

"""

import os as _os

from ..io import imread
from skimage import data_dir

def load(f):
    """Load an image file located in the data directory.

    Parameters
    ----------
    f : string
        File name.

    Returns
    -------
    img : ndarray
        Image loaded from skimage.data_dir.
    """
    return imread(_os.path.join(data_dir, f))

def camera():
    """Gray-level "camera" image, often used for segmentation
    and denoising examples.

    """
    return load("camera.png")

def lena():
    """Colour "Lena" image.

    The standard, yet sometimes controversial Lena test image was
    scanned from the November 1972 edition of Playboy magazine.  From
    an image processing perspective, this image is useful because it
    contains smooth, textured, shaded as well as detail areas.

    """
    return load("lena.png")

def checkerboard():
    """Checkerboard image.

    Checkerboards are often used in image calibration, since the
    corner-points are easy to locate.  Because of the many parallel
    edges, they also visualise distortions particularly well.

    """
    return load("chessboard_GRAY.png")

def coins():
    """Greek coins from Pompeii.

    This image shows several coins outlined against a gray background.
    It is especially useful in, e.g. segmentation tests, where
    individual objects need to be identified against a background.
    The background shares enough grey levels with the coins that a
    simple segmentation is not sufficient.

    Notes
    -----
    This image was downloaded from the
    `Brooklyn Museum Collection
    <http://www.brooklynmuseum.org/opencollection/archives/image/617/image>`__.

    No known copyright restrictions.

    """
    return load("coins.png")
<|endoftext|>


<commit_before># Copyright (c) 2015 Scopely, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os
import datetime
import zipfile
import logging

import pytz
import dateutil.tz
import boto

from skinflint.billreader import *

LOG = logging.getLogger(__name__)

class FileManager(object):

    def __init__(self, config):
        self.config = config
        self.cache_dir = self.config['cache_dir']
        self.cache_dir = os.path.expanduser(self.cache_dir)
        self.cache_dir = os.path.expandvars(self.cache_dir)
        if not os.path.isdir(self.cache_dir):
            os.mkdir(self.cache_dir)

    def _get_file_modified_time(self, file_path):
        stats = os.stat(file_path)
        return datetime.datetime.fromtimestamp(
            stats.st_atime, dateutil.tz.tzlocal())

    def _get_key_modified_time(self, key):
        ts = boto.utils.parse_ts(key.last_modified)
        return pytz.utc.localize(ts)

    def _download_and_unzip(self, key):
        LOG.debug('downloading %s', key.name)
        key_path = os.path.join(self.cache_dir, key.name)
        key.get_contents_to_filename(key_path)
        if key.name.endswith('.zip'):
            zf = zipfile.ZipFile(key_path)
            namelist = zf.namelist()
            dbf = namelist[0]
            LOG.debug('unzipping downloaded file')
            zf.extract(dbf, self.cache_dir)
            zf.close()
            LOG.debug('deleting zip file')
            os.unlink(key_path)

    def _check_key(self, key, file_name):
        if not os.path.isfile(file_name):
            LOG.debug('%s not in cache, downloading now', file_name)
            self._download_and_unzip(key, file_name)
            file_mod_time = self._get_file_modified_time(file_name)
        else:
            file_mod_time = self._get_file_modified_time(file_name)
            LOG.debug('file_mod_time: %s', file_mod_time)
            key_mod_time = self._get_key_modified_time(key)
            LOG.debug('key_mod_time: %s', key_mod_time)
            if key_mod_time > file_mod_time:
                LOG.debug('cached copy of %s out of date', file_name)
                self._download_and_unzip(key)

    def _get_bill_reader(self, billreader_cls, account_id, year, month):
        account_cfg = self.config['accounts'][account_id]
        s3 = boto.connect_s3(profile_name=account_cfg['profile'])
        bucket = s3.lookup(account_cfg['bucket'])
        key_name = billreader_cls.KeyName.format(
            id=account_id, year=year, month=month)
        key = bucket.lookup(key_name)
        if key is None:
            msg = 'Bucket (%s) does not contain Key (%s)' % (bucket, key_name)
            raise ValueError(msg)
        if key_name.endswith('.zip'):
            file_name = key_name[0:-4]
        else:
            file_name = key_name
        file_name = os.path.join(self.cache_dir, file_name)
        self._check_key(key, file_name)
        return billreader_cls(file_name)

    def get_detailed_billing_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            DetailedBillReportReader, account_id, year, month)

    def get_monthly_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            MonthlyReportReader, account_id, year, month)

    def get_monthly_allocation_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            MonthlyCostAllocationReportReader, account_id, year, month)
<commit_msg>Fixed a bug in filemanager.<commit_after># Copyright (c) 2015 Scopely, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os
import datetime
import zipfile
import logging

import pytz
import dateutil.tz
import boto

from skinflint.billreader import *

LOG = logging.getLogger(__name__)

class FileManager(object):

    def __init__(self, config):
        self.config = config
        self.cache_dir = self.config['cache_dir']
        self.cache_dir = os.path.expanduser(self.cache_dir)
        self.cache_dir = os.path.expandvars(self.cache_dir)
        if not os.path.isdir(self.cache_dir):
            os.mkdir(self.cache_dir)

    def _get_file_modified_time(self, file_path):
        stats = os.stat(file_path)
        return datetime.datetime.fromtimestamp(
            stats.st_atime, dateutil.tz.tzlocal())

    def _get_key_modified_time(self, key):
        ts = boto.utils.parse_ts(key.last_modified)
        return pytz.utc.localize(ts)

    def _download_and_unzip(self, key):
        LOG.debug('downloading %s', key.name)
        key_path = os.path.join(self.cache_dir, key.name)
        key.get_contents_to_filename(key_path)
        if key.name.endswith('.zip'):
            zf = zipfile.ZipFile(key_path)
            namelist = zf.namelist()
            dbf = namelist[0]
            LOG.debug('unzipping downloaded file')
            zf.extract(dbf, self.cache_dir)
            zf.close()
            LOG.debug('deleting zip file')
            os.unlink(key_path)

    def _check_key(self, key, file_name):
        if not os.path.isfile(file_name):
            LOG.debug('%s not in cache, downloading now', file_name)
            self._download_and_unzip(key)
            file_mod_time = self._get_file_modified_time(file_name)
        else:
            file_mod_time = self._get_file_modified_time(file_name)
            LOG.debug('file_mod_time: %s', file_mod_time)
            key_mod_time = self._get_key_modified_time(key)
            LOG.debug('key_mod_time: %s', key_mod_time)
            if key_mod_time > file_mod_time:
                LOG.debug('cached copy of %s out of date', file_name)
                self._download_and_unzip(key)

    def _get_bill_reader(self, billreader_cls, account_id, year, month):
        account_cfg = self.config['accounts'][account_id]
        s3 = boto.connect_s3(profile_name=account_cfg['profile'])
        bucket = s3.lookup(account_cfg['bucket'])
        key_name = billreader_cls.KeyName.format(
            id=account_id, year=year, month=month)
        key = bucket.lookup(key_name)
        if key is None:
            msg = 'Bucket (%s) does not contain Key (%s)' % (bucket, key_name)
            raise ValueError(msg)
        if key_name.endswith('.zip'):
            file_name = key_name[0:-4]
        else:
            file_name = key_name
        file_name = os.path.join(self.cache_dir, file_name)
        self._check_key(key, file_name)
        return billreader_cls(file_name)

    def get_detailed_billing_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            DetailedBillReportReader, account_id, year, month)

    def get_monthly_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            MonthlyReportReader, account_id, year, month)

    def get_monthly_allocation_report_reader(self, account_id, year, month):
        return self._get_bill_reader(
            MonthlyCostAllocationReportReader, account_id, year, month)
<|endoftext|>


<commit_before>import os
import re
from os import O_RDWR, O_CLOEXEC, O_NONBLOCK
from stat import S_ISCHR
from mmap import mmap
from pathlib import Path

class UioMap:
    def __init__( self, info ):
        self.index = int( re.fullmatch( r'map([0-9])', info.name ).group(1) )
        self.name = (info/'name').read_text().rstrip()
        self.address = int( (info/'addr').read_text(), 0 )
        self.size = int( (info/'size').read_text(), 0 )
        self._mmap = None
        self._mem = None

class Uio:
    def fileno( self ):
        return self._fd

    def __init__( self, path, blocking = True ):
        flags = O_RDWR | O_CLOEXEC
        if not blocking:
            flags |= O_NONBLOCK # for irq_recv
        self._fd = os.open( getattr( path, 'path', path ), flags )

        # build path to sysfs dir for obtaining metadata
        dev = os.stat( self._fd ).st_rdev
        dev = '{0}:{1}'.format( os.major(dev), os.minor(dev) )
        self.syspath = Path('/sys/dev/char')/dev;

        # enumerate memory mappings
        self._mappings = {}
        mapinfo = self.syspath/'maps';
        if mapinfo.is_dir():
            for m in map( UioMap, mapinfo.iterdir() ):
                self._mappings[ m.index ] = m
                self._mappings[ m.name ] = m

    # I return a memoryview since mmap doesn't allow creating writeable slices,
    # but note that slicing the mmap object itself does do the Right Thing with
    # regard to access size, e.g. m[0:4] will perform a 32-bit access instead of
    # four byte-accesses.  With memoryview objects you have to use .cast to the
    # right type, so I'm not really happy with either solution yet.
    def map( self, m=0, offset=None, size=None ):
        m = self._mappings[ m ]

        if m._mem is None:
            m._mmap = mmap( self._fd, m.size, offset = m.index << 12 )
            m._mem = memoryview( m._mmap )

        if size is None:
            if offset is None:
                return m._mmap
            if offset == 0:
                return m._mem
            return m._mem[offset:]
        return m._mem[offset:offset+size]

    def irq_enable( self ):
        os.write( self._fd, b'\x01\x00\x00\x00' )

    def irq_disable( self ):
        os.write( self._fd, b'\x00\x00\x00\x00' )

    # note: irq is disabled once received.  you need to reenable it
    #   - before handling it, if edge-triggered
    #   - after handling it, if level-triggered
    def irq_recv( self ):
        os.read( self._fd, 4 )
<commit_msg>accept paths relative to /dev/uio/<commit_after>import os
import re
from os import O_RDWR, O_CLOEXEC, O_NONBLOCK
from stat import S_ISCHR
from mmap import mmap
from pathlib import Path

class UioMap:
    def __init__( self, info ):
        self.index = int( re.fullmatch( r'map([0-9])', info.name ).group(1) )
        self.name = (info/'name').read_text().rstrip()
        self.address = int( (info/'addr').read_text(), 0 )
        self.size = int( (info/'size').read_text(), 0 )
        self._mmap = None
        self._mem = None

class Uio:
    def fileno( self ):
        return self._fd

    def __init__( self, path, blocking = True ):
        flags = O_RDWR | O_CLOEXEC
        if not blocking:
            flags |= O_NONBLOCK # for irq_recv
        path = Path( '/dev/uio', path )
        self._fd = os.open( path.path, flags )

        # build path to sysfs dir for obtaining metadata
        dev = os.stat( self._fd ).st_rdev
        dev = '{0}:{1}'.format( os.major(dev), os.minor(dev) )
        self.syspath = Path('/sys/dev/char')/dev;

        # enumerate memory mappings
        self._mappings = {}
        mapinfo = self.syspath/'maps';
        if mapinfo.is_dir():
            for m in map( UioMap, mapinfo.iterdir() ):
                self._mappings[ m.index ] = m
                self._mappings[ m.name ] = m

    # I return a memoryview since mmap doesn't allow creating writeable slices,
    # but note that slicing the mmap object itself does do the Right Thing with
    # regard to access size, e.g. m[0:4] will perform a 32-bit access instead of
    # four byte-accesses.  With memoryview objects you have to use .cast to the
    # right type, so I'm not really happy with either solution yet.
    def map( self, m=0, offset=None, size=None ):
        m = self._mappings[ m ]

        if m._mem is None:
            m._mmap = mmap( self._fd, m.size, offset = m.index << 12 )
            m._mem = memoryview( m._mmap )

        if size is None:
            if offset is None:
                return m._mmap
            if offset == 0:
                return m._mem
            return m._mem[offset:]
        return m._mem[offset:offset+size]

    def irq_enable( self ):
        os.write( self._fd, b'\x01\x00\x00\x00' )

    def irq_disable( self ):
        os.write( self._fd, b'\x00\x00\x00\x00' )

    # note: irq is disabled once received.  you need to reenable it
    #   - before handling it, if edge-triggered
    #   - after handling it, if level-triggered
    def irq_recv( self ):
        os.read( self._fd, 4 )
<|endoftext|>


<commit_before>'''
Main script to run LIVV.  This script records some user data, sets up the test
suite, runs the verification, and generates a website based on the results of the verification.

This script is broken into several main sections.  The first section defines the imports.
For each new module added to LIVV they must be added to this section for the script to
access them.  The libraryList in util.dependencies should be updated if any functionality
from outside the standard library is added.

To add or modify the test groupings there are several places that will need to be modified.
First, the base module (found in verification, validation, or performance) will need to have
the new test added to the cases dictionary.  A new entry may need to be put into the test specific
module as well; performance tests require this, verfication tests do not.  Finally, a new entry for
the test will need to be added to the appropriate test mapping in the RECORD TEST CASES section.

Execution of LIVV proceeds in the RUN TEST CASES section where the totality of the test
cases are recorded and run grouped by their respective delegate classes.  Each test case
is run using the run method in the class.  This method will run common functionality then
pass off to specialized methods for each test case.  Finally all of the information is
filled into an html template that contains all of the run information for a grouping of
verification.

Created on Dec 3, 2014

@authors: arbennett, jhkennedy
'''

###############################################################################
#                                  Imports                                    #
###############################################################################

# Standard python imports can be loaded any time
import os
import time
import getpass
import platform
import socket
import util.variables

from optparse import OptionParser

# Pull in the LIVV specific modules
import util.configurationHandler
import util.websetup
import verification.dome, verification.ismip, verification.shelf, verification.stream
import performance.dome, performance.gis
import validation.gis
import util.dependencies as dependencies
import util.cleanup

from verification.base import choices as verificationChoices
from performance.base import choices as performanceChoices
from validation.base import choices as validationChoices

###############################################################################
#                                  Options                                    #
###############################################################################
usage_string = "%prog [options]"
parser = OptionParser(usage=usage_string)
parser.add_option('--verification', action='store',
                  type='choice', dest='verification',
                  choices=verificationChoices(), default='all',
                  help='specifies the verification tests to run')

parser.add_option('--performance', action='store',
                  type='choice', dest='performance',
                  choices=performanceChoices(), default='none',
                  help='specifies the performance tests to run')

parser.add_option('--validation', action='store',
                  type='choice', dest='validation',
                  choices=validationChoices(), default='none',
                  help='specifies the validation tests to run')

parser.add_option('--comment', action='store',
                  type='string', dest='comment',
                  default='Test run of code',
                  help='Log a comment about this run')

parser.add_option('-o', '--outputDir', action='store',
                  type='string', dest='outputDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "www",
                  help='Location to output the LIVV webpages.')

parser.add_option('-i', '--inputDir', action='store',
                  type='string', dest='inputDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_test" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running verification.')

parser.add_option('-p', '--performanceDir', action='store',
                  type='string', dest='performanceDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_test" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running performance verification.')

parser.add_option('-b', '--benchmarkDir', action='store',
                  type='string', dest='benchmarkDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_bench" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running verification.')

parser.add_option('-m', '--machine', action='store',
                  type='string', dest='machineName', default='',
                  help='Load a preconfigured set of options for a specific machine.')

parser.add_option('-s', '--save', action="store_true", dest='save',
                  help='Store the configuration being run with the given machine name.')

# Get the options and the arguments
(options, args) = parser.parse_args()

###############################################################################
#                              Global Variables                               #
###############################################################################
util.variables.cwd            = os.path.dirname(os.path.abspath(__file__))
util.variables.configDir      = util.variables.cwd + os.sep + "configurations"
util.variables.inputDir       = options.inputDir
util.variables.benchmarkDir   = options.benchmarkDir
util.variables.performanceDir = options.performanceDir
util.variables.outputDir      = options.outputDir
util.variables.imgDir         = util.variables.outputDir + "/imgs"
util.variables.comment        = options.comment
util.variables.timestamp      = time.strftime("%m-%d-%Y %H:%M:%S")
util.variables.user           = getpass.getuser()
util.variables.websiteDir     = util.variables.cwd + "/web"
util.variables.templateDir    = util.variables.websiteDir + "/templates"
util.variables.indexDir       = util.variables.outputDir

# Modules that need to be loaded on big machines
util.variables.modules = [
           "python/2.7.5",
           "ncl/6.1.0",
           "nco/4.3.9",
           "python_matplotlib/1.3.1",
           "hdf5/1.8.11",
           "netcdf/4.1.3",
           "python_numpy/1.8.0",
           "python_netcdf4/1.0.6"
          ]

# A list of the information that should be looked for in the stdout of model output
util.variables.parserVars = [
              'Dycore Type',
              'Number of processors',
              'Number of timesteps',
              'Avg convergence rate'
             ]

# Variables to measure when parsing through timing summaries
util.variables.timingVars = ['Time'
             # 'Simple Glide',
             # 'Velocity Driver',
             # 'Initial Diagonal Solve',
             # 'IO Writeback'
             ]

# Dycores to try to parse output for
util.variables.dycores = ['glissade'] #["glide", "glissade", "glam", "albany", "bisicles"]

###############################################################################
#                               Main Execution                                #
###############################################################################
if __name__ == '__main__':
    print("------------------------------------------------------------------------------")
    print("  Land Ice Verification & Validation (LIVV)")
    print("------------------------------------------------------------------------------")
    dependencies.check()

    # Check if we are saving/loading the configuration and set up the machine name
    if options.machineName == '' and options.save:
        # Save the configuration with the default host name
        machineName = socket.gethostname()
        util.configurationHandler.save(machineName)
    elif options.save:
        # Save the configuration with the specified host name
        machineName = options.machineName
        util.configurationHandler.save(machineName)
    elif options.machineName == '':
        # Don't save the configuration and use the default host name
        machineName = socket.gethostname()
    else:
        # Try to load the machine name specified
        machineName = options.machineName
        vars = util.configurationHandler.load(machineName)
        util.variables.globals().update(vars)

    # Check if the user has a default config saved and use that if it does
    if os.path.exists(util.variables.configDir + os.sep + machineName + "_" + util.variables.user + "_default"):
        machineName = machineName + "_" + util.variables.user + "_default"
        vars = util.configurationHandler.load(machineName)
        util.variables.globals().update(vars)

    # Print out some information
    print(os.linesep + "  Current run: " + time.strftime("%m-%d-%Y %H:%M:%S"))
    print("  User: " + util.variables.user)
    print("  Config: " + machineName)
    print("  OS Type: " + platform.system() + " " + platform.release())
    print("  " + util.variables.comment + os.linesep)

    # Check to make sure the directory structure is okay
    for dir in [util.variables.inputDir, util.variables.benchmarkDir]:
        if not os.path.exists(dir):
            print("------------------------------------------------------------------------------")
            print("ERROR: Could not find " + dir + " for input")
            print("       Use the -i, -b, and -d flags to specify the locations of the model and comparison data.")
            print("       See README.md for more details.")
            print("------------------------------------------------------------------------------")
            exit(1)

    ###############################################################################
    #                              Record Test Cases                              #
    ###############################################################################
    verificationMapping = {
                           'dome' : verification.dome.Test,
                           'ismip' : verification.ismip.Test,
                           'shelf' : verification.shelf.Test,
                           'stream' : verification.stream.Test
                           }

    performanceMapping = {
                          'dome' : performance.dome.Test,
                          'gis' : performance.gis.Test
                          }

    validationMapping = {
                         'gis' : validation.gis.Test
                         }

    verificationTests = verification.base.choose(options.verification)
    performanceTests = performance.base.choose(options.performance)
    validationTests = validation.base.choose(options.validation)
    testMapping = {
                   "Verification" : verificationTests,
                   "Performance" : performanceTests,
                   "Validation" : validationTests
                   }

    ###############################################################################
    #                               Run Test Cases                                #
    ###############################################################################
    # Give a list of the tests that will be run
    if len(verificationTests) > 0:
        print("Running verification tests:")
        for case in verificationTests:
            print("  " + case)
    if len(performanceTests) > 0:
        print(os.linesep + "Running performance tests:")
        for case in performanceTests:
            print("  " + case)
    if len(validationTests) > 0:
        print(os.linesep + "Running validation tests:")
        for case in validationTests:
            print("  " + case)

    # Set up the directory structure and summary dictionaries for output
    verificationSummary, performanceSummary, validationSummary = dict(), dict(), dict()
    util.websetup.setup(verificationTests + performanceTests + validationTests)

    # Run the verification tests
    if len(verificationTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning verification test suite....")
        print("--------------------------------------------------------------------------")
    for test in verificationTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = verificationMapping[test]()
        newTest.run()
        verificationSummary[test] = newTest.summary
        newTest.generate()
        print("")

    # Run the performance verification
    if len(performanceTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning performance analysis....")
        print("--------------------------------------------------------------------------")
    for test in performanceTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = performanceMapping[test]()
        newTest.run()
        performanceSummary[test] = newTest.summary
        newTest.generate()
        print("")

    # Run the validation verification
    if len(validationTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning validation test suite....")
        print("--------------------------------------------------------------------------")
    for test in validationTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = validationMapping[test]()
        # Run the specific and bit for bit verification for each case of the test
        newTest.run()
        validationSummary[test] = newTest.summary
        # Generate the test-specific webpage
        newTest.generate()
        print("")

    # Create the site index
    print("Generating web pages in " + util.variables.outputDir + "....")
    util.websetup.generate(verificationSummary, performanceSummary, validationSummary)

    print("Cleaning up....")
    util.cleanup.clean()

    ###############################################################################
    #                        Finished.  Tell user about it.                       #
    ###############################################################################
    print("------------------------------------------------------------------------------")
    print("Finished running LIVV.  Results:  ")
    print("  Open " + util.variables.outputDir + "/index.html to see test results")
    print("------------------------------------------------------------------------------")
<commit_msg>Add python env hash-bang.<commit_after>#!/usr/bin/env python

'''
Main script to run LIVV.  This script records some user data, sets up the test
suite, runs the verification, and generates a website based on the results of the verification.

This script is broken into several main sections.  The first section defines the imports.
For each new module added to LIVV they must be added to this section for the script to
access them.  The libraryList in util.dependencies should be updated if any functionality
from outside the standard library is added.

To add or modify the test groupings there are several places that will need to be modified.
First, the base module (found in verification, validation, or performance) will need to have
the new test added to the cases dictionary.  A new entry may need to be put into the test specific
module as well; performance tests require this, verfication tests do not.  Finally, a new entry for
the test will need to be added to the appropriate test mapping in the RECORD TEST CASES section.

Execution of LIVV proceeds in the RUN TEST CASES section where the totality of the test
cases are recorded and run grouped by their respective delegate classes.  Each test case
is run using the run method in the class.  This method will run common functionality then
pass off to specialized methods for each test case.  Finally all of the information is
filled into an html template that contains all of the run information for a grouping of
verification.

Created on Dec 3, 2014

@authors: arbennett, jhkennedy
'''

###############################################################################
#                                  Imports                                    #
###############################################################################

# Standard python imports can be loaded any time
import os
import time
import getpass
import platform
import socket
import util.variables

from optparse import OptionParser

# Pull in the LIVV specific modules
import util.configurationHandler
import util.websetup
import verification.dome, verification.ismip, verification.shelf, verification.stream
import performance.dome, performance.gis
import validation.gis
import util.dependencies as dependencies
import util.cleanup

from verification.base import choices as verificationChoices
from performance.base import choices as performanceChoices
from validation.base import choices as validationChoices

###############################################################################
#                                  Options                                    #
###############################################################################
usage_string = "%prog [options]"
parser = OptionParser(usage=usage_string)
parser.add_option('--verification', action='store',
                  type='choice', dest='verification',
                  choices=verificationChoices(), default='all',
                  help='specifies the verification tests to run')

parser.add_option('--performance', action='store',
                  type='choice', dest='performance',
                  choices=performanceChoices(), default='none',
                  help='specifies the performance tests to run')

parser.add_option('--validation', action='store',
                  type='choice', dest='validation',
                  choices=validationChoices(), default='none',
                  help='specifies the validation tests to run')

parser.add_option('--comment', action='store',
                  type='string', dest='comment',
                  default='Test run of code',
                  help='Log a comment about this run')

parser.add_option('-o', '--outputDir', action='store',
                  type='string', dest='outputDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "www",
                  help='Location to output the LIVV webpages.')

parser.add_option('-i', '--inputDir', action='store',
                  type='string', dest='inputDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_test" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running verification.')

parser.add_option('-p', '--performanceDir', action='store',
                  type='string', dest='performanceDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_test" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running performance verification.')

parser.add_option('-b', '--benchmarkDir', action='store',
                  type='string', dest='benchmarkDir',
                  default=os.path.dirname(os.path.abspath(__file__)) + os.sep + "reg_bench" + os.sep + "linux-gnu" + os.sep + "higher-order",
                  help='Location of the input for running verification.')

parser.add_option('-m', '--machine', action='store',
                  type='string', dest='machineName', default='',
                  help='Load a preconfigured set of options for a specific machine.')

parser.add_option('-s', '--save', action="store_true", dest='save',
                  help='Store the configuration being run with the given machine name.')

# Get the options and the arguments
(options, args) = parser.parse_args()

###############################################################################
#                              Global Variables                               #
###############################################################################
util.variables.cwd            = os.path.dirname(os.path.abspath(__file__))
util.variables.configDir      = util.variables.cwd + os.sep + "configurations"
util.variables.inputDir       = options.inputDir
util.variables.benchmarkDir   = options.benchmarkDir
util.variables.performanceDir = options.performanceDir
util.variables.outputDir      = options.outputDir
util.variables.imgDir         = util.variables.outputDir + "/imgs"
util.variables.comment        = options.comment
util.variables.timestamp      = time.strftime("%m-%d-%Y %H:%M:%S")
util.variables.user           = getpass.getuser()
util.variables.websiteDir     = util.variables.cwd + "/web"
util.variables.templateDir    = util.variables.websiteDir + "/templates"
util.variables.indexDir       = util.variables.outputDir

# Modules that need to be loaded on big machines
util.variables.modules = [
           "python/2.7.5",
           "ncl/6.1.0",
           "nco/4.3.9",
           "python_matplotlib/1.3.1",
           "hdf5/1.8.11",
           "netcdf/4.1.3",
           "python_numpy/1.8.0",
           "python_netcdf4/1.0.6"
          ]

# A list of the information that should be looked for in the stdout of model output
util.variables.parserVars = [
              'Dycore Type',
              'Number of processors',
              'Number of timesteps',
              'Avg convergence rate'
             ]

# Variables to measure when parsing through timing summaries
util.variables.timingVars = ['Time'
             # 'Simple Glide',
             # 'Velocity Driver',
             # 'Initial Diagonal Solve',
             # 'IO Writeback'
             ]

# Dycores to try to parse output for
util.variables.dycores = ['glissade'] #["glide", "glissade", "glam", "albany", "bisicles"]

###############################################################################
#                               Main Execution                                #
###############################################################################
if __name__ == '__main__':
    print("------------------------------------------------------------------------------")
    print("  Land Ice Verification & Validation (LIVV)")
    print("------------------------------------------------------------------------------")
    dependencies.check()

    # Check if we are saving/loading the configuration and set up the machine name
    if options.machineName == '' and options.save:
        # Save the configuration with the default host name
        machineName = socket.gethostname()
        util.configurationHandler.save(machineName)
    elif options.save:
        # Save the configuration with the specified host name
        machineName = options.machineName
        util.configurationHandler.save(machineName)
    elif options.machineName == '':
        # Don't save the configuration and use the default host name
        machineName = socket.gethostname()
    else:
        # Try to load the machine name specified
        machineName = options.machineName
        vars = util.configurationHandler.load(machineName)
        util.variables.globals().update(vars)

    # Check if the user has a default config saved and use that if it does
    if os.path.exists(util.variables.configDir + os.sep + machineName + "_" + util.variables.user + "_default"):
        machineName = machineName + "_" + util.variables.user + "_default"
        vars = util.configurationHandler.load(machineName)
        util.variables.globals().update(vars)

    # Print out some information
    print(os.linesep + "  Current run: " + time.strftime("%m-%d-%Y %H:%M:%S"))
    print("  User: " + util.variables.user)
    print("  Config: " + machineName)
    print("  OS Type: " + platform.system() + " " + platform.release())
    print("  " + util.variables.comment + os.linesep)

    # Check to make sure the directory structure is okay
    for dir in [util.variables.inputDir, util.variables.benchmarkDir]:
        if not os.path.exists(dir):
            print("------------------------------------------------------------------------------")
            print("ERROR: Could not find " + dir + " for input")
            print("       Use the -i, -b, and -d flags to specify the locations of the model and comparison data.")
            print("       See README.md for more details.")
            print("------------------------------------------------------------------------------")
            exit(1)

    ###############################################################################
    #                              Record Test Cases                              #
    ###############################################################################
    verificationMapping = {
                           'dome' : verification.dome.Test,
                           'ismip' : verification.ismip.Test,
                           'shelf' : verification.shelf.Test,
                           'stream' : verification.stream.Test
                           }

    performanceMapping = {
                          'dome' : performance.dome.Test,
                          'gis' : performance.gis.Test
                          }

    validationMapping = {
                         'gis' : validation.gis.Test
                         }

    verificationTests = verification.base.choose(options.verification)
    performanceTests = performance.base.choose(options.performance)
    validationTests = validation.base.choose(options.validation)
    testMapping = {
                   "Verification" : verificationTests,
                   "Performance" : performanceTests,
                   "Validation" : validationTests
                   }

    ###############################################################################
    #                               Run Test Cases                                #
    ###############################################################################
    # Give a list of the tests that will be run
    if len(verificationTests) > 0:
        print("Running verification tests:")
        for case in verificationTests:
            print("  " + case)
    if len(performanceTests) > 0:
        print(os.linesep + "Running performance tests:")
        for case in performanceTests:
            print("  " + case)
    if len(validationTests) > 0:
        print(os.linesep + "Running validation tests:")
        for case in validationTests:
            print("  " + case)

    # Set up the directory structure and summary dictionaries for output
    verificationSummary, performanceSummary, validationSummary = dict(), dict(), dict()
    util.websetup.setup(verificationTests + performanceTests + validationTests)

    # Run the verification tests
    if len(verificationTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning verification test suite....")
        print("--------------------------------------------------------------------------")
    for test in verificationTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = verificationMapping[test]()
        newTest.run()
        verificationSummary[test] = newTest.summary
        newTest.generate()
        print("")

    # Run the performance verification
    if len(performanceTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning performance analysis....")
        print("--------------------------------------------------------------------------")
    for test in performanceTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = performanceMapping[test]()
        newTest.run()
        performanceSummary[test] = newTest.summary
        newTest.generate()
        print("")

    # Run the validation verification
    if len(validationTests) > 0:
        print("--------------------------------------------------------------------------")
        print("  Beginning validation test suite....")
        print("--------------------------------------------------------------------------")
    for test in validationTests:
        # Create a new instance of the specific test class (see verificationMapping for the mapping)
        newTest = validationMapping[test]()
        # Run the specific and bit for bit verification for each case of the test
        newTest.run()
        validationSummary[test] = newTest.summary
        # Generate the test-specific webpage
        newTest.generate()
        print("")

    # Create the site index
    print("Generating web pages in " + util.variables.outputDir + "....")
    util.websetup.generate(verificationSummary, performanceSummary, validationSummary)

    print("Cleaning up....")
    util.cleanup.clean()

    ###############################################################################
    #                        Finished.  Tell user about it.                       #
    ###############################################################################
    print("------------------------------------------------------------------------------")
    print("Finished running LIVV.  Results:  ")
    print("  Open " + util.variables.outputDir + "/index.html to see test results")
    print("------------------------------------------------------------------------------")
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import concurrent.futures
import json
import logging
from logging.config import dictConfig

import requests
import slumber
import yaml

dictConfig({
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '%(asctime)s %(levelname)s %(process)d [%(filename)s:%(lineno)d] - %(message)s',
        },
    },
    'handlers': {
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
    },
    'loggers': {
        '': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False
        },
    },
})
logger = logging.getLogger(__name__)

class Resource:
    """Utility class representing an API resource."""
    def __init__(self, name, api):
        self.name = name
        self.api = api

    def save(self):
        """Save JSON corresponding to the resource."""
        logger.info('Saving data for the [{}] resource.'.format(self.name))

        path = 'data/{}.json'.format(self.name)
        with open(path, 'w') as f:
            data = getattr(self.api, self.name).get()
            json.dump(data, f, indent=4, sort_keys=True)

        logger.info('Saved [{resource}] data to [{path}].'.format(resource=self.name, path=path))

if __name__ == '__main__':
    with open('settings.yml') as f:
        settings = yaml.load(f)

    mashape_api_root = settings['mashape_api']['root']
    mashape_api_key = settings['mashape_api']['key']

    headers = {'X-Mashape-Key': mashape_api_key}
    session = requests.Session()
    session.headers.update(headers)
    api = slumber.API(mashape_api_root, session=session)

    resources = [Resource(resource, api) for resource in settings['resources']]

    logger.info(
        'Saving resources. Target resources are: {}.'.format(
            ', '.join(resource.name for resource in resources)
        )
    )

    with concurrent.futures.ThreadPoolExecutor() as executor:
        for resource in resources:
            executor.submit(resource.save)

    logger.info('Resource saving complete.')
<commit_msg>Prevent loaded resources from being reloaded<commit_after>#!/usr/bin/env python3
import concurrent.futures
import json
import logging
from logging.config import dictConfig
import os

import requests
import slumber
import yaml

dictConfig({
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '%(asctime)s %(levelname)s %(process)d [%(filename)s:%(lineno)d] - %(message)s',
        },
    },
    'handlers': {
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
    },
    'loggers': {
        '': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False
        },
    },
})
logger = logging.getLogger(__name__)

class Resource:
    """Utility class representing an API resource."""
    def __init__(self, name, api):
        self.name = name
        self.api = api
        self.path = 'data/{}.json'.format(self.name)

    @property
    def exists(self):
        """Determine if the resource has already been loaded."""
        return os.path.exists(self.path)

    def save(self):
        """Save JSON corresponding to the resource."""
        logger.info('Saving data for the [{}] resource.'.format(self.name))

        with open(self.path, 'w') as f:
            data = getattr(self.api, self.name).get()
            json.dump(data, f, indent=4, sort_keys=True)

        logger.info('Saved [{resource}] data to [{path}].'.format(resource=self.name, path=self.path))

if __name__ == '__main__':
    with open('settings.yml') as f:
        settings = yaml.load(f)

    mashape_api_root = settings['mashape_api']['root']
    mashape_api_key = settings['mashape_api']['key']

    headers = {'X-Mashape-Key': mashape_api_key}
    session = requests.Session()
    session.headers.update(headers)
    api = slumber.API(mashape_api_root, session=session)

    with concurrent.futures.ThreadPoolExecutor() as executor:
        resources = [Resource(resource, api) for resource in settings['resources']]

        for resource in resources:
            # TODO: Add flag to force refresh (for use after a patch is released).
            if resource.exists:
                logger.info('Resource [{name}] found at [{path}].'.format(name=resource.name, path=resource.path))
            else:
                logger.info(
                    'Resource [{name}] not found. Saving to [{path}].'.format(
                        name=resource.name, path=resource.path
                    )
                )
                executor.submit(resource.save)
<|endoftext|>