<commit_before>#
# Copyright (c) 2014, Matthew Turner (maturner01.gmail.com)
#
# For the Tri-state EPSCoR Track II WC-WAVE Project
#
"""
Tools for working with IPW binary data and running the iSNOBAL model
"""

import os
import pandas as pd
import numpy as np

import struct
from collections import namedtuple, defaultdict

#: IPW standard. assumed unchanging since they've been the same for 20 years
BAND_TYPE_LOC = 1
BAND_INDEX_LOC = 2

#: Container for ISNOBAL Global Band information
GlobalBand = namedtuple("GlobalBand", 'byteorder nLines nSamps nBands')

#: Check if a header is starting
IsHeaderStart = lambda headerLine: headerLine.split()[0] == "!<header>"

#: ISNOBAL variable names to be looked up to make dataframes and write metadata
VARNAME_DICT = \
    {
        'in': ["I_lw", "T_a", "e_a", "u", "T_g", "S_n"],
        'em': ["R_n", "H", "L_v_E", "G", "M", "delta_Q", "E_s", "melt",
               "ro_predict", "cc_s"],
        'snow': ["z_s", "rho", "m_s", "h2o", "T_s_0", "T_s_l", "T_s",
                 "z_s_l", "h2o_sat"]
    }

#: Convert number of bytes to struct package code for unsigned integer type
PACK_DICT = \
    {
        1: 'B',
        2: 'H',
        4: 'I'
    }

class IPW:
    """
    Represents an IPW file. Provides a dataFrame attribute to access the
    variables and their floating point representation as a dataframe. The
    dataframe can be modified, the headers recalculated with
    recalculateHeaders, and then written back to IPW binary with
    writeBinary.

    >>> ipw = IPW().from_file("in.0000")
    >>> ipw.dataFrame.T_a = ipw.dataFrame.T_a + 1.0 # add 1 deg C to each temp
    >>> ipw.writeBinary("in.plusOne.000")

    """
    def __init__(self, dataFile):

        ipwLines = IPWLines(dataFile)
        fileType = os.path.basename(dataFile).split('.')[0]

        # _make_bands
        headerDict = _make_bands(ipwLines.headerLines, VARNAME_DICT[fileType])
        geoLines = _parse_geo_header(ipwLines.headerLines)

        bands = [band for band in headerDict.values()]
        nonglobalBands =\
            sorted([band for varname, band in headerDict.iteritems()
                    if varname != 'global'],
                   key=lambda b: b.bandIdx)

        df = _build_ipw_dataframe(nonglobalBands, ipwLines.binaryData)

        self.fileType = fileType
        self.headerDict = headerDict
        self.geoLines = geoLines
        self.bands = bands
        self.nonglobalBands = nonglobalBands
        self.dataFrame = df

    def recalculate_header(self):
        """
        Recalculate header values
        """
        _recalculate_headers(self.nonglobalBands, self.dataFrame)

    def write(self, fileName):
        """
        Write the IPW data to file
        """
        lastLine = "!<header> image -1 $Revision: 1.5 $"

        with open(fileName, 'wb') as f:
            for l in _bands_to_header_lines(self.headerDict):
                f.write(l + '\n')
            for l in self.geoLines:
                # geoLines have not been altered; still have line ends
                f.write(l)

            f.write(lastLine + '\n')

            f.write(_floatdf_to_binstring(self.nonglobalBands, self.dataFrame))

        return None

def _build_ipw_dataframe(nonglobalBands, binaryData):
    """
    Build a pandas DataFrame using header info to assign column names
    """
    colnames = [b.varname for b in nonglobalBands]

    dtype = _bands_to_dtype(nonglobalBands)

    intData = np.fromstring(binaryData, dtype=dtype)

    df = pd.DataFrame(intData, columns=colnames)

    for b in nonglobalBands:
        df[b.varname] = _calc_float_value(b, df[b.varname])

    return df

def _make_bands(headerLines, varnames):
    """
    Make a header dictionary that points to Band objects for each variable
    name.

    Returns: dict
    """
    # parse global information from global header
    for i, l in enumerate(headerLines[1:-1]):
        if IsHeaderStart(l):
            globalEndIdx = i
            break

    globalHeaderLines = headerLines[1:globalEndIdx+1]

    # tried a prettier dictionary comprehension, but wouldn't fly
    globalBandDict = defaultdict(int)
    for l in globalHeaderLines:
        if l:
            spl = l.strip().split()
            if spl[0] == 'byteorder':
                globalBandDict[spl[0]] = spl[2]
            else:
                globalBandDict[spl[0]] = int(spl[2])

    # these are the standard names in an ISNOBAL header file
    byteorder = globalBandDict['byteorder']
    nLines = globalBandDict['nlines']
    nSamps = globalBandDict['nsamps']
    nBands = globalBandDict['nbands']

    # this will be put into the return dictionary at the return statement
    globalBand = GlobalBand(byteorder, nLines, nSamps, nBands)

    # initialize a list of bands to put parsed information into
    bands = [Band() for i in range(nBands)]

    # bandDict = {'global': globalBand}
    for i, b in enumerate(bands):
        b.varname = varnames[i]
        b.bandIdx = i

    bandType = None
    bandIdx = None
    for l in headerLines[globalEndIdx:]:

        spl = l.strip().split()

        if IsHeaderStart(l):

            bandType = spl[BAND_TYPE_LOC]
            bandIdx = int(spl[BAND_INDEX_LOC])

            lqCounter = 0

        elif bandType == 'basic_image':
            # assign byte and bits info that's stored here
            setattr(bands[bandIdx], spl[0] + "_", int(spl[2]))

        elif bandType == 'lq':
            # assign integer and float min and max. ignore non-"map" fields
            if spl[0] == "map":
                # minimum values are listed first
                if lqCounter == 0:
                    bands[bandIdx].intMin = float(spl[2])
                    bands[bandIdx].floatMin = float(spl[3])
                    lqCounter += 1

                elif lqCounter == 1:
                    bands[bandIdx].intMax = float(spl[2])
                    bands[bandIdx].floatMax = float(spl[3])

    return dict(zip(['global']+varnames[:nBands], [globalBand]+bands))

def _parse_geo_header(headerLines):
    """
    For now, just returns all the geographic lines within the header lines.

    Returns: list of geographic header lines ONLY. not the last header line
    """
    firstGeoIdx = 0
    for idx, line in enumerate(headerLines):
        if IsHeaderStart(line) and line.split()[1] == 'geo':
            firstGeoIdx = idx
            break

    # exclude last line because it's just !<header> image -1 $Revision: 1.5 $
    return headerLines[firstGeoIdx:-1]

def _calc_float_value(band, integerValue):
    """
    Calculate a floating point value for the integer int_ given the min/max int
    and min/max floats in the given bandObj

    Returns: Floating point value of the mapped int_
    """
    # TODO create Band.floatRange
    floatRange = band.floatMax - band.floatMin

    # TODO create Band.scaleMult = floatRange/band.intMax
    return integerValue * (floatRange / band.intMax) + band.floatMin

def _bands_to_dtype(bands):
    """
    Given a list of Bands, convert them to a numpy.dtype for use in creating
    the IPW dataframe.
    """
    return np.dtype([(b.varname, 'uint' + str(b.bits_)) for b in bands])

def _bands_to_header_lines(bandsDict):
    """
    Convert the bands to a new header assuming the float ranges are up to date
    for the current dataframe, df.
    """
    firstLine = "!<header> basic_image_i -1 $Revision: 1.11 $"

    global_ = bandsDict['global']

    firstLines = [firstLine,
                  "byteorder = {0} ".format(global_.byteorder),
                  "nlines = {0} ".format(global_.nLines),
                  "nsamps = {0} ".format(global_.nSamps),
                  "nbands = {0} ".format(global_.nBands)]

    otherLines = []
    bands = [b for varname, b in bandsDict.iteritems() if varname != 'global']

    bands = sorted(bands, key=lambda b: b.bandIdx)

    # for some reason IPW has a space at the end of data lines
    for i, b in enumerate(bands):
        otherLines += ["!<header> basic_image {0} $Revision: 1.11 $".format(i),
                       "bytes = {0} ".format(b.bytes_),
                       "bits = {0} ".format(b.bits_)]

    for i, b in enumerate(bands):
        # IPW writes integer floats without even a dec point, so strip decimal
        intMin = int(b.intMin)
        intMax = int(b.intMax)
        floatMin = (b.floatMin, int(b.floatMin))[b.floatMin == int(b.floatMin)]
        floatMax = (b.floatMax, int(b.floatMax))[b.floatMax == int(b.floatMax)]
        otherLines += ["!<header> lq {0} $Revision: 1.6 $".format(i),
                       "map = {0} {1} ".format(intMin, floatMin),
                       "map = {0} {1} ".format(intMax, floatMax)]

    return firstLines + otherLines

def _floatdf_to_binstring(bands, df):
    """
    Convert the dataframe floating point data to a binary string.
    """
    # first convert df to an integer dataframe
    intDf = pd.DataFrame(dtype='uint64')

    for b in bands:
        # check that bands are appropriately made, that b.Max/Min really are
        assert (b.floatMax >= df[b.varname]).all(), \
            "Bad band: max not really max.\nb.floatMax = %s\n \
            df[b.varname].max()  = %s" % (b.floatMax, df[b.varname].max())
        assert (b.floatMin <= df[b.varname]).all(), \
            "Bad band: min not really min.\nb.floatMin = %s\n \
            df[b.varname].min()  = %s" % (b.floatMin, df[b.varname].min())

        # no need to include b.intMin, it's always zero
        mapFn = lambda x: \
            np.round(((x - b.floatMin) * b.intMax)/(b.floatMax - b.floatMin))

        intDf[b.varname] = mapFn(df[b.varname])

    packStr = "".join([PACK_DICT[b.bytes_] for b in bands])

    return "".join([struct.pack(packStr, *r[1]) for r in intDf.iterrows()])

# TODO: rename this to _recalculate_header
def _recalculate_headers(bands, dataframe):
    """
    Recalculate the minimum and maximum of each band in bands given a dataframe
    that contains data for each band.

    Returns: None
    """
    assert list(dataframe.columns) == [b.varname for b in bands], \
        "DataFrame column names do not match bands' variable names!"

    for band in bands:
        band.floatMin = dataframe[band.varname].min()
        band.floatMax = dataframe[band.varname].max()

        if band.floatMin == band.floatMax:
            band.floatMax = band.floatMin + 1.0

    return None

class Band(object):
    """
    Container for band information
    """
    def __init__(self, varname="", bandIdx=0, nBytes=0, nBits=0, intMin=0.0,
                 intMax=0.0, floatMin=0.0, floatMax=0.0):
        """
        Can either pass this information or create an all-None Band.
        """
        self.varname = varname

        self.bandIdx = bandIdx

        self.bytes_ = nBytes
        self.bits_ = nBits

        self.intMin = float(intMin)
        self.intMax = float(intMax)
        self.floatMin = float(floatMin)
        self.floatMax = float(floatMax)

class IPWLines:
    """
    Data structure to wrap header and binary parts of an IPW file.

    Arguments: ipwFile -- file name pointing to an IPW file
    """
    def __init__(self, ipwFile):

        with open(ipwFile, 'rb') as f:
            lines = f.readlines()

        self.headerLines = lines[:-1]

        self.binaryData = lines[-1]

<commit_msg>added ISNOBAL wrapper with function isnobal in isnobal_adaptor<commit_after>#
# Copyright (c) 2014, Matthew Turner (maturner01.gmail.com)
#
# For the Tri-state EPSCoR Track II WC-WAVE Project
#
"""
Tools for working with IPW binary data and running the iSNOBAL model
"""

import os
import logging
import pandas as pd
import numpy as np
import subprocess
import struct

from collections import namedtuple, defaultdict

#: IPW standard. assumed unchanging since they've been the same for 20 years
BAND_TYPE_LOC = 1
BAND_INDEX_LOC = 2

#: Container for ISNOBAL Global Band information
GlobalBand = namedtuple("GlobalBand", 'byteorder nLines nSamps nBands')

#: Check if a header is starting
IsHeaderStart = lambda headerLine: headerLine.split()[0] == "!<header>"

#: ISNOBAL variable names to be looked up to make dataframes and write metadata
VARNAME_DICT = \
    {
        'in': ["I_lw", "T_a", "e_a", "u", "T_g", "S_n"],
        'em': ["R_n", "H", "L_v_E", "G", "M", "delta_Q", "E_s", "melt",
               "ro_predict", "cc_s"],
        'snow': ["z_s", "rho", "m_s", "h2o", "T_s_0", "T_s_l", "T_s",
                 "z_s_l", "h2o_sat"]
    }

#: Convert number of bytes to struct package code for unsigned integer type
PACK_DICT = \
    {
        1: 'B',
        2: 'H',
        4: 'I'
    }

def isnobal(data_tstep=60, nsteps=8754, init_img="data/init.ipw",
            precip_file="data/ppt_desc", mask_file="tl2p5mask.ipw",
            input_prefix="in", output_frequency=1):
    """
    Wrapper for running the ISNOBAL
    (http://cgiss.boisestate.edu/~hpm/software/IPW/man1/isnobal.html) model.
    """
    isnobalcmd = "".join(["isnobal ",
                          " -t " + data_tstep,
                          " -n " + nsteps,
                          " -I " + init_img,
                          " -p " + precip_file,
                          " -m " + mask_file,
                          " -i " + input_prefix,
                          " -O " + output_frequency,
                          " -e em ",
                          " -s snow"])
    logging.debug("ISNOBAL shell command: " + isnobalcmd)

    output = subprocess.check_output(isnobalcmd, shell=True)

    logging.debug("ISNOBAL process output: " + output)

class IPW:
    """
    Represents an IPW file. Provides a dataFrame attribute to access the
    variables and their floating point representation as a dataframe. The
    dataframe can be modified, the headers recalculated with
    recalculateHeaders, and then written back to IPW binary with
    writeBinary.

    >>> ipw = IPW().from_file("in.0000")
    >>> ipw.dataFrame.T_a = ipw.dataFrame.T_a + 1.0 # add 1 deg C to each temp
    >>> ipw.writeBinary("in.plusOne.000")

    """
    def __init__(self, dataFile):

        ipwLines = IPWLines(dataFile)
        fileType = os.path.basename(dataFile).split('.')[0]

        # _make_bands
        headerDict = _make_bands(ipwLines.headerLines, VARNAME_DICT[fileType])
        geoLines = _parse_geo_header(ipwLines.headerLines)

        bands = [band for band in headerDict.values()]
        nonglobalBands =\
            sorted([band for varname, band in headerDict.iteritems()
                    if varname != 'global'],
                   key=lambda b: b.bandIdx)

        df = _build_ipw_dataframe(nonglobalBands, ipwLines.binaryData)

        self.fileType = fileType
        self.headerDict = headerDict
        self.geoLines = geoLines
        self.bands = bands
        self.nonglobalBands = nonglobalBands
        self.dataFrame = df

    def recalculate_header(self):
        """
        Recalculate header values
        """
        _recalculate_headers(self.nonglobalBands, self.dataFrame)

    def write(self, fileName):
        """
        Write the IPW data to file
        """
        lastLine = "!<header> image -1 $Revision: 1.5 $"

        with open(fileName, 'wb') as f:
            for l in _bands_to_header_lines(self.headerDict):
                f.write(l + '\n')
            for l in self.geoLines:
                # geoLines have not been altered; still have line ends
                f.write(l)

            f.write(lastLine + '\n')

            f.write(_floatdf_to_binstring(self.nonglobalBands, self.dataFrame))

        return None

def _build_ipw_dataframe(nonglobalBands, binaryData):
    """
    Build a pandas DataFrame using header info to assign column names
    """
    colnames = [b.varname for b in nonglobalBands]

    dtype = _bands_to_dtype(nonglobalBands)

    intData = np.fromstring(binaryData, dtype=dtype)

    df = pd.DataFrame(intData, columns=colnames)

    for b in nonglobalBands:
        df[b.varname] = _calc_float_value(b, df[b.varname])

    return df

def _make_bands(headerLines, varnames):
    """
    Make a header dictionary that points to Band objects for each variable
    name.

    Returns: dict
    """
    # parse global information from global header
    for i, l in enumerate(headerLines[1:-1]):
        if IsHeaderStart(l):
            globalEndIdx = i
            break

    globalHeaderLines = headerLines[1:globalEndIdx+1]

    # tried a prettier dictionary comprehension, but wouldn't fly
    globalBandDict = defaultdict(int)
    for l in globalHeaderLines:
        if l:
            spl = l.strip().split()
            if spl[0] == 'byteorder':
                globalBandDict[spl[0]] = spl[2]
            else:
                globalBandDict[spl[0]] = int(spl[2])

    # these are the standard names in an ISNOBAL header file
    byteorder = globalBandDict['byteorder']
    nLines = globalBandDict['nlines']
    nSamps = globalBandDict['nsamps']
    nBands = globalBandDict['nbands']

    # this will be put into the return dictionary at the return statement
    globalBand = GlobalBand(byteorder, nLines, nSamps, nBands)

    # initialize a list of bands to put parsed information into
    bands = [Band() for i in range(nBands)]

    # bandDict = {'global': globalBand}
    for i, b in enumerate(bands):
        b.varname = varnames[i]
        b.bandIdx = i

    bandType = None
    bandIdx = None
    for l in headerLines[globalEndIdx:]:

        spl = l.strip().split()

        if IsHeaderStart(l):

            bandType = spl[BAND_TYPE_LOC]
            bandIdx = int(spl[BAND_INDEX_LOC])

            lqCounter = 0

        elif bandType == 'basic_image':
            # assign byte and bits info that's stored here
            setattr(bands[bandIdx], spl[0] + "_", int(spl[2]))

        elif bandType == 'lq':
            # assign integer and float min and max. ignore non-"map" fields
            if spl[0] == "map":
                # minimum values are listed first
                if lqCounter == 0:
                    bands[bandIdx].intMin = float(spl[2])
                    bands[bandIdx].floatMin = float(spl[3])
                    lqCounter += 1

                elif lqCounter == 1:
                    bands[bandIdx].intMax = float(spl[2])
                    bands[bandIdx].floatMax = float(spl[3])

    return dict(zip(['global']+varnames[:nBands], [globalBand]+bands))

def _parse_geo_header(headerLines):
    """
    For now, just returns all the geographic lines within the header lines.

    Returns: list of geographic header lines ONLY. not the last header line
    """
    firstGeoIdx = 0
    for idx, line in enumerate(headerLines):
        if IsHeaderStart(line) and line.split()[1] == 'geo':
            firstGeoIdx = idx
            break

    # exclude last line because it's just !<header> image -1 $Revision: 1.5 $
    return headerLines[firstGeoIdx:-1]

def _calc_float_value(band, integerValue):
    """
    Calculate a floating point value for the integer int_ given the min/max int
    and min/max floats in the given bandObj

    Returns: Floating point value of the mapped int_
    """
    # TODO create Band.floatRange
    floatRange = band.floatMax - band.floatMin

    # TODO create Band.scaleMult = floatRange/band.intMax
    return integerValue * (floatRange / band.intMax) + band.floatMin

def _bands_to_dtype(bands):
    """
    Given a list of Bands, convert them to a numpy.dtype for use in creating
    the IPW dataframe.
    """
    return np.dtype([(b.varname, 'uint' + str(b.bits_)) for b in bands])

def _bands_to_header_lines(bandsDict):
    """
    Convert the bands to a new header assuming the float ranges are up to date
    for the current dataframe, df.
    """
    firstLine = "!<header> basic_image_i -1 $Revision: 1.11 $"

    global_ = bandsDict['global']

    firstLines = [firstLine,
                  "byteorder = {0} ".format(global_.byteorder),
                  "nlines = {0} ".format(global_.nLines),
                  "nsamps = {0} ".format(global_.nSamps),
                  "nbands = {0} ".format(global_.nBands)]

    otherLines = []
    bands = [b for varname, b in bandsDict.iteritems() if varname != 'global']

    bands = sorted(bands, key=lambda b: b.bandIdx)

    # for some reason IPW has a space at the end of data lines
    for i, b in enumerate(bands):
        otherLines += ["!<header> basic_image {0} $Revision: 1.11 $".format(i),
                       "bytes = {0} ".format(b.bytes_),
                       "bits = {0} ".format(b.bits_)]

    for i, b in enumerate(bands):
        # IPW writes integer floats without even a dec point, so strip decimal
        intMin = int(b.intMin)
        intMax = int(b.intMax)
        floatMin = (b.floatMin, int(b.floatMin))[b.floatMin == int(b.floatMin)]
        floatMax = (b.floatMax, int(b.floatMax))[b.floatMax == int(b.floatMax)]
        otherLines += ["!<header> lq {0} $Revision: 1.6 $".format(i),
                       "map = {0} {1} ".format(intMin, floatMin),
                       "map = {0} {1} ".format(intMax, floatMax)]

    return firstLines + otherLines

def _floatdf_to_binstring(bands, df):
    """
    Convert the dataframe floating point data to a binary string.
    """
    # first convert df to an integer dataframe
    intDf = pd.DataFrame(dtype='uint64')

    for b in bands:
        # check that bands are appropriately made, that b.Max/Min really are
        assert (b.floatMax >= df[b.varname]).all(), \
            "Bad band: max not really max.\nb.floatMax = %s\n \
            df[b.varname].max()  = %s" % (b.floatMax, df[b.varname].max())
        assert (b.floatMin <= df[b.varname]).all(), \
            "Bad band: min not really min.\nb.floatMin = %s\n \
            df[b.varname].min()  = %s" % (b.floatMin, df[b.varname].min())

        # no need to include b.intMin, it's always zero
        mapFn = lambda x: \
            np.round(((x - b.floatMin) * b.intMax)/(b.floatMax - b.floatMin))

        intDf[b.varname] = mapFn(df[b.varname])

    packStr = "".join([PACK_DICT[b.bytes_] for b in bands])

    return "".join([struct.pack(packStr, *r[1]) for r in intDf.iterrows()])

# TODO: rename this to _recalculate_header
def _recalculate_headers(bands, dataframe):
    """
    Recalculate the minimum and maximum of each band in bands given a dataframe
    that contains data for each band.

    Returns: None
    """
    assert list(dataframe.columns) == [b.varname for b in bands], \
        "DataFrame column names do not match bands' variable names!"

    for band in bands:
        band.floatMin = dataframe[band.varname].min()
        band.floatMax = dataframe[band.varname].max()

        if band.floatMin == band.floatMax:
            band.floatMax = band.floatMin + 1.0

    return None

class Band(object):
    """
    Container for band information
    """
    def __init__(self, varname="", bandIdx=0, nBytes=0, nBits=0, intMin=0.0,
                 intMax=0.0, floatMin=0.0, floatMax=0.0):
        """
        Can either pass this information or create an all-None Band.
        """
        self.varname = varname

        self.bandIdx = bandIdx

        self.bytes_ = nBytes
        self.bits_ = nBits

        self.intMin = float(intMin)
        self.intMax = float(intMax)
        self.floatMin = float(floatMin)
        self.floatMax = float(floatMax)

class IPWLines:
    """
    Data structure to wrap header and binary parts of an IPW file.

    Arguments: ipwFile -- file name pointing to an IPW file
    """
    def __init__(self, ipwFile):

        with open(ipwFile, 'rb') as f:
            lines = f.readlines()

        self.headerLines = lines[:-1]

        self.binaryData = lines[-1]

<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import bisect
import getopt
import logging
import os
import random
import re
import string
import sys

from math import sqrt
from threading import currentThread, Thread
from time import time

try:
    from httplib import HTTPConnection
except ImportError:
    from http.client import HTTPConnection

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

__program__ = 'pyspeedtest'
__version__ = '1.2'

class SpeedTest(object):

    DOWNLOAD_FILES = [
        '/speedtest/random350x350.jpg',
        '/speedtest/random500x500.jpg',
        '/speedtest/random1500x1500.jpg'
    ]

    UPLOAD_FILES = [
        132884,
        493638
    ]

    def __init__(self, host=None, verbose=False, http_debug=0):
        self.host = host
        self.verbose = verbose
        self.http_debug = http_debug

    def downloadthread(self, connection, url):
        connection.request('GET', url, None, {'Connection': 'Keep-Alive'})
        response = connection.getresponse()
        self_thread = currentThread()
        self_thread.downloaded = len(response.read())

    def download(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        total_downloaded = 0
        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)
        total_start_time = time()
        for current_file in SpeedTest.DOWNLOAD_FILES:
            threads = []
            for run in range(runs):
                thread = Thread(
                    target=self.downloadthread,
                    args=(connections[run],
                          '%s?x=%d' % (current_file, int(time() * 1000))))
                thread.run_number = run
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                total_downloaded += thread.downloaded
                logging.info('Run %d for %s finished',
                             thread.run_number, current_file)
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to download %d bytes',
                     total_ms, total_downloaded)
        return total_downloaded * 8000 / total_ms

    def uploadthread(self, connection, data):
        url = '/speedtest/upload.php?x=%s' % random.random()
        connection.request('POST', url, data, {
            'Connection': 'Keep-Alive',
            'Content-Type': 'application/x-www-form-urlencoded'
        })
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        self_thread = currentThread()
        self_thread.uploaded = int(reply.split('=')[1])

    def upload(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)

        post_data = []
        alphabet = string.digits + string.ascii_letters
        for current_file_size in SpeedTest.UPLOAD_FILES:
            values = {
                'content0': ''.join(
                    random.choice(alphabet) for i in range(current_file_size))
            }
            post_data.append(urlencode(values))

        total_uploaded = 0
        total_start_time = time()
        for data in post_data:
            threads = []
            for run in range(runs):
                thread = Thread(target=self.uploadthread,
                                args=(connections[run], data))
                thread.run_number = run
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                logging.info('Run %d for %d bytes finished',
                             thread.run_number, thread.uploaded)
                total_uploaded += thread.uploaded
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to upload %d bytes',
                     total_ms, total_uploaded)
        return total_uploaded * 8000 / total_ms

    def ping(self, server=None):
        if server is None:
            server = self.host

        if server is None:
            raise Exception('No server specified')

        connection = HTTPConnection(server)
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        times = []
        worst = 0
        for _ in range(5):
            total_start_time = time()
            connection.request(
                'GET',
                '/speedtest/latency.txt?x=%d' % random.random(),
                None,
                {'Connection': 'Keep-Alive'})
            response = connection.getresponse()
            response.read()
            total_ms = time() - total_start_time
            times.append(total_ms)
            if total_ms > worst:
                worst = total_ms
        times.remove(worst)
        total_ms = sum(times) * 250  # * 1000 / number of tries (4) = 250
        connection.close()
        logging.info('Latency for %s - %d', server, total_ms)
        return total_ms

    def chooseserver(self):
        connection = HTTPConnection('www.speedtest.net')
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        now = int(time() * 1000)
        extra_headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
        }
        connection.request(
            'GET', '/speedtest-config.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        match = re.search(
            r'<client ip="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        location = None
        if match is None:
            logging.info('Failed to retrieve coordinates')
            return None
        location = match.groups()
        logging.info('Your IP: %s\nYour latitude: %s\nYour longitude: %s' %
                     location)
        connection.request(
            'GET', '/speedtest-servers.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        server_list = re.findall(
            r'<server url="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        my_lat = float(location[1])
        my_lon = float(location[2])
        sorted_server_list = []
        for server in server_list:
            s_lat = float(server[1])
            s_lon = float(server[2])
            distance = sqrt(pow(s_lat - my_lat, 2) + pow(s_lon - my_lon, 2))
            bisect.insort_left(sorted_server_list, (distance, server[0]))
        best_server = (999999, '')
        for server in sorted_server_list[:10]:
            logging.info(server[1])
            match = re.search(
                r'http://([^/]+)/speedtest/upload\.php', server[1])
            if match is None:
                continue
            server_host = match.groups()[0]
            latency = self.ping(server_host)
            if latency < best_server[0]:
                best_server = (latency, server_host)
        print('Using server:', best_server[1])
        return best_server[1]

def error(*args):
    print(*args, file=sys.stderr)

def usage():
    return '''\
version: %s

usage: %s [-h] [-v] [-r N] [-m M] [-d L]

Test your bandwidth speed using Speedtest.net servers.

optional arguments:
  -h, --help         show this help message and exit
  -v                 enabled verbose mode
  -r N, --runs=N     use N runs (default is 2)
  -m M, --mode=M     test mode: 1 - download
                               2 - upload
                               4 - ping
                               1 + 2 + 4 = 7 - all (default)
  -d L, --debug=L    set http connection debug level (default is 0)
  -s H, --server=H   use specific server
''' % (__version__, os.path.basename(sys.argv[0]))

def parseargs():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:],
            'hr:vm:d:s:',
            [
                'help',
                'runs=',
                'mode=',
                'debug=',
                'server=',
            ]
        )
    except getopt.GetoptError as err:
        error(err)
        error()
        error(usage())
        sys.exit(1)
    else:
        return opts

def main():
    opts = parseargs()
    speedtest = SpeedTest()

    mode = 7
    runs = 2

    for opt, arg in opts:
        if opt == '-v':
            speedtest.verbose = True
        elif opt in ('-h', '--help'):
            print(usage())
            sys.exit()
        elif opt in ('-r', '--runs'):
            try:
                runs = int(arg)
            except ValueError:
                error('Bad runs value')
                sys.exit(1)
        elif opt in ('-m', '--mode'):
            try:
                mode = int(arg)
            except ValueError:
                error('Bad mode value')
                sys.exit(1)
        elif opt in ('-d', '--debug'):
            try:
                speedtest.http_debug = int(arg)
            except ValueError:
                error('Bad debug value')
                sys.exit(1)
        elif opt == '-s':
            speedtest.host = arg

    logging.basicConfig(
        format='%(message)s',
        level=logging.INFO if speedtest.verbose else logging.ERROR)

    if mode & 4 == 4 and speedtest.host is not None:
        print('Ping: %d ms' % speedtest.ping())

    if mode & 1 == 1:
        print('Download speed: %s' % pretty_speed(speedtest.download(runs)))

    if mode & 2 == 2:
        print('Upload speed: %s' % pretty_speed(speedtest.upload(runs)))

def pretty_speed(speed):
    units = ['bps', 'Kbps', 'Mbps', 'Gbps']
    unit = 0
    while speed >= 1024:
        speed /= 1024
        unit += 1
    return '%0.2f %s' % (speed, units[unit])

if __name__ == '__main__':
    main()
<commit_msg>Fix indentation of usage message<commit_after>#!/usr/bin/env python

from __future__ import print_function

import bisect
import getopt
import logging
import os
import random
import re
import string
import sys

from math import sqrt
from threading import currentThread, Thread
from time import time

try:
    from httplib import HTTPConnection
except ImportError:
    from http.client import HTTPConnection

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

__program__ = 'pyspeedtest'
__version__ = '1.2'

class SpeedTest(object):

    DOWNLOAD_FILES = [
        '/speedtest/random350x350.jpg',
        '/speedtest/random500x500.jpg',
        '/speedtest/random1500x1500.jpg'
    ]

    UPLOAD_FILES = [
        132884,
        493638
    ]

    def __init__(self, host=None, verbose=False, http_debug=0):
        self.host = host
        self.verbose = verbose
        self.http_debug = http_debug

    def downloadthread(self, connection, url):
        connection.request('GET', url, None, {'Connection': 'Keep-Alive'})
        response = connection.getresponse()
        self_thread = currentThread()
        self_thread.downloaded = len(response.read())

    def download(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        total_downloaded = 0
        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)
        total_start_time = time()
        for current_file in SpeedTest.DOWNLOAD_FILES:
            threads = []
            for run in range(runs):
                thread = Thread(
                    target=self.downloadthread,
                    args=(connections[run],
                          '%s?x=%d' % (current_file, int(time() * 1000))))
                thread.run_number = run
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                total_downloaded += thread.downloaded
                logging.info('Run %d for %s finished',
                             thread.run_number, current_file)
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to download %d bytes',
                     total_ms, total_downloaded)
        return total_downloaded * 8000 / total_ms

    def uploadthread(self, connection, data):
        url = '/speedtest/upload.php?x=%s' % random.random()
        connection.request('POST', url, data, {
            'Connection': 'Keep-Alive',
            'Content-Type': 'application/x-www-form-urlencoded'
        })
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        self_thread = currentThread()
        self_thread.uploaded = int(reply.split('=')[1])

    def upload(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)

        post_data = []
        alphabet = string.digits + string.ascii_letters
        for current_file_size in SpeedTest.UPLOAD_FILES:
            values = {
                'content0': ''.join(
                    random.choice(alphabet) for i in range(current_file_size))
            }
            post_data.append(urlencode(values))

        total_uploaded = 0
        total_start_time = time()
        for data in post_data:
            threads = []
            for run in range(runs):
                thread = Thread(target=self.uploadthread,
                                args=(connections[run], data))
                thread.run_number = run
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                logging.info('Run %d for %d bytes finished',
                             thread.run_number, thread.uploaded)
                total_uploaded += thread.uploaded
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to upload %d bytes',
                     total_ms, total_uploaded)
        return total_uploaded * 8000 / total_ms

    def ping(self, server=None):
        if server is None:
            server = self.host

        if server is None:
            raise Exception('No server specified')

        connection = HTTPConnection(server)
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        times = []
        worst = 0
        for _ in range(5):
            total_start_time = time()
            connection.request(
                'GET',
                '/speedtest/latency.txt?x=%d' % random.random(),
                None,
                {'Connection': 'Keep-Alive'})
            response = connection.getresponse()
            response.read()
            total_ms = time() - total_start_time
            times.append(total_ms)
            if total_ms > worst:
                worst = total_ms
        times.remove(worst)
        total_ms = sum(times) * 250  # * 1000 / number of tries (4) = 250
        connection.close()
        logging.info('Latency for %s - %d', server, total_ms)
        return total_ms

    def chooseserver(self):
        connection = HTTPConnection('www.speedtest.net')
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        now = int(time() * 1000)
        extra_headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
        }
        connection.request(
            'GET', '/speedtest-config.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        match = re.search(
            r'<client ip="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        location = None
        if match is None:
            logging.info('Failed to retrieve coordinates')
            return None
        location = match.groups()
        logging.info('Your IP: %s\nYour latitude: %s\nYour longitude: %s' %
                     location)
        connection.request(
            'GET', '/speedtest-servers.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        server_list = re.findall(
            r'<server url="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        my_lat = float(location[1])
        my_lon = float(location[2])
        sorted_server_list = []
        for server in server_list:
            s_lat = float(server[1])
            s_lon = float(server[2])
            distance = sqrt(pow(s_lat - my_lat, 2) + pow(s_lon - my_lon, 2))
            bisect.insort_left(sorted_server_list, (distance, server[0]))
        best_server = (999999, '')
        for server in sorted_server_list[:10]:
            logging.info(server[1])
            match = re.search(
                r'http://([^/]+)/speedtest/upload\.php', server[1])
            if match is None:
                continue
            server_host = match.groups()[0]
            latency = self.ping(server_host)
            if latency < best_server[0]:
                best_server = (latency, server_host)
        print('Using server:', best_server[1])
        return best_server[1]

def error(*args):
    print(*args, file=sys.stderr)

def usage():
    return '''\
version: %s

usage: %s [-h] [-v] [-r N] [-m M] [-d L]

Test your bandwidth speed using Speedtest.net servers.

optional arguments:
  -h, --help         show this help message and exit
  -v                 enabled verbose mode
  -r N, --runs=N     use N runs (default is 2)
  -m M, --mode=M     test mode: 1 - download
                                2 - upload
                                4 - ping
                                1 + 2 + 4 = 7 - all (default)
  -d L, --debug=L    set http connection debug level (default is 0)
  -s H, --server=H   use specific server
''' % (__version__, os.path.basename(sys.argv[0]))

def parseargs():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:],
            'hr:vm:d:s:',
            [
                'help',
                'runs=',
                'mode=',
                'debug=',
                'server=',
            ]
        )
    except getopt.GetoptError as err:
        error(err)
        error()
        error(usage())
        sys.exit(1)
    else:
        return opts

def main():
    opts = parseargs()
    speedtest = SpeedTest()

    mode = 7
    runs = 2

    for opt, arg in opts:
        if opt == '-v':
            speedtest.verbose = True
        elif opt in ('-h', '--help'):
            print(usage())
            sys.exit()
        elif opt in ('-r', '--runs'):
            try:
                runs = int(arg)
            except ValueError:
                error('Bad runs value')
                sys.exit(1)
        elif opt in ('-m', '--mode'):
            try:
                mode = int(arg)
            except ValueError:
                error('Bad mode value')
                sys.exit(1)
        elif opt in ('-d', '--debug'):
            try:
                speedtest.http_debug = int(arg)
            except ValueError:
                error('Bad debug value')
                sys.exit(1)
        elif opt == '-s':
            speedtest.host = arg

    logging.basicConfig(
        format='%(message)s',
        level=logging.INFO if speedtest.verbose else logging.ERROR)

    if mode & 4 == 4 and speedtest.host is not None:
        print('Ping: %d ms' % speedtest.ping())

    if mode & 1 == 1:
        print('Download speed: %s' % pretty_speed(speedtest.download(runs)))

    if mode & 2 == 2:
        print('Upload speed: %s' % pretty_speed(speedtest.upload(runs)))

def pretty_speed(speed):
    units = ['bps', 'Kbps', 'Mbps', 'Gbps']
    unit = 0
    while speed >= 1024:
        speed /= 1024
        unit += 1
    return '%0.2f %s' % (speed, units[unit])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import random
import string

from passlib.handlers.sha2_crypt import sha256_crypt

from app.account.user import User
from app.app import app
from app.rest import execute_post

def hash_password(accounts: list):
    for account in accounts:
        if account['active']:
            account['password'] = sha256_crypt.encrypt(account['password'])

def add_token(documents: list):
    for document in documents:
        token = generate_token()
        while app.data.driver.db['accounts'].find_one({'token': token}) is not None:
            token = generate_token()
        document["token"] = token

def generate_token() -> str:
    return (''.join(random.choice(string.ascii_uppercase)
                    for x in range(10)))

def set_byUser(resource_name: str, items: list):
    from app.app import app
    if 'byUser' in app.config['DOMAIN'][resource_name]['schema']:
        for item in items:
            item['byUser'] = User.actual['_id']

def set_default_database_if_empty(accounts: list):
    for account in accounts:
        if 'defaultDatabase' not in account:
            account['defaultDatabase'] = account['databases'][0]

def add_or_get_inactive_account(events: list):
    # todo if register we need to make sure that user does not add the account again another time (usability?)
    for event in events:
        if event['@type'] == 'Receive':
            _add_or_get_inactive_account_id(event, 'unregisteredReceiver', 'receiver')
        elif event['@type'] == 'Allocate':
            _add_or_get_inactive_account_id(event, 'unregisteredTo', 'to')

def _add_or_get_inactive_account_id(event, field_name, recipient_field_name):
    if field_name in event:
        try:
            _id = app.data.driver.db.accounts.find_one(
                {
                    'email': event[field_name]['email'],
                    'databases': {'$in': User.actual['databases']}  # We look for just accounts that share our database
                }
            )['_id']
        except TypeError:  # No account
            event[field_name]['databases'] = User.actual['databases']
            event[field_name]['active'] = False
            _id = execute_post('accounts', event[field_name])['_id']
        event[recipient_field_name] = _id
        del event[field_name]
<commit_msg>Fix account creation per superusers without specific databases<commit_after>import random
import string

from passlib.handlers.sha2_crypt import sha256_crypt

from app.account.user import User, Role
from app.app import app
from app.rest import execute_post

def hash_password(accounts: list):
    for account in accounts:
        if account['active']:
            account['password'] = sha256_crypt.encrypt(account['password'])

def add_token(documents: list):
    for document in documents:
        token = generate_token()
        while app.data.driver.db['accounts'].find_one({'token': token}) is not None:
            token = generate_token()
        document["token"] = token

def generate_token() -> str:
    return (''.join(random.choice(string.ascii_uppercase)
                    for x in range(10)))

def set_byUser(resource_name: str, items: list):
    from app.app import app
    if 'byUser' in app.config['DOMAIN'][resource_name]['schema']:
        for item in items:
            item['byUser'] = User.actual['_id']

def set_default_database_if_empty(accounts: list):
    for account in accounts:
        if 'defaultDatabase' not in account and account['role'] != Role.SUPERUSER:
            account['defaultDatabase'] = account['databases'][0]

def add_or_get_inactive_account(events: list):
    # todo if register we need to make sure that user does not add the account again another time (usability?)
    for event in events:
        if event['@type'] == 'Receive':
            _add_or_get_inactive_account_id(event, 'unregisteredReceiver', 'receiver')
        elif event['@type'] == 'Allocate':
            _add_or_get_inactive_account_id(event, 'unregisteredTo', 'to')

def _add_or_get_inactive_account_id(event, field_name, recipient_field_name):
    if field_name in event:
        try:
            _id = app.data.driver.db.accounts.find_one(
                {
                    'email': event[field_name]['email'],
                    'databases': {'$in': User.actual['databases']}  # We look for just accounts that share our database
                }
            )['_id']
        except TypeError:  # No account
            event[field_name]['databases'] = User.actual['databases']
            event[field_name]['active'] = False
            _id = execute_post('accounts', event[field_name])['_id']
        event[recipient_field_name] = _id
        del event[field_name]
<|endoftext|>


<commit_before># Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import sys

from glanceclient.common import utils
import glanceclient.v1.images

@utils.arg('--name', metavar='<NAME>',
           help='Filter images to those that have this name.')
@utils.arg('--status', metavar='<STATUS>',
           help='Filter images to those that have this status.')
@utils.arg('--container-format', metavar='<CONTAINER_FORMAT>',
           help='Filter images to those that have this container format.')
@utils.arg('--disk-format', metavar='<DISK_FORMAT>',
           help='Filter images to those that have this disk format.')
@utils.arg('--size-min', metavar='<SIZE>',
           help='Filter images to those with a size greater than this.')
@utils.arg('--size-max', metavar='<SIZE>',
           help='Filter images to those with a size less than this.')
@utils.arg('--property-filter', metavar='<KEY=VALUE>',
           help="Filter images by a user-defined image property.",
            action='append', dest='properties', default=[])
def do_image_list(gc, args):
    """List images."""
    filter_keys = ['name', 'status', 'container_format', 'disk_format',
               'size_min', 'size_max']
    filter_items = [(key, getattr(args, key)) for key in filter_keys]
    filters = dict([item for item in filter_items if item[1] is not None])

    if args.properties:
        property_filter_items = [p.split('=', 1) for p in args.properties]
        filters['properties'] = dict(property_filter_items)

    images = gc.images.list(filters=filters)
    columns = ['ID', 'Name', 'Disk Format', 'Container Format',
               'Size', 'Status']
    utils.print_list(images, columns)

def _image_show(image):
    # Flatten image properties dict for display
    info = copy.deepcopy(image._info)
    for (k, v) in info.pop('properties').iteritems():
        info['Property \'%s\'' % k] = v

    utils.print_dict(info)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to describe.')
def do_image_show(gc, args):
    """Describe a specific image."""
    image = gc.images.get(args.id)
    _image_show(image)

@utils.arg('--id', metavar='<IMAGE_ID>',
           help='ID of image to reserve.')
@utils.arg('--name', metavar='<NAME>',
           help='Name of image.')
@utils.arg('--disk-format', metavar='<CONTAINER_FORMAT>',
           help='Disk format of image.')
@utils.arg('--container-format', metavar='<DISK_FORMAT>',
           help='Container format of image.')
@utils.arg('--owner', metavar='<TENANT_ID>',
           help='Tenant who should own image.')
@utils.arg('--size', metavar='<SIZE>',
           help=('Size of image data (in bytes). Only used with'
                 ' \'--location\' and \'--copy_from\'.'))
@utils.arg('--min-disk', metavar='<DISK_GB>',
           help='Minimum size of disk needed to boot image (in gigabytes).')
@utils.arg('--min-ram', metavar='<DISK_RAM>',
           help='Minimum amount of ram needed to boot image (in megabytes).')
@utils.arg('--location', metavar='<IMAGE_URL>',
           help=('URL where the data for this image already resides.'
                 ' For example, if the image data is stored in the filesystem'
                 ' local to the glance server at \'/usr/share/image.tar.gz\','
                 ' you would specify \'file:///usr/share/image.tar.gz\'.'))
@utils.arg('--file', metavar='<FILE>',
           help=('Local file that contains disk image to be uploaded during'
                 ' creation. Alternatively, images can be passed to the client'
                 ' via stdin.'))
@utils.arg('--checksum', metavar='<CHECKSUM>',
           help='Hash of image data used Glance can use for verification.')
@utils.arg('--copy-from', metavar='<IMAGE_URL>',
           help=('Similar to \'--location\' in usage, but this indicates that'
                 ' the Glance server should immediately copy the data and'
                 ' store it in its configured image store.'))
@utils.arg('--public', action='store_true', default=False,
           help='Make image accessible to the public.')
@utils.arg('--is-protected', type=utils.string_to_bool,
           help='Prevent image from being deleted.')
@utils.arg('--property', metavar="<key=value>", action='append', default=[],
           help=("Arbitrary property to associate with image. "
                 "May be used multiple times."))
def do_image_create(gc, args):
    # Filter out None values
    fields = dict(filter(lambda x: x[1] is not None, vars(args).items()))

    fields['is_public'] = fields.pop('public')

    if 'is_protected' in fields:
        fields['protected'] = fields.pop('is_protected')

    raw_properties = fields.pop('property')
    fields['properties'] = {}
    for datum in raw_properties:
        key, value = datum.split('=', 1)
        fields['properties'][key] = value

    # Filter out values we can't use
    CREATE_PARAMS = glanceclient.v1.images.CREATE_PARAMS
    fields = dict(filter(lambda x: x[0] in CREATE_PARAMS, fields.items()))

    if 'location' not in fields and 'copy_from' not in fields:
        if args.file:
            fields['data'] = open(args.file, 'r')
        else:
            fields['data'] = sys.stdin

    image = gc.images.create(**fields)
    _image_show(image)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to modify.')
@utils.arg('--name', metavar='<NAME>',
           help='Name of image.')
@utils.arg('--disk-format', metavar='<CONTAINER_FORMAT>',
           help='Disk format of image.')
@utils.arg('--container-format', metavar='<DISK_FORMAT>',
           help='Container format of image.')
@utils.arg('--owner', metavar='<TENANT_ID>',
           help='Tenant who should own image.')
@utils.arg('--size', metavar='<SIZE>',
           help='Size of image data (in bytes).')
@utils.arg('--min-disk', metavar='<DISK_GB>',
           help='Minimum size of disk needed to boot image (in gigabytes).')
@utils.arg('--min-ram', metavar='<DISK_RAM>',
           help='Minimum amount of ram needed to boot image (in megabytes).')
@utils.arg('--location', metavar='<IMAGE_URL>',
           help=('URL where the data for this image already resides.'
                 ' For example, if the image data is stored in the filesystem'
                 ' local to the glance server at \'/usr/share/image.tar.gz\','
                 ' you would specify \'file:///usr/share/image.tar.gz\'.'))
@utils.arg('--file', metavar='<FILE>',
           help=('Local file that contains disk image to be uploaded during'
                 ' update. Alternatively, images can be passed to the client'
                 ' via stdin.'))
@utils.arg('--checksum', metavar='<CHECKSUM>',
           help='Hash of image data used Glance can use for verification.')
@utils.arg('--copy-from', metavar='<IMAGE_URL>',
           help=('Similar to \'--location\' in usage, but this indicates that'
                 ' the Glance server should immediately copy the data and'
                 ' store it in its configured image store.'))
@utils.arg('--is-public', type=utils.string_to_bool,
           help='Make image accessible to the public.')
@utils.arg('--is-protected', type=utils.string_to_bool,
           help='Prevent image from being deleted.')
@utils.arg('--property', metavar="<key=value>", action='append', default=[],
           help=("Arbitrary property to associate with image. "
                 "May be used multiple times."))
@utils.arg('--purge-props', action='store_true', default=False,
           help=("If this flag is present, delete all image properties "
                 "not explicitly set in the update request. Otherwise, "
                 "those properties not referenced are preserved."))
def do_image_update(gc, args):
    # Filter out None values
    fields = dict(filter(lambda x: x[1] is not None, vars(args).items()))

    image_id = fields.pop('id')

    if 'is_protected' in fields:
        fields['protected'] = fields.pop('is_protected')

    raw_properties = fields.pop('property')
    fields['properties'] = {}
    for datum in raw_properties:
        key, value = datum.split('=', 1)
        fields['properties'][key] = value

    # Filter out values we can't use
    UPDATE_PARAMS = glanceclient.v1.images.UPDATE_PARAMS
    fields = dict(filter(lambda x: x[0] in UPDATE_PARAMS, fields.items()))

    if 'location' not in fields and 'copy_from' not in fields:
        if args.file:
            fields['data'] = open(args.file, 'r')
        else:
            fields['data'] = sys.stdin

    image = gc.images.update(image_id, purge_props=args.purge_props, **fields)
    _image_show(image)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to delete.')
def do_image_delete(gc, args):
    """Delete a specific image."""
    gc.images.delete(args.id)

@utils.arg('--image-id', metavar='<IMAGE_ID>',
           help='Filter results by an image ID.')
@utils.arg('--tenant-id', metavar='<TENANT_ID>',
           help='Filter results by a tenant ID.')
def do_member_list(gc, args):
    if args.image_id and args.tenant_id:
        print 'Unable to filter members by both --image-id and --tenant-id.'
        sys.exit(1)
    elif args.image_id:
        kwargs = {'image': args.image_id}
    elif args.tenant_id:
        kwargs = {'member': args.tenant_id}
    else:
        print 'Unable to list all members. Specify --image-id or --tenant-id'
        sys.exit(1)

    members = gc.image_members.list(**kwargs)
    columns = ['Image ID', 'Member ID', 'Can Share']
    utils.print_list(members, columns)

@utils.arg('image_id', metavar='<IMAGE_ID>',
           help='Image to add member to.')
@utils.arg('tenant_id', metavar='<TENANT_ID>',
           help='Tenant to add as member')
@utils.arg('--can-share', action='store_true', default=False,
           help='Allow the specified tenant to share this image.')
def do_member_create(gc, args):
    gc.image_members.create(args.image_id, args.tenant_id, args.can_share)

@utils.arg('image_id', metavar='<IMAGE_ID>',
           help='Image to add member to.')
@utils.arg('tenant_id', metavar='<TENANT_ID>',
           help='Tenant to add as member')
def do_member_delete(gc, args):
    gc.image_members.delete(args.image_id, args.tenant_id)
<commit_msg>Add --is-public to image-create<commit_after># Copyright 2012 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import copy
import sys

from glanceclient.common import utils
import glanceclient.v1.images

@utils.arg('--name', metavar='<NAME>',
           help='Filter images to those that have this name.')
@utils.arg('--status', metavar='<STATUS>',
           help='Filter images to those that have this status.')
@utils.arg('--container-format', metavar='<CONTAINER_FORMAT>',
           help='Filter images to those that have this container format.')
@utils.arg('--disk-format', metavar='<DISK_FORMAT>',
           help='Filter images to those that have this disk format.')
@utils.arg('--size-min', metavar='<SIZE>',
           help='Filter images to those with a size greater than this.')
@utils.arg('--size-max', metavar='<SIZE>',
           help='Filter images to those with a size less than this.')
@utils.arg('--property-filter', metavar='<KEY=VALUE>',
           help="Filter images by a user-defined image property.",
            action='append', dest='properties', default=[])
def do_image_list(gc, args):
    """List images."""
    filter_keys = ['name', 'status', 'container_format', 'disk_format',
               'size_min', 'size_max']
    filter_items = [(key, getattr(args, key)) for key in filter_keys]
    filters = dict([item for item in filter_items if item[1] is not None])

    if args.properties:
        property_filter_items = [p.split('=', 1) for p in args.properties]
        filters['properties'] = dict(property_filter_items)

    images = gc.images.list(filters=filters)
    columns = ['ID', 'Name', 'Disk Format', 'Container Format',
               'Size', 'Status']
    utils.print_list(images, columns)

def _image_show(image):
    # Flatten image properties dict for display
    info = copy.deepcopy(image._info)
    for (k, v) in info.pop('properties').iteritems():
        info['Property \'%s\'' % k] = v

    utils.print_dict(info)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to describe.')
def do_image_show(gc, args):
    """Describe a specific image."""
    image = gc.images.get(args.id)
    _image_show(image)

@utils.arg('--id', metavar='<IMAGE_ID>',
           help='ID of image to reserve.')
@utils.arg('--name', metavar='<NAME>',
           help='Name of image.')
@utils.arg('--disk-format', metavar='<CONTAINER_FORMAT>',
           help='Disk format of image.')
@utils.arg('--container-format', metavar='<DISK_FORMAT>',
           help='Container format of image.')
@utils.arg('--owner', metavar='<TENANT_ID>',
           help='Tenant who should own image.')
@utils.arg('--size', metavar='<SIZE>',
           help=('Size of image data (in bytes). Only used with'
                 ' \'--location\' and \'--copy_from\'.'))
@utils.arg('--min-disk', metavar='<DISK_GB>',
           help='Minimum size of disk needed to boot image (in gigabytes).')
@utils.arg('--min-ram', metavar='<DISK_RAM>',
           help='Minimum amount of ram needed to boot image (in megabytes).')
@utils.arg('--location', metavar='<IMAGE_URL>',
           help=('URL where the data for this image already resides.'
                 ' For example, if the image data is stored in the filesystem'
                 ' local to the glance server at \'/usr/share/image.tar.gz\','
                 ' you would specify \'file:///usr/share/image.tar.gz\'.'))
@utils.arg('--file', metavar='<FILE>',
           help=('Local file that contains disk image to be uploaded during'
                 ' creation. Alternatively, images can be passed to the client'
                 ' via stdin.'))
@utils.arg('--checksum', metavar='<CHECKSUM>',
           help='Hash of image data used Glance can use for verification.')
@utils.arg('--copy-from', metavar='<IMAGE_URL>',
           help=('Similar to \'--location\' in usage, but this indicates that'
                 ' the Glance server should immediately copy the data and'
                 ' store it in its configured image store.'))
#NOTE(bcwaldon): This will be removed once devstack is updated
# to use --is-public
@utils.arg('--public', action='store_true', default=False,
           help=argparse.SUPPRESS)
@utils.arg('--is-public', type=utils.string_to_bool,
           help='Make image accessible to the public.')
@utils.arg('--is-protected', type=utils.string_to_bool,
           help='Prevent image from being deleted.')
@utils.arg('--property', metavar="<key=value>", action='append', default=[],
           help=("Arbitrary property to associate with image. "
                 "May be used multiple times."))
def do_image_create(gc, args):
    # Filter out None values
    fields = dict(filter(lambda x: x[1] is not None, vars(args).items()))

    fields['is_public'] = fields.get('is_public') or fields.pop('public')

    if 'is_protected' in fields:
        fields['protected'] = fields.pop('is_protected')

    raw_properties = fields.pop('property')
    fields['properties'] = {}
    for datum in raw_properties:
        key, value = datum.split('=', 1)
        fields['properties'][key] = value

    # Filter out values we can't use
    CREATE_PARAMS = glanceclient.v1.images.CREATE_PARAMS
    fields = dict(filter(lambda x: x[0] in CREATE_PARAMS, fields.items()))

    if 'location' not in fields and 'copy_from' not in fields:
        if args.file:
            fields['data'] = open(args.file, 'r')
        else:
            fields['data'] = sys.stdin

    image = gc.images.create(**fields)
    _image_show(image)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to modify.')
@utils.arg('--name', metavar='<NAME>',
           help='Name of image.')
@utils.arg('--disk-format', metavar='<CONTAINER_FORMAT>',
           help='Disk format of image.')
@utils.arg('--container-format', metavar='<DISK_FORMAT>',
           help='Container format of image.')
@utils.arg('--owner', metavar='<TENANT_ID>',
           help='Tenant who should own image.')
@utils.arg('--size', metavar='<SIZE>',
           help='Size of image data (in bytes).')
@utils.arg('--min-disk', metavar='<DISK_GB>',
           help='Minimum size of disk needed to boot image (in gigabytes).')
@utils.arg('--min-ram', metavar='<DISK_RAM>',
           help='Minimum amount of ram needed to boot image (in megabytes).')
@utils.arg('--location', metavar='<IMAGE_URL>',
           help=('URL where the data for this image already resides.'
                 ' For example, if the image data is stored in the filesystem'
                 ' local to the glance server at \'/usr/share/image.tar.gz\','
                 ' you would specify \'file:///usr/share/image.tar.gz\'.'))
@utils.arg('--file', metavar='<FILE>',
           help=('Local file that contains disk image to be uploaded during'
                 ' update. Alternatively, images can be passed to the client'
                 ' via stdin.'))
@utils.arg('--checksum', metavar='<CHECKSUM>',
           help='Hash of image data used Glance can use for verification.')
@utils.arg('--copy-from', metavar='<IMAGE_URL>',
           help=('Similar to \'--location\' in usage, but this indicates that'
                 ' the Glance server should immediately copy the data and'
                 ' store it in its configured image store.'))
@utils.arg('--is-public', type=utils.string_to_bool,
           help='Make image accessible to the public.')
@utils.arg('--is-protected', type=utils.string_to_bool,
           help='Prevent image from being deleted.')
@utils.arg('--property', metavar="<key=value>", action='append', default=[],
           help=("Arbitrary property to associate with image. "
                 "May be used multiple times."))
@utils.arg('--purge-props', action='store_true', default=False,
           help=("If this flag is present, delete all image properties "
                 "not explicitly set in the update request. Otherwise, "
                 "those properties not referenced are preserved."))
def do_image_update(gc, args):
    # Filter out None values
    fields = dict(filter(lambda x: x[1] is not None, vars(args).items()))

    image_id = fields.pop('id')

    if 'is_protected' in fields:
        fields['protected'] = fields.pop('is_protected')

    raw_properties = fields.pop('property')
    fields['properties'] = {}
    for datum in raw_properties:
        key, value = datum.split('=', 1)
        fields['properties'][key] = value

    # Filter out values we can't use
    UPDATE_PARAMS = glanceclient.v1.images.UPDATE_PARAMS
    fields = dict(filter(lambda x: x[0] in UPDATE_PARAMS, fields.items()))

    if 'location' not in fields and 'copy_from' not in fields:
        if args.file:
            fields['data'] = open(args.file, 'r')
        else:
            fields['data'] = sys.stdin

    image = gc.images.update(image_id, purge_props=args.purge_props, **fields)
    _image_show(image)

@utils.arg('id', metavar='<IMAGE_ID>', help='ID of image to delete.')
def do_image_delete(gc, args):
    """Delete a specific image."""
    gc.images.delete(args.id)

@utils.arg('--image-id', metavar='<IMAGE_ID>',
           help='Filter results by an image ID.')
@utils.arg('--tenant-id', metavar='<TENANT_ID>',
           help='Filter results by a tenant ID.')
def do_member_list(gc, args):
    if args.image_id and args.tenant_id:
        print 'Unable to filter members by both --image-id and --tenant-id.'
        sys.exit(1)
    elif args.image_id:
        kwargs = {'image': args.image_id}
    elif args.tenant_id:
        kwargs = {'member': args.tenant_id}
    else:
        print 'Unable to list all members. Specify --image-id or --tenant-id'
        sys.exit(1)

    members = gc.image_members.list(**kwargs)
    columns = ['Image ID', 'Member ID', 'Can Share']
    utils.print_list(members, columns)

@utils.arg('image_id', metavar='<IMAGE_ID>',
           help='Image to add member to.')
@utils.arg('tenant_id', metavar='<TENANT_ID>',
           help='Tenant to add as member')
@utils.arg('--can-share', action='store_true', default=False,
           help='Allow the specified tenant to share this image.')
def do_member_create(gc, args):
    gc.image_members.create(args.image_id, args.tenant_id, args.can_share)

@utils.arg('image_id', metavar='<IMAGE_ID>',
           help='Image to add member to.')
@utils.arg('tenant_id', metavar='<TENANT_ID>',
           help='Tenant to add as member')
def do_member_delete(gc, args):
    gc.image_members.delete(args.image_id, args.tenant_id)
<|endoftext|>


<commit_before>import sys

import re
from subprocess import check_call

import hailctl
from .cluster_config import ClusterConfig

DEFAULT_PROPERTIES = {
    "spark:spark.driver.maxResultSize": "0",
    "spark:spark.task.maxFailures": "20",
    "spark:spark.kryoserializer.buffer.max": "1g",
    "spark:spark.driver.extraJavaOptions": "-Xss4M",
    "spark:spark.executor.extraJavaOptions": "-Xss4M",
    "hdfs:dfs.replication": "1",
    'dataproc:dataproc.logging.stackdriver.enable': 'false',
    'dataproc:dataproc.monitoring.stackdriver.enable': 'false'
}

# master machine type to memory map, used for setting spark.driver.memory property
MACHINE_MEM = {
    'n1-standard-1': 3.75,
    'n1-standard-2': 7.5,
    'n1-standard-4': 15,
    'n1-standard-8': 30,
    'n1-standard-16': 60,
    'n1-standard-32': 120,
    'n1-standard-64': 240,
    'n1-highmem-2': 13,
    'n1-highmem-4': 26,
    'n1-highmem-8': 52,
    'n1-highmem-16': 104,
    'n1-highmem-32': 208,
    'n1-highmem-64': 416,
    'n1-highcpu-2': 1.8,
    'n1-highcpu-4': 3.6,
    'n1-highcpu-8': 7.2,
    'n1-highcpu-16': 14.4,
    'n1-highcpu-32': 28.8,
    'n1-highcpu-64': 57.6
}

SPARK_VERSION = '2.4.0'
IMAGE_VERSION = '1.4-debian9'

def init_parser(parser):
    parser.add_argument('name', type=str, help='Cluster name.')

    # arguments with default parameters
    parser.add_argument('--master-machine-type', '--master', '-m', default='n1-highmem-8', type=str,
                        help='Master machine type (default: %(default)s).')
    parser.add_argument('--master-memory-fraction', default=0.8, type=float,
                        help='Fraction of master memory allocated to the JVM. '
                             'Use a smaller value to reserve more memory '
                             'for Python. (default: %(default)s)')
    parser.add_argument('--master-boot-disk-size', default=100, type=int,
                        help='Disk size of master machine, in GB (default: %(default)s).')
    parser.add_argument('--num-master-local-ssds', default=0, type=int,
                        help='Number of local SSDs to attach to the master machine (default: %(default)s).')
    parser.add_argument('--num-preemptible-workers', '--n-pre-workers', '-p', default=0, type=int,
                        help='Number of preemptible worker machines (default: %(default)s).')
    parser.add_argument('--num-worker-local-ssds', default=0, type=int,
                        help='Number of local SSDs to attach to each worker machine (default: %(default)s).')
    parser.add_argument('--num-workers', '--n-workers', '-w', default=2, type=int,
                        help='Number of worker machines (default: %(default)s).')
    parser.add_argument('--preemptible-worker-boot-disk-size', default=40, type=int,
                        help='Disk size of preemptible machines, in GB (default: %(default)s).')
    parser.add_argument('--worker-boot-disk-size', default=40, type=int,
                        help='Disk size of worker machines, in GB (default: %(default)s).')
    parser.add_argument('--worker-machine-type', '--worker',
                        help='Worker machine type (default: n1-standard-8, or n1-highmem-8 with --vep).')
    parser.add_argument('--zone', default='us-central1-b',
                        help='Compute zone for the cluster (default: %(default)s).')
    parser.add_argument('--properties',
                        help='Additional configuration properties for the cluster')
    parser.add_argument('--metadata',
                        help='Comma-separated list of metadata to add: KEY1=VALUE1,KEY2=VALUE2...')
    parser.add_argument('--packages', '--pkgs',
                        help='Comma-separated list of Python packages to be installed on the master node.')
    parser.add_argument('--project', help='Google Cloud project to start cluster (defaults to currently set project).')
    parser.add_argument('--configuration',
                        help='Google Cloud configuration to start cluster (defaults to currently set configuration).')
    parser.add_argument('--max-idle', type=str, help='If specified, maximum idle time before shutdown (e.g. 60m).')
    parser.add_argument('--max-age', type=str, help='If specified, maximum age before shutdown (e.g. 60m).')
    parser.add_argument('--bucket', type=str,
                        help='The Google Cloud Storage bucket to use for cluster staging (just the bucket name, no gs:// prefix).')

    # specify custom Hail jar and zip
    parser.add_argument('--jar', help='Non-default Hail JAR to install. Warning: experimental.')
    parser.add_argument('--wheel', help='Non-default Hail Python wheel install. Warning: experimental.')

    # initialization action flags
    parser.add_argument('--init', default='', help='Comma-separated list of init scripts to run.')
    parser.add_argument('--init_timeout', default='20m',
                        help='Flag to specify a timeout period for the initialization action')
    parser.add_argument('--vep',
                        help='Install VEP for the specified reference genome.',
                        required=False,
                        choices=['GRCh37', 'GRCh38'])
    parser.add_argument('--dry-run', action='store_true', help="Print gcloud dataproc command, but don't run it.")

def main(args, pass_through_args):
    conf = ClusterConfig()
    conf.extend_flag('image-version', IMAGE_VERSION)

    deploy_metadata = hailctl._deploy_metadata['dataproc']

    conf.extend_flag('properties', DEFAULT_PROPERTIES)
    if args.properties:
        conf.parse_and_extend('properties', args.properties)

    # default to highmem machines if using VEP
    if not args.worker_machine_type:
        args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8'

    # default initialization script to start up cluster with
    conf.extend_flag('initialization-actions',
                     [deploy_metadata['init_notebook.py']])

    # add VEP init script
    if args.vep:
        conf.extend_flag('initialization-actions', [deploy_metadata[f'vep-{args.vep}.sh']])
    # add custom init scripts
    if args.init:
        conf.extend_flag('initialization-actions', args.init.split(','))

    if args.metadata:
        conf.parse_and_extend('metadata', args.metadata)

    if args.jar and args.wheel:
        jar = args.jar
        wheel = args.wheel
    elif args.jar or args.wheel:
        sys.stderr.write('ERROR: pass both --jar and --wheel or neither')
        sys.exit(1)
    else:
        jar = deploy_metadata['jar']
        wheel = deploy_metadata['wheel']
    conf.extend_flag('metadata', {'JAR': jar, 'ZIP': wheel})

    # if Python packages requested, add metadata variable
    if args.packages:
        metadata_pkgs = conf.flags['metadata'].get('PKGS')
        packages = []
        split_regex = r'[|,]'
        if metadata_pkgs:
            packages.extend(re.split(split_regex, metadata_pkgs))

        packages.extend(re.split(split_regex, args.packages))
        conf.extend_flag('metadata', {'PKGS': '|'.join(packages)})

    conf.extend_flag('properties',
                     {"spark:spark.driver.memory": "{driver_memory}g".format(
                         driver_memory=str(int(MACHINE_MEM[args.master_machine_type] * args.master_memory_fraction)))})
    conf.flags['master-machine-type'] = args.master_machine_type
    conf.flags['master-boot-disk-size'] = '{}GB'.format(args.master_boot_disk_size)
    conf.flags['num-master-local-ssds'] = args.num_master_local_ssds
    conf.flags['num-preemptible-workers'] = args.num_preemptible_workers
    conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds
    conf.flags['num-workers'] = args.num_workers
    conf.flags['preemptible-worker-boot-disk-size'] = '{}GB'.format(args.preemptible_worker_boot_disk_size)
    conf.flags['worker-boot-disk-size'] = args.worker_boot_disk_size
    conf.flags['worker-machine-type'] = args.worker_machine_type
    conf.flags['zone'] = args.zone
    conf.flags['initialization-action-timeout'] = args.init_timeout
    if args.configuration:
        conf.flags['configuration'] = args.configuration
    if args.project:
        conf.flags['project'] = args.project
    if args.bucket:
        conf.flags['bucket'] = args.bucket

    # command to start cluster
    cmd = conf.get_command(args.name)

    if args.max_idle or args.max_age:
        cmd.insert(1, 'beta')
    if args.max_idle:
        cmd.append('--max-idle={}'.format(args.max_idle))
    if args.max_age:
        cmd.append('--max-age={}'.format(args.max_age))

    # print underlying gcloud command
    print(' '.join(cmd[:5]) + ' \\\n    ' + ' \\\n    '.join(cmd[5:]))

    # spin up cluster
    if not args.dry_run:
        print("Starting cluster '{}'...".format(args.name))
        check_call(cmd)
<commit_msg>[hailctl] Add creator label to `hailctl start` (#6276)<commit_after>import sys

import re
from subprocess import check_call

import hailctl
from .cluster_config import ClusterConfig
from .utils import safe_call

DEFAULT_PROPERTIES = {
    "spark:spark.driver.maxResultSize": "0",
    "spark:spark.task.maxFailures": "20",
    "spark:spark.kryoserializer.buffer.max": "1g",
    "spark:spark.driver.extraJavaOptions": "-Xss4M",
    "spark:spark.executor.extraJavaOptions": "-Xss4M",
    "hdfs:dfs.replication": "1",
    'dataproc:dataproc.logging.stackdriver.enable': 'false',
    'dataproc:dataproc.monitoring.stackdriver.enable': 'false'
}

# master machine type to memory map, used for setting spark.driver.memory property
MACHINE_MEM = {
    'n1-standard-1': 3.75,
    'n1-standard-2': 7.5,
    'n1-standard-4': 15,
    'n1-standard-8': 30,
    'n1-standard-16': 60,
    'n1-standard-32': 120,
    'n1-standard-64': 240,
    'n1-highmem-2': 13,
    'n1-highmem-4': 26,
    'n1-highmem-8': 52,
    'n1-highmem-16': 104,
    'n1-highmem-32': 208,
    'n1-highmem-64': 416,
    'n1-highcpu-2': 1.8,
    'n1-highcpu-4': 3.6,
    'n1-highcpu-8': 7.2,
    'n1-highcpu-16': 14.4,
    'n1-highcpu-32': 28.8,
    'n1-highcpu-64': 57.6
}

SPARK_VERSION = '2.4.0'
IMAGE_VERSION = '1.4-debian9'

def init_parser(parser):
    parser.add_argument('name', type=str, help='Cluster name.')

    # arguments with default parameters
    parser.add_argument('--master-machine-type', '--master', '-m', default='n1-highmem-8', type=str,
                        help='Master machine type (default: %(default)s).')
    parser.add_argument('--master-memory-fraction', default=0.8, type=float,
                        help='Fraction of master memory allocated to the JVM. '
                             'Use a smaller value to reserve more memory '
                             'for Python. (default: %(default)s)')
    parser.add_argument('--master-boot-disk-size', default=100, type=int,
                        help='Disk size of master machine, in GB (default: %(default)s).')
    parser.add_argument('--num-master-local-ssds', default=0, type=int,
                        help='Number of local SSDs to attach to the master machine (default: %(default)s).')
    parser.add_argument('--num-preemptible-workers', '--n-pre-workers', '-p', default=0, type=int,
                        help='Number of preemptible worker machines (default: %(default)s).')
    parser.add_argument('--num-worker-local-ssds', default=0, type=int,
                        help='Number of local SSDs to attach to each worker machine (default: %(default)s).')
    parser.add_argument('--num-workers', '--n-workers', '-w', default=2, type=int,
                        help='Number of worker machines (default: %(default)s).')
    parser.add_argument('--preemptible-worker-boot-disk-size', default=40, type=int,
                        help='Disk size of preemptible machines, in GB (default: %(default)s).')
    parser.add_argument('--worker-boot-disk-size', default=40, type=int,
                        help='Disk size of worker machines, in GB (default: %(default)s).')
    parser.add_argument('--worker-machine-type', '--worker',
                        help='Worker machine type (default: n1-standard-8, or n1-highmem-8 with --vep).')
    parser.add_argument('--zone', default='us-central1-b',
                        help='Compute zone for the cluster (default: %(default)s).')
    parser.add_argument('--properties',
                        help='Additional configuration properties for the cluster')
    parser.add_argument('--metadata',
                        help='Comma-separated list of metadata to add: KEY1=VALUE1,KEY2=VALUE2...')
    parser.add_argument('--packages', '--pkgs',
                        help='Comma-separated list of Python packages to be installed on the master node.')
    parser.add_argument('--project', help='Google Cloud project to start cluster (defaults to currently set project).')
    parser.add_argument('--configuration',
                        help='Google Cloud configuration to start cluster (defaults to currently set configuration).')
    parser.add_argument('--max-idle', type=str, help='If specified, maximum idle time before shutdown (e.g. 60m).')
    parser.add_argument('--max-age', type=str, help='If specified, maximum age before shutdown (e.g. 60m).')
    parser.add_argument('--bucket', type=str,
                        help='The Google Cloud Storage bucket to use for cluster staging (just the bucket name, no gs:// prefix).')

    # specify custom Hail jar and zip
    parser.add_argument('--jar', help='Non-default Hail JAR to install. Warning: experimental.')
    parser.add_argument('--wheel', help='Non-default Hail Python wheel install. Warning: experimental.')

    # initialization action flags
    parser.add_argument('--init', default='', help='Comma-separated list of init scripts to run.')
    parser.add_argument('--init_timeout', default='20m',
                        help='Flag to specify a timeout period for the initialization action')
    parser.add_argument('--vep',
                        help='Install VEP for the specified reference genome.',
                        required=False,
                        choices=['GRCh37', 'GRCh38'])
    parser.add_argument('--dry-run', action='store_true', help="Print gcloud dataproc command, but don't run it.")

def main(args, pass_through_args):
    conf = ClusterConfig()
    conf.extend_flag('image-version', IMAGE_VERSION)

    deploy_metadata = hailctl._deploy_metadata['dataproc']

    conf.extend_flag('properties', DEFAULT_PROPERTIES)
    if args.properties:
        conf.parse_and_extend('properties', args.properties)

    # default to highmem machines if using VEP
    if not args.worker_machine_type:
        args.worker_machine_type = 'n1-highmem-8' if args.vep else 'n1-standard-8'

    # default initialization script to start up cluster with
    conf.extend_flag('initialization-actions',
                     [deploy_metadata['init_notebook.py']])

    # add VEP init script
    if args.vep:
        conf.extend_flag('initialization-actions', [deploy_metadata[f'vep-{args.vep}.sh']])
    # add custom init scripts
    if args.init:
        conf.extend_flag('initialization-actions', args.init.split(','))

    if args.metadata:
        conf.parse_and_extend('metadata', args.metadata)

    if args.jar and args.wheel:
        jar = args.jar
        wheel = args.wheel
    elif args.jar or args.wheel:
        sys.stderr.write('ERROR: pass both --jar and --wheel or neither')
        sys.exit(1)
    else:
        jar = deploy_metadata['jar']
        wheel = deploy_metadata['wheel']
    conf.extend_flag('metadata', {'JAR': jar, 'ZIP': wheel})

    # if Python packages requested, add metadata variable
    if args.packages:
        metadata_pkgs = conf.flags['metadata'].get('PKGS')
        packages = []
        split_regex = r'[|,]'
        if metadata_pkgs:
            packages.extend(re.split(split_regex, metadata_pkgs))

        packages.extend(re.split(split_regex, args.packages))
        conf.extend_flag('metadata', {'PKGS': '|'.join(packages)})

    conf.extend_flag('properties',
                     {"spark:spark.driver.memory": "{driver_memory}g".format(
                         driver_memory=str(int(MACHINE_MEM[args.master_machine_type] * args.master_memory_fraction)))})
    conf.flags['master-machine-type'] = args.master_machine_type
    conf.flags['master-boot-disk-size'] = '{}GB'.format(args.master_boot_disk_size)
    conf.flags['num-master-local-ssds'] = args.num_master_local_ssds
    conf.flags['num-preemptible-workers'] = args.num_preemptible_workers
    conf.flags['num-worker-local-ssds'] = args.num_worker_local_ssds
    conf.flags['num-workers'] = args.num_workers
    conf.flags['preemptible-worker-boot-disk-size'] = '{}GB'.format(args.preemptible_worker_boot_disk_size)
    conf.flags['worker-boot-disk-size'] = args.worker_boot_disk_size
    conf.flags['worker-machine-type'] = args.worker_machine_type
    conf.flags['zone'] = args.zone
    conf.flags['initialization-action-timeout'] = args.init_timeout
    if args.configuration:
        conf.flags['configuration'] = args.configuration
    if args.project:
        conf.flags['project'] = args.project
    if args.bucket:
        conf.flags['bucket'] = args.bucket

    label = safe_call('gcloud', 'config', 'get-value', 'account')
    conf.flags['labels'] = 'creator=' + label.replace('.', '-dot-').replace('@', '-at-')  # no special chars

    # command to start cluster
    cmd = conf.get_command(args.name)

    if args.max_idle or args.max_age:
        cmd.insert(1, 'beta')
    if args.max_idle:
        cmd.append('--max-idle={}'.format(args.max_idle))
    if args.max_age:
        cmd.append('--max-age={}'.format(args.max_age))

    # print underlying gcloud command
    print(' '.join(cmd[:5]) + ' \\\n    ' + ' \\\n    '.join(cmd[5:]))

    # spin up cluster
    if not args.dry_run:
        print("Starting cluster '{}'...".format(args.name))
        check_call(cmd)
<|endoftext|>


<commit_before>#!/usr/bin/env python

sx=0
sy=0
rx=0
ry=0

total = 2

houses = set()
houses.add( (sx,sy) )
houses.add( (rx,ry) )

with open("../input/03.txt") as fileobj:
	for word in fileobj:
		for ch in word:
			total+=1
			if total % 2 == 0:
				if ch == '^':
					sx+=1
				if ch == 'v':
					sx-=1
				if ch == '<':
					sy-=1
				if ch == '>':
					sy+=1
				houses.add( (sx,sy) )
			else:
				if ch == '^':
					rx+=1
				if ch == 'v':
					rx-=1
				if ch == '<':
					ry-=1
				if ch == '>':
					ry+=1
				houses.add( (rx,ry) )

	print "total presents: %s" % total
	print "total houses visited: %s" % len(houses)

<commit_msg>Minor update to day 3 part 2.<commit_after>#!/usr/bin/env python

sx=0
sy=0
rx=0
ry=0

total = 2

houses = set()
houses.add( (0,0) ) # starting point

with open("../input/03.txt") as fileobj:
	for word in fileobj:
		for ch in word:
			total+=1
			if total % 2 == 0:	# santa
				if ch == '^':
					sx+=1
				if ch == 'v':
					sx-=1
				if ch == '<':
					sy-=1
				if ch == '>':
					sy+=1
				houses.add( (sx,sy) )
			else:				# robo-santa
				if ch == '^':
					rx+=1
				if ch == 'v':
					rx-=1
				if ch == '<':
					ry-=1
				if ch == '>':
					ry+=1
				houses.add( (rx,ry) )

	print "total presents: %s" % total
	print "total houses visited: %s" % len(houses)

<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

import inspect

from wxpy.compatible import PY2

def _ipython(local, banner):
    from IPython.terminal.embed import InteractiveShellEmbed
    from IPython.terminal.ipapp import load_default_config

    InteractiveShellEmbed.clear_instance()
    shell = InteractiveShellEmbed.instance(
        banner1=banner,
        user_ns=local,
        config=load_default_config()
    )
    shell()

def _bpython(local, banner):
    # noinspection PyUnresolvedReferences,PyPackageRequirements
    import bpython

    bpython.embed(locals_=local, banner=banner)

def _python(local, banner):
    import code

    try:
        # noinspection PyUnresolvedReferences
        import readline
    except ImportError:
        pass
    else:
        import rlcompleter
        readline.parse_and_bind('tab:complete')
    if PY2:
        banner = banner.encode('utf-8')

    code.interact(local=local, banner=banner)

def embed(local=None, banner='', shell=None):
    """
    | 进入交互式的 Python 命令行界面，并堵塞当前线程
    | 支持使用 ipython, bpython 以及原生 python

    :param str shell:
        | 指定命令行类型，可设为 'ipython'，'bpython'，'python'，或它们的首字母；
        | 若为 `None`，则按上述优先级进入首个可用的 Python 命令行。
    :param dict local: 设定本地变量环境，若为 `None`，则获取进入之前的变量环境。
    :param str banner: 设定欢迎内容，将在进入命令行后展示。
    """

    import inspect

    if local is None:
        local = inspect.currentframe().f_back.f_locals

    if isinstance(shell, str):
        shell = shell.strip().lower()
        if shell.startswith('b'):
            shell = _bpython
        elif shell.startswith('i'):
            shell = _ipython
        elif shell.startswith('p') or not shell:
            shell = _python

    for _shell in shell, _ipython, _bpython, _python:
        try:
            _shell(local=local, banner=banner)
        except (TypeError, ImportError):
            continue
        except KeyboardInterrupt:
            break
        else:
            break

def get_arg_parser():
    import argparse

    ap = argparse.ArgumentParser(
        description='Run a wxpy-ready python console.')

    ap.add_argument(
        'bot', type=str, nargs='*',
        help='One or more variable name(s) for bot(s) to init (default: None).')

    ap.add_argument(
        '-c', '--cache', action='store_true',
        help='Cache session(s) for a short time, or load session(s) from cache '
             '(default: disabled).')

    ap.add_argument(
        '-q', '--console_qr', type=int, default=False, metavar='width',
        help='The width for console_qr (default: None).')

    ap.add_argument(
        '-l', '--logging_level', type=str, default='INFO', metavar='level',
        help='Logging level (default: INFO).')

    ap.add_argument(
        '-s', '--shell', type=str, default=None, metavar='shell',
        help='Specify which shell to use: ipython, bpython, or python '
             '(default: the first available).')

    ap.add_argument(
        '-v', '--version', action='store_true',
        help='Show version and exit.')

    return ap

def shell_entry():
    import re

    import logging
    import wxpy

    arg_parser = get_arg_parser()
    args = arg_parser.parse_args()

    if not args.bot:
        arg_parser.print_help()
        return

    if args.version:
        print(wxpy.version_details)
        return

    def get_logging_level():
        logging_level = args.logging_level.upper()
        for level in 'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET':
            if level.startswith(logging_level):
                return getattr(logging, level)
        else:
            return logging.INFO

    logging.basicConfig(level=get_logging_level())

    try:
        bots = dict()
        for name in args.bot:
            if not re.match(r'\w+$', name):
                continue
            cache_path = 'wxpy_{}.pkl'.format(name) if args.cache else None
            bots[name] = wxpy.Bot(cache_path=cache_path, console_qr=args.console_qr)
    except KeyboardInterrupt:
        return

    banner = 'from wxpy import *\n'

    for k, v in bots.items():
        banner += '{}: {}\n'.format(k, v)

    module_members = dict(inspect.getmembers(wxpy))

    embed(
        local=dict(module_members, **bots),
        banner=banner,
        shell=args.shell
    )
<commit_msg>fix `wxpy -v/—version` command<commit_after># coding: utf-8
from __future__ import unicode_literals

import inspect

from wxpy.compatible import PY2

def _ipython(local, banner):
    from IPython.terminal.embed import InteractiveShellEmbed
    from IPython.terminal.ipapp import load_default_config

    InteractiveShellEmbed.clear_instance()
    shell = InteractiveShellEmbed.instance(
        banner1=banner,
        user_ns=local,
        config=load_default_config()
    )
    shell()

def _bpython(local, banner):
    # noinspection PyUnresolvedReferences,PyPackageRequirements
    import bpython

    bpython.embed(locals_=local, banner=banner)

def _python(local, banner):
    import code

    try:
        # noinspection PyUnresolvedReferences
        import readline
    except ImportError:
        pass
    else:
        import rlcompleter
        readline.parse_and_bind('tab:complete')
    if PY2:
        banner = banner.encode('utf-8')

    code.interact(local=local, banner=banner)

def embed(local=None, banner='', shell=None):
    """
    | 进入交互式的 Python 命令行界面，并堵塞当前线程
    | 支持使用 ipython, bpython 以及原生 python

    :param str shell:
        | 指定命令行类型，可设为 'ipython'，'bpython'，'python'，或它们的首字母；
        | 若为 `None`，则按上述优先级进入首个可用的 Python 命令行。
    :param dict local: 设定本地变量环境，若为 `None`，则获取进入之前的变量环境。
    :param str banner: 设定欢迎内容，将在进入命令行后展示。
    """

    import inspect

    if local is None:
        local = inspect.currentframe().f_back.f_locals

    if isinstance(shell, str):
        shell = shell.strip().lower()
        if shell.startswith('b'):
            shell = _bpython
        elif shell.startswith('i'):
            shell = _ipython
        elif shell.startswith('p') or not shell:
            shell = _python

    for _shell in shell, _ipython, _bpython, _python:
        try:
            _shell(local=local, banner=banner)
        except (TypeError, ImportError):
            continue
        except KeyboardInterrupt:
            break
        else:
            break

def get_arg_parser():
    import argparse

    ap = argparse.ArgumentParser(
        description='Run a wxpy-ready python console.')

    ap.add_argument(
        'bot', type=str, nargs='*',
        help='One or more variable name(s) for bot(s) to init (default: None).')

    ap.add_argument(
        '-c', '--cache', action='store_true',
        help='Cache session(s) for a short time, or load session(s) from cache '
             '(default: disabled).')

    ap.add_argument(
        '-q', '--console_qr', type=int, default=False, metavar='width',
        help='The width for console_qr (default: None).')

    ap.add_argument(
        '-l', '--logging_level', type=str, default='INFO', metavar='level',
        help='Logging level (default: INFO).')

    ap.add_argument(
        '-s', '--shell', type=str, default=None, metavar='shell',
        help='Specify which shell to use: ipython, bpython, or python '
             '(default: the first available).')

    ap.add_argument(
        '-v', '--version', action='store_true',
        help='Show version and exit.')

    return ap

def shell_entry():
    import re

    import logging
    import wxpy

    arg_parser = get_arg_parser()
    args = arg_parser.parse_args()

    if args.bot:
        def get_logging_level():
            logging_level = args.logging_level.upper()
            for level in 'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET':
                if level.startswith(logging_level):
                    return getattr(logging, level)
            else:
                return logging.INFO

        logging.basicConfig(level=get_logging_level())

        try:
            bots = dict()
            for name in args.bot:
                if not re.match(r'\w+$', name):
                    continue
                cache_path = 'wxpy_{}.pkl'.format(name) if args.cache else None
                bots[name] = wxpy.Bot(cache_path=cache_path, console_qr=args.console_qr)
        except KeyboardInterrupt:
            return

        banner = 'from wxpy import *\n'

        for k, v in bots.items():
            banner += '{}: {}\n'.format(k, v)

        module_members = dict(inspect.getmembers(wxpy))

        embed(
            local=dict(module_members, **bots),
            banner=banner,
            shell=args.shell
        )
    elif args.version:
        print(wxpy.version_details)
    else:
        arg_parser.print_help()
<|endoftext|>