<commit_before>#!/usr/bin/env python3

############################################################################
#
# File: GeocodeData.py
# Last Edit: 2015-02-21
# Author: Matthew Leeds <mwl458@gmail.com>
# Purpose: This script reads in traffic accident data provided by the
# the City of Birmingham in CSV format, cleans up the data a bit, and
# makes geocoding requests to Google's API.
# Usage: Just make sure the file names are accurate and replace the
# GOOGLE_API_KEY variable with your own from
# https://code.google.com/apis/console (and make sure to allow your current
# public IP address from that console). Then run 'python3 GeocodeData.py'.
#
############################################################################

import csv
import json
from urllib.request import urlopen
from urllib.parse import urlencode
from time import sleep

INFILE = 'Bham_Traffic_Accidents_2014.csv'
OUTFILE = 'Bham_Geocodings_2014.json'
GOOGLE_BASE_URL = 'https://maps.googleapis.com/maps/api/geocode/json?'
GOOGLE_API_KEY = 'YOUR_API_KEY' # Google API key
MAX_API_REQUESTS = 2500 # per day
# an estimated minimum bounding rectangle around Alabama formatted for Google
ALABAMA_BOUNDS = '35.046674,-88.751659|30.077183,-84.687088'
CITY_STATE = 'Birmingham, AL' # appended to the address for geocoding

def main():
    global overAPILimit
    overAPILimit = False
    allGeocodings = {}
    # Load data from a previous run if it exists
    try:
        with open(OUTFILE) as f:
            allGeocodings = json.load(f)
    except FileNotFoundError as e:
        pass
    # first grab all the unique locations in the data
    uniqueLocations = []
    print('>> Reading data from ' + INFILE)
    with open(INFILE) as f:
        c = csv.reader(f)
        for i, record in enumerate(c):
            if i == 0:
                headers = record
            else:
                location = record[headers.index('Location')]
                # drop missing values and non-intersection-related accidents
                if not ('NO DESCRIPTION AVAILABLE' in location or location.startswith('Between ')):
                    uniqueLocations.append(location)
    uniqueLocations = list(set(uniqueLocations)) # remove duplicates
    print('>> Identified ' + str(len(uniqueLocations)) + ' unique locations.')
    # Now process the data
    print('>> Making geocoding requests to Google...')
    numSuccess = 0
    numFailure = 0
    numAPIRequests = 0
    for location in uniqueLocations:
        # don't bother if we have the data from a previous run
        if location in allGeocodings:
            continue
        # check if we're over the request limit
        numAPIRequests += 1
        if numAPIRequests >= MAX_API_REQUESTS or overAPILimit:
            print('>> Error: Exceeded API Usage limits. Try again in 24 hours.')
            break
        result = process(location)
        if len(result) > 0:
            allGeocodings[location] = result
            numSuccess += 1
        else:
            numFailure += 1
    print('>> Complete. Successes = ' + str(numSuccess) + ', Failures = ' + str(numFailure))
    # Now write the data out to a file
    print('>> Writing geocodings to ' + OUTFILE)
    with open(OUTFILE, 'w') as f:
        json.dump(allGeocodings, f, sort_keys=True, indent=4, separators=(',', ': '))

# This takes an intersection in the format 'ROAD/HWY at ROAD/HWY',
# cleans it up and makes the request.
# on success: returns '(lat, long)'
# on failure: returns ''
def process(location):
    location += ' ' + CITY_STATE
    try:
        response = geocodeGoogle(location)
    except Exception as e:
        print('Caught Error:\n' + str(e))
        return ''
    if response[0] != '(':
        print('>> Error ' + response)
        if response == 'REQUEST_DENIED' or response == 'OVER_QUERY_LIMIT':
            overAPILimit = True
        return ''
    return response

# uses the Google Maps API to geocode an address
# on success: returns '(lat, lng)'
# on failure: returns the status code (a string)
def geocodeGoogle(address):
    # the bounds parameter biases results to locations within Alabama
    params = {'address': address,
              'bounds': ALABAMA_BOUNDS,
              'key': GOOGLE_API_KEY}
    url = GOOGLE_BASE_URL + urlencode(params, safe='/,|')
    rawreply = urlopen(url).read()
    reply = json.loads(rawreply.decode('utf-8'))
    sleep(0.1) # stay under usage limit
    if reply['status'] == 'OK':
        # assume the first result is correct, but only include intersections
        if 'intersection' in reply['results'][0]['types']:
            # round to 6 decimal places (~0.1m precision)
            return '(' + str(round(reply['results'][0]['geometry']['location']['lat'], 6)) + ', ' + \
                         str(round(reply['results'][0]['geometry']['location']['lng'], 6)) + ')'
        else:
            return 'NOT_AN_INTERSECTION'
    else:
        return reply['status']

if __name__=='__main__':
    main()
<commit_msg>Print error locations.<commit_after>#!/usr/bin/env python3

############################################################################
#
# File: GeocodeData.py
# Last Edit: 2015-02-21
# Author: Matthew Leeds <mwl458@gmail.com>
# Purpose: This script reads in traffic accident data provided by the
# the City of Birmingham in CSV format, cleans up the data a bit, and
# makes geocoding requests to Google's API.
# Usage: Just make sure the file names are accurate and replace the
# GOOGLE_API_KEY variable with your own from
# https://code.google.com/apis/console (and make sure to allow your current
# public IP address from that console). Then run 'python3 GeocodeData.py'.
#
############################################################################

import csv
import json
from urllib.request import urlopen
from urllib.parse import urlencode
from time import sleep

INFILE = 'Bham_Traffic_Accidents_2014.csv'
OUTFILE = 'Bham_Geocodings_2014.json'
GOOGLE_BASE_URL = 'https://maps.googleapis.com/maps/api/geocode/json?'
GOOGLE_API_KEY = 'YOUR_API_KEY' # Google API key
MAX_API_REQUESTS = 2500 # per day
# an estimated minimum bounding rectangle around Alabama formatted for Google
ALABAMA_BOUNDS = '35.046674,-88.751659|30.077183,-84.687088'
CITY_STATE = 'Birmingham, AL' # appended to the address for geocoding

def main():
    global overAPILimit
    overAPILimit = False
    allGeocodings = {}
    # Load data from a previous run if it exists
    try:
        with open(OUTFILE) as f:
            allGeocodings = json.load(f)
    except FileNotFoundError as e:
        pass
    # first grab all the unique locations in the data
    uniqueLocations = []
    print('>> Reading data from ' + INFILE)
    with open(INFILE) as f:
        c = csv.reader(f)
        for i, record in enumerate(c):
            if i == 0:
                headers = record
            else:
                location = record[headers.index('Location')]
                # drop missing values and non-intersection-related accidents
                if not ('NO DESCRIPTION AVAILABLE' in location or location.startswith('Between ')):
                    uniqueLocations.append(location)
    uniqueLocations = list(set(uniqueLocations)) # remove duplicates
    print('>> Identified ' + str(len(uniqueLocations)) + ' unique locations.')
    # Now process the data
    print('>> Making geocoding requests to Google...')
    numSuccess = 0
    numFailure = 0
    numAPIRequests = 0
    for location in uniqueLocations:
        # don't bother if we have the data from a previous run
        if location in allGeocodings:
            continue
        # check if we're over the request limit
        numAPIRequests += 1
        if numAPIRequests >= MAX_API_REQUESTS or overAPILimit:
            print('>> Error: Exceeded API Usage limits. Try again in 24 hours.')
            break
        result = process(location)
        if len(result) > 0:
            allGeocodings[location] = result
            numSuccess += 1
        else:
            numFailure += 1
    print('>> Complete. Successes = ' + str(numSuccess) + ', Failures = ' + str(numFailure))
    # Now write the data out to a file
    print('>> Writing geocodings to ' + OUTFILE)
    with open(OUTFILE, 'w') as f:
        json.dump(allGeocodings, f, sort_keys=True, indent=4, separators=(',', ': '))

# This takes an intersection in the format 'ROAD/HWY at ROAD/HWY',
# cleans it up and makes the request.
# on success: returns '(lat, long)'
# on failure: returns ''
def process(location):
    location += ' ' + CITY_STATE
    try:
        response = geocodeGoogle(location)
    except Exception as e:
        print('Caught Error:\n' + str(e))
        return ''
    if response[0] != '(':
        print('>> Error ' + response + ": " + location)
        if response == 'REQUEST_DENIED' or response == 'OVER_QUERY_LIMIT':
            overAPILimit = True
        return ''
    return response

# uses the Google Maps API to geocode an address
# on success: returns '(lat, lng)'
# on failure: returns the status code (a string)
def geocodeGoogle(address):
    # the bounds parameter biases results to locations within Alabama
    params = {'address': address,
              'bounds': ALABAMA_BOUNDS,
              'key': GOOGLE_API_KEY}
    url = GOOGLE_BASE_URL + urlencode(params, safe='/,|')
    rawreply = urlopen(url).read()
    reply = json.loads(rawreply.decode('utf-8'))
    sleep(0.1) # stay under usage limit
    if reply['status'] == 'OK':
        # assume the first result is correct, but only include intersections
        if 'intersection' in reply['results'][0]['types']:
            # round to 6 decimal places (~0.1m precision)
            return '(' + str(round(reply['results'][0]['geometry']['location']['lat'], 6)) + ', ' + \
                         str(round(reply['results'][0]['geometry']['location']['lng'], 6)) + ')'
        else:
            return 'NOT_AN_INTERSECTION'
    else:
        return reply['status']

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>import pathlib

def skip_example(example, board):
    ex_dir = pathlib.Path('examples/') / example
    bsp = pathlib.Path("hw/bsp")

    # board without family
    board_dir = bsp / board

    if board_dir.exists():
        family = ""
        mk_contents = ""
    else:
        # otherwise look into family
        board_dir = list(bsp.glob("*/boards/" + board))
        if not board_dir:
            # Skip unknown boards
            return True

        board_dir = list(board_dir)[0]

        family_dir = board_dir.parent.parent
        family = family_dir.name

        # family CMake
        family_mk = family_dir / "family.cmake"

        # family.mk
        if not family_mk.exists():
            family_mk = family_dir / "family.mk"

        mk_contents = family_mk.read_text()

    # Find the mcu, first in family mk then board mk
    if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents:
        board_mk = board_dir / "board.cmake"
        if not board_mk.exists():
            board_mk = board_dir / "board.mk"

        mk_contents = board_mk.read_text()

    for token in mk_contents.split():
        if "CFG_TUSB_MCU=OPT_MCU_" in token:
            # Strip " because cmake files has them.
            token = token.strip("\"")
            _, opt_mcu = token.split("=")
            mcu = opt_mcu[len("OPT_MCU_"):]

    # Skip all OPT_MCU_NONE these are WIP port
    if mcu == "NONE":
        return True

    skip_file = ex_dir / "skip.txt"
    only_file = ex_dir / "only.txt"

    if skip_file.exists() and only_file.exists():
        raise RuntimeError("Only have a skip or only file. Not both.")
    elif skip_file.exists():
        skips = skip_file.read_text().split()
        return ("mcu:" + mcu in skips or
                "board:" + board in skips or
                "family:" + family in skips)
    elif only_file.exists():
        onlys = only_file.read_text().split()
        return not ("mcu:" + mcu in onlys or
                    "board:" + board in onlys or
                    "family:" + family in onlys)

    return False
<commit_msg>fix build with fomu/fomu<commit_after>import pathlib

def skip_example(example, board):
    ex_dir = pathlib.Path('examples/') / example
    bsp = pathlib.Path("hw/bsp")

    if (bsp / board / "board.mk").exists():
        # board without family
        board_dir = bsp / board
        family = ""
        mk_contents = ""
    else:
        # board within family
        board_dir = list(bsp.glob("*/boards/" + board))
        if not board_dir:
            # Skip unknown boards
            return True

        board_dir = list(board_dir)[0]

        family_dir = board_dir.parent.parent
        family = family_dir.name

        # family CMake
        family_mk = family_dir / "family.cmake"

        # family.mk
        if not family_mk.exists():
            family_mk = family_dir / "family.mk"

        mk_contents = family_mk.read_text()

    # Find the mcu, first in family mk then board mk
    if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents:
        board_mk = board_dir / "board.cmake"
        if not board_mk.exists():
            board_mk = board_dir / "board.mk"

        mk_contents = board_mk.read_text()

    for token in mk_contents.split():
        if "CFG_TUSB_MCU=OPT_MCU_" in token:
            # Strip " because cmake files has them.
            token = token.strip("\"")
            _, opt_mcu = token.split("=")
            mcu = opt_mcu[len("OPT_MCU_"):]

    # Skip all OPT_MCU_NONE these are WIP port
    if mcu == "NONE":
        return True

    skip_file = ex_dir / "skip.txt"
    only_file = ex_dir / "only.txt"

    if skip_file.exists() and only_file.exists():
        raise RuntimeError("Only have a skip or only file. Not both.")
    elif skip_file.exists():
        skips = skip_file.read_text().split()
        return ("mcu:" + mcu in skips or
                "board:" + board in skips or
                "family:" + family in skips)
    elif only_file.exists():
        onlys = only_file.read_text().split()
        return not ("mcu:" + mcu in onlys or
                    "board:" + board in onlys or
                    "family:" + family in onlys)

    return False
<|endoftext|>


<commit_before># This file is part of the MicroPython project, http://micropython.org/
#
# The MIT License (MIT)
#
# Copyright (c) 2016 Rami Ali
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

""" gen-cpydiff generates documentation which outlines operations that differ between MicroPython
    and CPython. This script is called by the docs Makefile for html and Latex and may be run
    manually using the command make gen-cpydiff. """

import os
import errno
import subprocess
import time
import re
from collections import namedtuple

# MicroPython supports syntax of CPython 3.4 with some features from 3.5, and
# such version should be used to test for differences. If your default python3
# executable is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
if os.name == "nt":
    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe")
    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/windows/micropython.exe")
else:
    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/micropython")

TESTPATH = "../tests/cpydiff/"
DOCPATH = "../docs/genrst/"
INDEXTEMPLATE = "../docs/differences/index_template.txt"
INDEX = "index.rst"

HEADER = ".. This document was generated by tools/gen-cpydiff.py\n\n"
UIMPORTLIST = {"struct", "collections", "json"}
CLASSMAP = {"Core": "Core language", "Types": "Builtin types"}
INDEXPRIORITY = ["syntax", "core_language", "builtin_types", "modules"]
RSTCHARS = ["=", "-", "~", "`", ":"]
SPLIT = '"""\n|categories: |description: |cause: |workaround: '
TAB = "    "

Output = namedtuple(
    "output",
    [
        "name",
        "class_",
        "desc",
        "cause",
        "workaround",
        "code",
        "output_cpy",
        "output_upy",
        "status",
    ],
)

def readfiles():
    """Reads test files"""
    tests = list(filter(lambda x: x.endswith(".py"), os.listdir(TESTPATH)))
    tests.sort()
    files = []

    for test in tests:
        text = open(TESTPATH + test, "r").read()

        try:
            class_, desc, cause, workaround, code = [
                x.rstrip() for x in list(filter(None, re.split(SPLIT, text)))
            ]
            output = Output(test, class_, desc, cause, workaround, code, "", "", "")
            files.append(output)
        except IndexError:
            print("Incorrect format in file " + TESTPATH + test)

    return files

def uimports(code):
    """converts CPython module names into MicroPython equivalents"""
    for uimport in UIMPORTLIST:
        uimport = bytes(uimport, "utf8")
        code = code.replace(uimport, b"u" + uimport)
    return code

def run_tests(tests):
    """executes all tests"""
    results = []
    for test in tests:
        with open(TESTPATH + test.name, "rb") as f:
            input_cpy = f.read()
        input_upy = uimports(input_cpy)

        process = subprocess.Popen(
            CPYTHON3,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output_cpy = [com.decode("utf8") for com in process.communicate(input_cpy)]

        process = subprocess.Popen(
            MICROPYTHON,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output_upy = [com.decode("utf8") for com in process.communicate(input_upy)]

        if output_cpy[0] == output_upy[0] and output_cpy[1] == output_upy[1]:
            status = "Supported"
            print("Supported operation!\nFile: " + TESTPATH + test.name)
        else:
            status = "Unsupported"

        output = Output(
            test.name,
            test.class_,
            test.desc,
            test.cause,
            test.workaround,
            test.code,
            output_cpy,
            output_upy,
            status,
        )
        results.append(output)

    results.sort(key=lambda x: x.class_)
    return results

def indent(block, spaces):
    """indents paragraphs of text for rst formatting"""
    new_block = ""
    for line in block.split("\n"):
        new_block += spaces + line + "\n"
    return new_block

def gen_table(contents):
    """creates a table given any set of columns"""
    xlengths = []
    ylengths = []
    for column in contents:
        col_len = 0
        for entry in column:
            lines = entry.split("\n")
            for line in lines:
                col_len = max(len(line) + 2, col_len)
        xlengths.append(col_len)
    for i in range(len(contents[0])):
        ymax = 0
        for j in range(len(contents)):
            ymax = max(ymax, len(contents[j][i].split("\n")))
        ylengths.append(ymax)

    table_divider = "+" + "".join(["-" * i + "+" for i in xlengths]) + "\n"
    table = table_divider
    for i in range(len(ylengths)):
        row = [column[i] for column in contents]
        row = [entry + "\n" * (ylengths[i] - len(entry.split("\n"))) for entry in row]
        row = [entry.split("\n") for entry in row]
        for j in range(ylengths[i]):
            k = 0
            for entry in row:
                width = xlengths[k]
                table += "".join(["| {:{}}".format(entry[j], width - 1)])
                k += 1
            table += "|\n"
        table += table_divider
    return table + "\n"

def gen_rst(results):
    """creates restructured text documents to display tests"""

    # make sure the destination directory exists
    try:
        os.mkdir(DOCPATH)
    except OSError as e:
        if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
            raise

    toctree = []
    class_ = []
    for output in results:
        section = output.class_.split(",")
        for i in range(len(section)):
            section[i] = section[i].rstrip()
            if section[i] in CLASSMAP:
                section[i] = CLASSMAP[section[i]]
            if i >= len(class_) or section[i] != class_[i]:
                if i == 0:
                    filename = section[i].replace(" ", "_").lower()
                    rst = open(DOCPATH + filename + ".rst", "w")
                    rst.write(HEADER)
                    rst.write(section[i] + "\n")
                    rst.write(RSTCHARS[0] * len(section[i]))
                    rst.write(time.strftime("\nGenerated %a %d %b %Y %X UTC\n\n", time.gmtime()))
                    toctree.append(filename)
                else:
                    rst.write(section[i] + "\n")
                    rst.write(RSTCHARS[min(i, len(RSTCHARS) - 1)] * len(section[i]))
                    rst.write("\n\n")
        class_ = section
        rst.write(".. _cpydiff_%s:\n\n" % output.name.rsplit(".", 1)[0])
        rst.write(output.desc + "\n")
        rst.write("~" * len(output.desc) + "\n\n")
        if output.cause != "Unknown":
            rst.write("**Cause:** " + output.cause + "\n\n")
        if output.workaround != "Unknown":
            rst.write("**Workaround:** " + output.workaround + "\n\n")

        rst.write("Sample code::\n\n" + indent(output.code, TAB) + "\n")
        output_cpy = indent("".join(output.output_cpy[0:2]), TAB).rstrip()
        output_cpy = ("::\n\n" if output_cpy != "" else "") + output_cpy
        output_upy = indent("".join(output.output_upy[0:2]), TAB).rstrip()
        output_upy = ("::\n\n" if output_upy != "" else "") + output_upy
        table = gen_table([["CPy output:", output_cpy], ["uPy output:", output_upy]])
        rst.write(table)

    template = open(INDEXTEMPLATE, "r")
    index = open(DOCPATH + INDEX, "w")
    index.write(HEADER)
    index.write(template.read())
    for section in INDEXPRIORITY:
        if section in toctree:
            index.write(indent(section + ".rst", TAB))
            toctree.remove(section)
    for section in toctree:
        index.write(indent(section + ".rst", TAB))

def main():
    """Main function"""

    # set search path so that test scripts find the test modules (and no other ones)
    os.environ["PYTHONPATH"] = TESTPATH
    os.environ["MICROPYPATH"] = TESTPATH

    files = readfiles()
    results = run_tests(files)
    gen_rst(results)

main()
<commit_msg>tools/gen-cpydiff.py: Don't rename foo to ufoo in diff output.<commit_after># This file is part of the MicroPython project, http://micropython.org/
#
# The MIT License (MIT)
#
# Copyright (c) 2016 Rami Ali
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

""" gen-cpydiff generates documentation which outlines operations that differ between MicroPython
    and CPython. This script is called by the docs Makefile for html and Latex and may be run
    manually using the command make gen-cpydiff. """

import os
import errno
import subprocess
import time
import re
from collections import namedtuple

# MicroPython supports syntax of CPython 3.4 with some features from 3.5, and
# such version should be used to test for differences. If your default python3
# executable is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
if os.name == "nt":
    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe")
    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/windows/micropython.exe")
else:
    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/micropython")

TESTPATH = "../tests/cpydiff/"
DOCPATH = "../docs/genrst/"
INDEXTEMPLATE = "../docs/differences/index_template.txt"
INDEX = "index.rst"

HEADER = ".. This document was generated by tools/gen-cpydiff.py\n\n"
CLASSMAP = {"Core": "Core language", "Types": "Builtin types"}
INDEXPRIORITY = ["syntax", "core_language", "builtin_types", "modules"]
RSTCHARS = ["=", "-", "~", "`", ":"]
SPLIT = '"""\n|categories: |description: |cause: |workaround: '
TAB = "    "

Output = namedtuple(
    "output",
    [
        "name",
        "class_",
        "desc",
        "cause",
        "workaround",
        "code",
        "output_cpy",
        "output_upy",
        "status",
    ],
)

def readfiles():
    """Reads test files"""
    tests = list(filter(lambda x: x.endswith(".py"), os.listdir(TESTPATH)))
    tests.sort()
    files = []

    for test in tests:
        text = open(TESTPATH + test, "r").read()

        try:
            class_, desc, cause, workaround, code = [
                x.rstrip() for x in list(filter(None, re.split(SPLIT, text)))
            ]
            output = Output(test, class_, desc, cause, workaround, code, "", "", "")
            files.append(output)
        except IndexError:
            print("Incorrect format in file " + TESTPATH + test)

    return files

def run_tests(tests):
    """executes all tests"""
    results = []
    for test in tests:
        with open(TESTPATH + test.name, "rb") as f:
            input_py = f.read()

        process = subprocess.Popen(
            CPYTHON3,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output_cpy = [com.decode("utf8") for com in process.communicate(input_py)]

        process = subprocess.Popen(
            MICROPYTHON,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output_upy = [com.decode("utf8") for com in process.communicate(input_py)]

        if output_cpy[0] == output_upy[0] and output_cpy[1] == output_upy[1]:
            status = "Supported"
            print("Supported operation!\nFile: " + TESTPATH + test.name)
        else:
            status = "Unsupported"

        output = Output(
            test.name,
            test.class_,
            test.desc,
            test.cause,
            test.workaround,
            test.code,
            output_cpy,
            output_upy,
            status,
        )
        results.append(output)

    results.sort(key=lambda x: x.class_)
    return results

def indent(block, spaces):
    """indents paragraphs of text for rst formatting"""
    new_block = ""
    for line in block.split("\n"):
        new_block += spaces + line + "\n"
    return new_block

def gen_table(contents):
    """creates a table given any set of columns"""
    xlengths = []
    ylengths = []
    for column in contents:
        col_len = 0
        for entry in column:
            lines = entry.split("\n")
            for line in lines:
                col_len = max(len(line) + 2, col_len)
        xlengths.append(col_len)
    for i in range(len(contents[0])):
        ymax = 0
        for j in range(len(contents)):
            ymax = max(ymax, len(contents[j][i].split("\n")))
        ylengths.append(ymax)

    table_divider = "+" + "".join(["-" * i + "+" for i in xlengths]) + "\n"
    table = table_divider
    for i in range(len(ylengths)):
        row = [column[i] for column in contents]
        row = [entry + "\n" * (ylengths[i] - len(entry.split("\n"))) for entry in row]
        row = [entry.split("\n") for entry in row]
        for j in range(ylengths[i]):
            k = 0
            for entry in row:
                width = xlengths[k]
                table += "".join(["| {:{}}".format(entry[j], width - 1)])
                k += 1
            table += "|\n"
        table += table_divider
    return table + "\n"

def gen_rst(results):
    """creates restructured text documents to display tests"""

    # make sure the destination directory exists
    try:
        os.mkdir(DOCPATH)
    except OSError as e:
        if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
            raise

    toctree = []
    class_ = []
    for output in results:
        section = output.class_.split(",")
        for i in range(len(section)):
            section[i] = section[i].rstrip()
            if section[i] in CLASSMAP:
                section[i] = CLASSMAP[section[i]]
            if i >= len(class_) or section[i] != class_[i]:
                if i == 0:
                    filename = section[i].replace(" ", "_").lower()
                    rst = open(DOCPATH + filename + ".rst", "w")
                    rst.write(HEADER)
                    rst.write(section[i] + "\n")
                    rst.write(RSTCHARS[0] * len(section[i]))
                    rst.write(time.strftime("\nGenerated %a %d %b %Y %X UTC\n\n", time.gmtime()))
                    toctree.append(filename)
                else:
                    rst.write(section[i] + "\n")
                    rst.write(RSTCHARS[min(i, len(RSTCHARS) - 1)] * len(section[i]))
                    rst.write("\n\n")
        class_ = section
        rst.write(".. _cpydiff_%s:\n\n" % output.name.rsplit(".", 1)[0])
        rst.write(output.desc + "\n")
        rst.write("~" * len(output.desc) + "\n\n")
        if output.cause != "Unknown":
            rst.write("**Cause:** " + output.cause + "\n\n")
        if output.workaround != "Unknown":
            rst.write("**Workaround:** " + output.workaround + "\n\n")

        rst.write("Sample code::\n\n" + indent(output.code, TAB) + "\n")
        output_cpy = indent("".join(output.output_cpy[0:2]), TAB).rstrip()
        output_cpy = ("::\n\n" if output_cpy != "" else "") + output_cpy
        output_upy = indent("".join(output.output_upy[0:2]), TAB).rstrip()
        output_upy = ("::\n\n" if output_upy != "" else "") + output_upy
        table = gen_table([["CPy output:", output_cpy], ["uPy output:", output_upy]])
        rst.write(table)

    template = open(INDEXTEMPLATE, "r")
    index = open(DOCPATH + INDEX, "w")
    index.write(HEADER)
    index.write(template.read())
    for section in INDEXPRIORITY:
        if section in toctree:
            index.write(indent(section + ".rst", TAB))
            toctree.remove(section)
    for section in toctree:
        index.write(indent(section + ".rst", TAB))

def main():
    """Main function"""

    # set search path so that test scripts find the test modules (and no other ones)
    os.environ["PYTHONPATH"] = TESTPATH
    os.environ["MICROPYPATH"] = TESTPATH

    files = readfiles()
    results = run_tests(files)
    gen_rst(results)

main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2

"""Calculate texture properties for a ROI."""

import argparse
import glob
import re
import numpy as np
import skimage

import dwi.plot
import dwi.texture
import dwi.util
import dwi.dwimage

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--input', '-i', metavar='FILENAME', required=True,
            nargs='+', default=[], help='input ASCII file')
    p.add_argument('--basic', action='store_true',
            help='get basic texture properties')
    p.add_argument('--lbp', action='store_true',
            help='get local binary patterns')
    args = p.parse_args()
    return args

def normalize(pmap):
    """Normalize images within given range and convert to byte maps."""
    in_range = (0, 0.03)
    pmap = skimage.exposure.rescale_intensity(pmap, in_range=in_range)
    pmap = skimage.img_as_ubyte(pmap)
    return pmap

args = parse_args()
for infile in args.input:
    dwimage = dwi.dwimage.load(infile)[0]
    img = dwimage.image[0,:,:,0]
    basename = dwimage.basename
    if 1 in img.shape:
        img.shape = dwi.util.make2d(img.size)
    if args.verbose > 1:
        print 'Image shape: %s' % (img.shape,)

    # Write basic properties.
    if args.basic:
        img_normalized = normalize(img)
        propnames = ['median', 'mean', 'stddev']
        props = [np.median(img), np.mean(img), np.std(img)]
        propnames += dwi.texture.PROPNAMES
        props += dwi.texture.get_coprops_img(img_normalized)

        outfile = 'props_%s' % basename
        if args.verbose:
            print 'Writing (%s) to %s' % (', '.join(propnames), outfile)
        with open(outfile, 'w') as f:
            f.write(' '.join(map(str, props)) + '\n')

    # Write LBP properties.
    if args.lbp:
        lbp_freq_data, n_patterns = dwi.texture.get_lbp_freqs(img)
        lbp_freq_data.shape = (-1, n_patterns)

        outfile = 'lbpf_%s' % basename
        if args.verbose:
            print 'Writing LBP frequencies to %s' % outfile
        with open(outfile, 'w') as f:
            for patterns in lbp_freq_data:
                f.write(' '.join(map(str, patterns)) + '\n')

    img = img[50:150, 50:150]
    lbp_data, lbp_freq_data, patterns = dwi.texture.get_lbp_freqs(img)
    freqs = np.rollaxis(lbp_freq_data, 2)
    dwi.plot.show_images([[img, lbp_data], freqs[:5], freqs[5:]])
<commit_msg>Fixed<commit_after>#!/usr/bin/env python2

"""Calculate texture properties for a ROI."""

import argparse
import glob
import re
import numpy as np
import skimage

import dwi.plot
import dwi.texture
import dwi.util
import dwi.dwimage

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--input', '-i', metavar='FILENAME', required=True,
            nargs='+', default=[], help='input ASCII file')
    p.add_argument('--basic', action='store_true',
            help='get basic texture properties')
    p.add_argument('--lbp', action='store_true',
            help='get local binary patterns')
    args = p.parse_args()
    return args

def normalize(pmap):
    """Normalize images within given range and convert to byte maps."""
    in_range = (0, 0.03)
    pmap = skimage.exposure.rescale_intensity(pmap, in_range=in_range)
    pmap = skimage.img_as_ubyte(pmap)
    return pmap

args = parse_args()
for infile in args.input:
    dwimage = dwi.dwimage.load(infile)[0]
    img = dwimage.image[0,:,:,0]
    basename = dwimage.basename
    if 1 in img.shape:
        img.shape = dwi.util.make2d(img.size)
    if args.verbose > 1:
        print 'Image shape: %s' % (img.shape,)

    # Write basic properties.
    if args.basic:
        img_normalized = normalize(img)
        propnames = ['median', 'mean', 'stddev']
        props = [np.median(img), np.mean(img), np.std(img)]
        propnames += dwi.texture.PROPNAMES
        props += dwi.texture.get_coprops_img(img_normalized)

        outfile = 'props_%s' % basename
        if args.verbose:
            print 'Writing (%s) to %s' % (', '.join(propnames), outfile)
        with open(outfile, 'w') as f:
            f.write(' '.join(map(str, props)) + '\n')

    # Write LBP properties.
    if args.lbp:
        _, lbp_freq_data, n_patterns = dwi.texture.get_lbp_freqs(img)
        lbp_freq_data.shape = (-1, n_patterns)

        outfile = 'lbpf_%s' % basename
        if args.verbose:
            print 'Writing LBP frequencies to %s' % outfile
        with open(outfile, 'w') as f:
            for patterns in lbp_freq_data:
                f.write(' '.join(map(str, patterns)) + '\n')

    #img = img[50:150, 50:150]
    #lbp_data, lbp_freq_data, patterns = dwi.texture.get_lbp_freqs(img)
    #freqs = np.rollaxis(lbp_freq_data, 2)
    #dwi.plot.show_images([[img, lbp_data], freqs[:5], freqs[5:]])
<|endoftext|>


<commit_before>#!/usr/bin/env python2

"""Calculate texture properties for a masked area."""

from __future__ import absolute_import, division, print_function
import argparse
from collections import defaultdict
import logging

import numpy as np

import dwi.files
import dwi.mask
import dwi.standardize
import dwi.texture
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
                   help='increase verbosity')
    p.add_argument('--input', required=True,
                   help='input image')
    p.add_argument('--mask',
                   help='mask file to use')
    p.add_argument('--mode', metavar='MODE', required=True,
                   help='imaging mode specification')
    p.add_argument('--method', metavar='METHOD', required=True,
                   help='method')
    p.add_argument('--slices', default='maxfirst',
                   help='slice selection (maxfirst, max, all)')
    p.add_argument('--winspec', default='5',
                   help='window specification (side length, all, mbb)')
    p.add_argument('--portion', type=float, default=0,
                   help='portion of selected voxels required for each window')
    p.add_argument('--voxel', choices=('all', 'mean'), default='all',
                   help='voxel to output (all, mean)')
    p.add_argument('--output', metavar='FILENAME', required=True,
                   help='output texture map file')
    return p.parse_args()

def max_mask(mask, winsize):
    """Return a mask that has the voxels selected that have the maximum number
    of surrounding voxels selected in the original mask.
    """
    d = defaultdict(list)
    for pos, win in dwi.util.sliding_window(mask, winsize, mask=mask):
        d[np.count_nonzero(win)].append(pos)
    r = np.zeros_like(mask)
    for pos in d[max(d)]:
        r[pos] = True
    return r

def portion_mask(mask, winsize, portion=1, resort_to_max=True):
    """Return a mask that selects (only) voxels that have the window at each
    selected voxel origin up to a minimum portion in the original mask selected
    (1 means the whole window must be selected, 0 gives the original mask).

    If resort_to_max is true, the window with maximum number of selected voxels
    is used in case the resulting mask would otherwise be empty.
    """
    r = np.zeros_like(mask)
    for pos, win in dwi.util.sliding_window(mask, winsize, mask=mask):
        if np.count_nonzero(win) / win.size >= portion:
            r[pos] = True
    if resort_to_max and np.count_nonzero(r) == 0:
        r = max_mask(mask, winsize)
    return r

def main():
    args = parse_args()
    loglevel = logging.INFO if args.verbose else logging.WARNING
    logging.basicConfig(level=loglevel, stream=logging.sys.stdout)

    logging.info('Reading image: %s', args.input)
    img, attrs = dwi.files.read_pmap(args.input)
    if args.mode == 'T2':
        assert attrs['echotimes'][0] == 0  # TODO: Could be another?
    img = img[..., 0]
    assert img.ndim == 3

    if args.mask is None:
        mask = np.zeros_like(img, dtype=np.bool)
        mbb = dwi.util.bbox(img, pad=30)
        logging.info('MBB mask: %s', mbb)
        mask[mbb] = True
        mask = dwi.mask.Mask3D(mask)
    else:
        logging.info('Using mask: %s', args.mask)
        mask = dwi.mask.read_mask(args.mask)
        if isinstance(mask, dwi.mask.Mask):
            mask = mask.convert_to_3d(img.shape[0])

    if img.shape != mask.shape():
        raise Exception('Image shape {} does not match mask shape {}'.format(
            img.shape, mask.shape()))
    if mask.n_selected() == 0:
        raise ValueError('Empty mask.')

    if args.slices == 'maxfirst':
        slice_indices = [mask.max_slices()[0]]
    elif args.slices == 'max':
        slice_indices = mask.max_slices()
    elif args.slices == 'all':
        slice_indices = mask.selected_slices()
    else:
        raise Exception('Invalid slice set specification', args.slices)

    # Zero other slices in mask.
    for i in range(len(mask.array)):
        if i not in slice_indices:
            mask.array[i, :, :] = 0

    # Use only selected slices to save memory.
    if args.voxel == 'mean':
        img = img[slice_indices].copy()
        mask.array = mask.array[slice_indices].copy()

    # Get portion mask.
    if args.winspec in ('all', 'mbb'):
        pmask = mask.array  # Some methods don't use window.
    elif args.winspec.isdigit():
        winsize = int(args.winspec)
        assert winsize > 0
        winshape = (1, winsize, winsize)
        pmask = portion_mask(mask.array, winshape, portion=args.portion)
    else:
        raise ValueError('Invalid window spec: {}'.format(args.winspec))

    logging.info('Image: %s, slice: %s, voxels: %s, window: %s', img.shape,
                 slice_indices, np.count_nonzero(mask.array), args.winspec)

    logging.info('Calculating %s texture features for %s...', args.method,
                 args.mode)

    if args.voxel == 'mean':
        dwi.rcParams['texture.avg'] = True
    else:
        dwi.rcParams['texture.avg'] = False
        if args.mode.startswith('T2w') and args.method.startswith('gabor'):
            # These result arrays can get quite huge (if float64).
            dwi.rcParams['texture.path'] = args.output

    if args.method in ('glcm', 'glcm_mbb'):
        img = dwi.util.quantize(dwi.util.normalize(img, args.mode))

    tmap, names = dwi.texture.get_texture(img, args.method, args.winspec,
                                          pmask)
    attrs['parameters'] = names
    # Number of windows, or resulting texture map volume in general.
    attrs['tmap_voxels'] = np.count_nonzero(pmask)

    logging.info('Writing shape %s, type %s to %s', tmap.shape, tmap.dtype,
                 args.output)
    if dwi.rcParams['texture.path']:
        attrs['shape'] = tmap.shape
        attrs['dtype'] = str(tmap.dtype)
        tmap.attrs.update(attrs)
    else:
        dwi.files.write_pmap(args.output, tmap, attrs)

if __name__ == '__main__':
    main()
<commit_msg>no mbb padding<commit_after>#!/usr/bin/env python2

"""Calculate texture properties for a masked area."""

from __future__ import absolute_import, division, print_function
import argparse
from collections import defaultdict
import logging

import numpy as np

import dwi.files
import dwi.mask
import dwi.standardize
import dwi.texture
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
                   help='increase verbosity')
    p.add_argument('--input', required=True,
                   help='input image')
    p.add_argument('--mask',
                   help='mask file to use')
    p.add_argument('--mode', metavar='MODE', required=True,
                   help='imaging mode specification')
    p.add_argument('--method', metavar='METHOD', required=True,
                   help='method')
    p.add_argument('--slices', default='maxfirst',
                   help='slice selection (maxfirst, max, all)')
    p.add_argument('--winspec', default='5',
                   help='window specification (side length, all, mbb)')
    p.add_argument('--portion', type=float, default=0,
                   help='portion of selected voxels required for each window')
    p.add_argument('--voxel', choices=('all', 'mean'), default='all',
                   help='voxel to output (all, mean)')
    p.add_argument('--output', metavar='FILENAME', required=True,
                   help='output texture map file')
    return p.parse_args()

def max_mask(mask, winsize):
    """Return a mask that has the voxels selected that have the maximum number
    of surrounding voxels selected in the original mask.
    """
    d = defaultdict(list)
    for pos, win in dwi.util.sliding_window(mask, winsize, mask=mask):
        d[np.count_nonzero(win)].append(pos)
    r = np.zeros_like(mask)
    for pos in d[max(d)]:
        r[pos] = True
    return r

def portion_mask(mask, winsize, portion=1, resort_to_max=True):
    """Return a mask that selects (only) voxels that have the window at each
    selected voxel origin up to a minimum portion in the original mask selected
    (1 means the whole window must be selected, 0 gives the original mask).

    If resort_to_max is true, the window with maximum number of selected voxels
    is used in case the resulting mask would otherwise be empty.
    """
    r = np.zeros_like(mask)
    for pos, win in dwi.util.sliding_window(mask, winsize, mask=mask):
        if np.count_nonzero(win) / win.size >= portion:
            r[pos] = True
    if resort_to_max and np.count_nonzero(r) == 0:
        r = max_mask(mask, winsize)
    return r

def main():
    args = parse_args()
    loglevel = logging.INFO if args.verbose else logging.WARNING
    logging.basicConfig(level=loglevel, stream=logging.sys.stdout)

    logging.info('Reading image: %s', args.input)
    img, attrs = dwi.files.read_pmap(args.input)
    if args.mode == 'T2':
        assert attrs['echotimes'][0] == 0  # TODO: Could be another?
    img = img[..., 0]
    assert img.ndim == 3

    if args.mask is None:
        mask = np.zeros_like(img, dtype=np.bool)
        mbb = dwi.util.bbox(img, pad=0)
        logging.info('MBB mask: %s', mbb)
        mask[mbb] = True
        mask = dwi.mask.Mask3D(mask)
    else:
        logging.info('Using mask: %s', args.mask)
        mask = dwi.mask.read_mask(args.mask)
        if isinstance(mask, dwi.mask.Mask):
            mask = mask.convert_to_3d(img.shape[0])

    if img.shape != mask.shape():
        raise Exception('Image shape {} does not match mask shape {}'.format(
            img.shape, mask.shape()))
    if mask.n_selected() == 0:
        raise ValueError('Empty mask.')

    if args.slices == 'maxfirst':
        slice_indices = [mask.max_slices()[0]]
    elif args.slices == 'max':
        slice_indices = mask.max_slices()
    elif args.slices == 'all':
        slice_indices = mask.selected_slices()
    else:
        raise Exception('Invalid slice set specification', args.slices)

    # Zero other slices in mask.
    for i in range(len(mask.array)):
        if i not in slice_indices:
            mask.array[i, :, :] = 0

    # Use only selected slices to save memory.
    if args.voxel == 'mean':
        img = img[slice_indices].copy()
        mask.array = mask.array[slice_indices].copy()

    # Get portion mask.
    if args.winspec in ('all', 'mbb'):
        pmask = mask.array  # Some methods don't use window.
    elif args.winspec.isdigit():
        winsize = int(args.winspec)
        assert winsize > 0
        winshape = (1, winsize, winsize)
        pmask = portion_mask(mask.array, winshape, portion=args.portion)
    else:
        raise ValueError('Invalid window spec: {}'.format(args.winspec))

    logging.info('Image: %s, slice: %s, voxels: %s, window: %s', img.shape,
                 slice_indices, np.count_nonzero(mask.array), args.winspec)

    logging.info('Calculating %s texture features for %s...', args.method,
                 args.mode)

    if args.voxel == 'mean':
        dwi.rcParams['texture.avg'] = True
    else:
        dwi.rcParams['texture.avg'] = False
        if args.mode.startswith('T2w') and args.method.startswith('gabor'):
            # These result arrays can get quite huge (if float64).
            dwi.rcParams['texture.path'] = args.output

    if args.method in ('glcm', 'glcm_mbb'):
        img = dwi.util.quantize(dwi.util.normalize(img, args.mode))

    tmap, names = dwi.texture.get_texture(img, args.method, args.winspec,
                                          pmask)
    attrs['parameters'] = names
    # Number of windows, or resulting texture map volume in general.
    attrs['tmap_voxels'] = np.count_nonzero(pmask)

    logging.info('Writing shape %s, type %s to %s', tmap.shape, tmap.dtype,
                 args.output)
    if dwi.rcParams['texture.path']:
        attrs['shape'] = tmap.shape
        attrs['dtype'] = str(tmap.dtype)
        tmap.attrs.update(attrs)
    else:
        dwi.files.write_pmap(args.output, tmap, attrs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import git
import json
from subprocess import run, PIPE, CalledProcessError
from pathlib import Path
from tempfile import TemporaryDirectory

def decommify(name):
    return ' '.join(name.split(', ')[::-1])

# List of repositories whose commits should be counted as contributions
codependents = [('https://github.com/grabbles/grabbit.git', '0.2.6')]

# Last shablona commit
origin_commit = 'd72caaf5933907ed699d57faddaec7bfc836ce6f'

git_root = Path(git.Repo('.', search_parent_directories=True).working_dir)
zenodo_file = git_root / '.zenodo.json'

zenodo = json.loads(zenodo_file.read_text()) if zenodo_file.exists() else {}

orig_creators = zenodo.get('creators', [])
creator_map = {decommify(creator['name']): creator
               for creator in orig_creators}

shortlog = run(['git', 'shortlog', '-ns', f'{origin_commit}..'], stdout=PIPE)
counts = [line.split('\t', 1)[::-1]
          for line in shortlog.stdout.decode().split('\n') if line]

# Get additional commit counts from dependencies
with TemporaryDirectory() as tmpdir:
    tmppath = Path(tmpdir)
    for repo in codependents:
        try:
            repo, ref = repo
        except (TypeError, ValueError):
            ref = None
        repo_dir = str(tmppath / repo.rsplit('/', 1)[1].split('.', 1)[0])
        try:
            clone = run(['git', 'clone', '-q', repo, repo_dir], check=True)
        except CalledProcessError as err:
            raise RuntimeError("Could not clone {}".format(repo)) from err

        if ref is None:
            tag = run(['git', '-C', repo_dir, 'tag'], stdout=PIPE)
            # latest tag
            ref = tag.stdout.decode().strip().rsplit('\n', 1)[1]

        dep_shortlog = run(
            ['git', '-C', repo_dir, 'shortlog', '-ns', ref],
            stdout=PIPE)
        counts.extend(line.split('\t', 1)[::-1]
                      for line in dep_shortlog.stdout.decode().split('\n')
                      if line)

commit_counts = {}
for committer, commits in counts:
    commit_counts[committer] = commit_counts.get(committer, 0) + int(commits)

# Stable sort:
# Number of commits in reverse order
# Ties broken by alphabetical order of first name
committers = [committer
              for committer, _ in sorted(commit_counts.items(),
                                         key=lambda x: (-x[1], x[0]))]

# Tal to the top
first_author = 'Tal Yarkoni'
if committers[0] != first_author:
    committers.remove(first_author)
    committers.insert(0, first_author)

creators = [
    creator_map.get(committer, {'name': committer})
    for committer in committers
    ]

zenodo['creators'] = creators
zenodo_file.write_text(json.dumps(zenodo, indent=2) + '\n')
<commit_msg>MNT: Implement Zenodo blacklist<commit_after>#!/usr/bin/env python3
import git
import json
from subprocess import run, PIPE, CalledProcessError
from pathlib import Path
from tempfile import TemporaryDirectory

def decommify(name):
    return ' '.join(name.split(', ')[::-1])

# Users who have asked not to be cited at this time
# XXX We should add a shortlog since the most recent tag and explicitly note
# that a blacklisted user has contributed again recently, and verify they still
# do not want to be cited.
blacklist = {'Cecile Madjar'}

# List of repositories whose commits should be counted as contributions
codependents = [('https://github.com/grabbles/grabbit.git', '0.2.6')]

# Last shablona commit
origin_commit = 'd72caaf5933907ed699d57faddaec7bfc836ce6f'

git_root = Path(git.Repo('.', search_parent_directories=True).working_dir)
zenodo_file = git_root / '.zenodo.json'

zenodo = json.loads(zenodo_file.read_text()) if zenodo_file.exists() else {}

orig_creators = zenodo.get('creators', [])
creator_map = {decommify(creator['name']): creator
               for creator in orig_creators}

shortlog = run(['git', 'shortlog', '-ns', f'{origin_commit}..'], stdout=PIPE)
counts = [line.split('\t', 1)[::-1]
          for line in shortlog.stdout.decode().split('\n') if line]

# Get additional commit counts from dependencies
with TemporaryDirectory() as tmpdir:
    tmppath = Path(tmpdir)
    for repo in codependents:
        try:
            repo, ref = repo
        except (TypeError, ValueError):
            ref = None
        repo_dir = str(tmppath / repo.rsplit('/', 1)[1].split('.', 1)[0])
        try:
            clone = run(['git', 'clone', '-q', repo, repo_dir], check=True)
        except CalledProcessError as err:
            raise RuntimeError("Could not clone {}".format(repo)) from err

        if ref is None:
            tag = run(['git', '-C', repo_dir, 'tag'], stdout=PIPE)
            # latest tag
            ref = tag.stdout.decode().strip().rsplit('\n', 1)[1]

        dep_shortlog = run(
            ['git', '-C', repo_dir, 'shortlog', '-ns', ref],
            stdout=PIPE)
        counts.extend(line.split('\t', 1)[::-1]
                      for line in dep_shortlog.stdout.decode().split('\n')
                      if line)

commit_counts = {}
for committer, commits in counts:
    commit_counts[committer] = commit_counts.get(committer, 0) + int(commits)

# Stable sort:
# Number of commits in reverse order
# Ties broken by alphabetical order of first name
committers = [committer
              for committer, _ in sorted(commit_counts.items(),
                                         key=lambda x: (-x[1], x[0]))]

# Tal to the top
first_author = 'Tal Yarkoni'
if committers[0] != first_author:
    committers.remove(first_author)
    committers.insert(0, first_author)

creators = [
    creator_map.get(committer, {'name': committer})
    for committer in committers
    if committer not in blacklist
    ]

zenodo['creators'] = creators
zenodo_file.write_text(json.dumps(zenodo, indent=2) + '\n')
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
# Copyright 2013 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# Using the REST API, populates the DB with some sample data
# Based on python-ceilometerclient/ceilometerclient/common/http.py

import httplib
import json
import logging
import socket
import urlparse
import uuid

LOG = logging.getLogger(__name__)

def log_curl_request(conn, base_url, url, method, kwargs):
    curl = ['curl -i -X %s' % method]

    for (key, value) in kwargs['headers'].items():
        header = '-H \'%s: %s\'' % (key, value)
        curl.append(header)

    if 'body' in kwargs:
        curl.append('-d \'%s\'' % kwargs['body'])

    curl.append('http://%s:%d%s%s' % (conn.host, conn.port, base_url, url))
    LOG.debug(' '.join(curl))

def log_http_response(resp, body=None):
    status = (resp.version / 10.0, resp.status, resp.reason)
    dump = ['\nHTTP/%.1f %s %s' % status]
    dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
    dump.append('')
    if body:
        dump.extend([body, ''])
    LOG.debug('\n'.join(dump))

def make_connection_url(base_url, url):
    return '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))

def http_request(conn, base_url, url, method, **kwargs):
    log_curl_request(conn, base_url, url, method, kwargs)

    try:
        conn_url = make_connection_url(base_url, url)
        conn.request(method, conn_url, **kwargs)
        resp = conn.getresponse()
    except socket.gaierror as e:
        message = ('Error finding address for %(url)s: %(e)s' %
                   {'url': url, 'e': e})
        raise RuntimeError(message)
    except (socket.error, socket.timeout) as e:
        message = ('Error communicating with %(endpoint)s %(e)s' %
                   {'endpoint': 'http://%s:%d' % (conn.host, conn.port),
                    'e': e})
        raise RuntimeError(message)

    if 300 <= resp.status < 600:
        LOG.warn('Request returned failure/redirect status.')
        raise RuntimeError('Status code %d returned' % resp.status)

    body = resp.read()

    log_http_response(resp, body)

    return resp, body

def json_request(conn, base_url, url, method, **kwargs):
    kwargs.setdefault('headers', {})
    kwargs['headers'].setdefault('Content-Type', 'application/json')
    kwargs['headers'].setdefault('Accept', 'application/json')

    if 'body' in kwargs:
        kwargs['body'] = json.dumps(kwargs['body'])

    resp, body = http_request(conn, base_url, url, method, **kwargs)
    content_type = resp.getheader('content-type', None)

    if resp.status == 204 or resp.status == 205 or content_type is None:
        body = None
    elif 'application/json' in content_type:
        try:
            body = json.loads(body)
        except ValueError:
            LOG.error('Could not decode response body as JSON')
    else:
        body = None

    return resp, body

def create_resource_class(conn, base_url, name, service_type):
    return json_request(conn, base_url, '/resource_classes', 'POST',
                        body=dict(name=name, service_type=service_type))

def create_rack(conn, base_url, name, slots, subnet, capacities):
    return json_request(conn, base_url, '/racks', 'POST',
                        body=dict(name=name, slots=slots, subnet=subnet,
                                  capacities=capacities))

def set_nodes_on_rack(conn, base_url, rack_url, nodes):
    return json_request(conn, base_url, rack_url, 'PUT',
                        body=dict(nodes=nodes))

def create_flavor(conn, base_url, flavor):
    return json_request(conn, base_url, '/flavors', 'POST', body=flavor)

def get_location(base_url, resp):
    return urlparse.urlparse(resp.getheader('location')).path[len(base_url):]

def generate_uuid():
    return str(uuid.uuid4())

if __name__ == '__main__':
    logging.basicConfig(format='%(message)s', level=logging.DEBUG)

    conn = httplib.HTTPConnection('localhost', 6385)

    base_url = '/v1'

    resp, body = create_resource_class(conn, base_url,
                                       name='m1', service_type='compute')
    m1_url = get_location(base_url, resp)

    capacities = [
        dict(name='total_cpu', value='12', unit='count'),
        dict(name='total_memory', value='3072', unit='MB'),
    ]
    resp, body = create_rack(conn, base_url,
                             name='rack1', slots=30,
                             subnet='192.168.1.0/24',
                             capacities=capacities)
    rack1_url = get_location(base_url, resp)

    nodes = []
    while len(nodes) < 30:
        nodes.append(dict(id=generate_uuid()))
    set_nodes_on_rack(conn, base_url, rack1_url, nodes)

    flavors = [
        dict(name='m1.small',
             capacities=[
                dict(name='cpu', value='1', unit='count'),
                dict(name='memory', value='1.7', unit='GB'),
                dict(name='storage', value='160', unit='GB'),
             ]),
        dict(name='m1.medium',
             capacities=[
                dict(name='cpu', value='1', unit='count'),
                dict(name='memory', value='3.75', unit='GB'),
                dict(name='storage', value='410', unit='GB'),
             ]),
        dict(name='m1.large',
             capacities=[
                dict(name='cpu', value='2', unit='count'),
                dict(name='memory', value='7.5', unit='GB'),
                dict(name='storage', value='840', unit='GB'),
             ]),
        dict(name='m1.xlarge',
             capacities=[
                dict(name='cpu', value='4', unit='count'),
                dict(name='memory', value='15', unit='GB'),
                dict(name='storage', value='1680', unit='GB'),
             ]),
    ]
    for flavor in flavors:
        create_flavor(conn, base_url, flavor)
<commit_msg>Fix sample data for flavors.<commit_after>#!/usr/bin/env python

# Copyright 2012 OpenStack LLC.
# All Rights Reserved.
# Copyright 2013 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# Using the REST API, populates the DB with some sample data
# Based on python-ceilometerclient/ceilometerclient/common/http.py

import httplib
import json
import logging
import socket
import urlparse
import uuid

LOG = logging.getLogger(__name__)

def log_curl_request(conn, base_url, url, method, kwargs):
    curl = ['curl -i -X %s' % method]

    for (key, value) in kwargs['headers'].items():
        header = '-H \'%s: %s\'' % (key, value)
        curl.append(header)

    if 'body' in kwargs:
        curl.append('-d \'%s\'' % kwargs['body'])

    curl.append('http://%s:%d%s%s' % (conn.host, conn.port, base_url, url))
    LOG.debug(' '.join(curl))

def log_http_response(resp, body=None):
    status = (resp.version / 10.0, resp.status, resp.reason)
    dump = ['\nHTTP/%.1f %s %s' % status]
    dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
    dump.append('')
    if body:
        dump.extend([body, ''])
    LOG.debug('\n'.join(dump))

def make_connection_url(base_url, url):
    return '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))

def http_request(conn, base_url, url, method, **kwargs):
    log_curl_request(conn, base_url, url, method, kwargs)

    try:
        conn_url = make_connection_url(base_url, url)
        conn.request(method, conn_url, **kwargs)
        resp = conn.getresponse()
    except socket.gaierror as e:
        message = ('Error finding address for %(url)s: %(e)s' %
                   {'url': url, 'e': e})
        raise RuntimeError(message)
    except (socket.error, socket.timeout) as e:
        message = ('Error communicating with %(endpoint)s %(e)s' %
                   {'endpoint': 'http://%s:%d' % (conn.host, conn.port),
                    'e': e})
        raise RuntimeError(message)

    if 300 <= resp.status < 600:
        LOG.warn('Request returned failure/redirect status.')
        raise RuntimeError('Status code %d returned' % resp.status)

    body = resp.read()

    log_http_response(resp, body)

    return resp, body

def json_request(conn, base_url, url, method, **kwargs):
    kwargs.setdefault('headers', {})
    kwargs['headers'].setdefault('Content-Type', 'application/json')
    kwargs['headers'].setdefault('Accept', 'application/json')

    if 'body' in kwargs:
        kwargs['body'] = json.dumps(kwargs['body'])

    resp, body = http_request(conn, base_url, url, method, **kwargs)
    content_type = resp.getheader('content-type', None)

    if resp.status == 204 or resp.status == 205 or content_type is None:
        body = None
    elif 'application/json' in content_type:
        try:
            body = json.loads(body)
        except ValueError:
            LOG.error('Could not decode response body as JSON')
    else:
        body = None

    return resp, body

def create_resource_class(conn, base_url, name, service_type):
    return json_request(conn, base_url, '/resource_classes', 'POST',
                        body=dict(name=name, service_type=service_type))

def create_rack(conn, base_url, name, slots, subnet, capacities):
    return json_request(conn, base_url, '/racks', 'POST',
                        body=dict(name=name, slots=slots, subnet=subnet,
                                  capacities=capacities))

def set_nodes_on_rack(conn, base_url, rack_url, nodes):
    return json_request(conn, base_url, rack_url, 'PUT',
                        body=dict(nodes=nodes))

def create_flavor(conn, base_url, resource_class_url, flavor):
    return json_request(conn, base_url + resource_class_url, '/flavors', 'PUT', body=flavor)

def get_location(base_url, resp):
    return urlparse.urlparse(resp.getheader('location')).path[len(base_url):]

def generate_uuid():
    return str(uuid.uuid4())

if __name__ == '__main__':
    logging.basicConfig(format='%(message)s', level=logging.DEBUG)

    conn = httplib.HTTPConnection('localhost', 6385)

    base_url = '/v1'

    resp, body = create_resource_class(conn, base_url,
                                       name='m1', service_type='compute')
    m1_url = get_location(base_url, resp)

    capacities = [
        dict(name='total_cpu', value='12', unit='count'),
        dict(name='total_memory', value='3072', unit='MB'),
    ]
    resp, body = create_rack(conn, base_url,
                             name='rack1', slots=30,
                             subnet='192.168.1.0/24',
                             capacities=capacities)
    rack1_url = get_location(base_url, resp)

    nodes = []
    while len(nodes) < 30:
        nodes.append(dict(id=generate_uuid()))
    set_nodes_on_rack(conn, base_url, rack1_url, nodes)

    flavors = [
        dict(name='m1.small',
             capacities=[
                dict(name='cpu', value='1', unit='count'),
                dict(name='memory', value='1.7', unit='GB'),
                dict(name='storage', value='160', unit='GB'),
             ]),
        dict(name='m1.medium',
             capacities=[
                dict(name='cpu', value='1', unit='count'),
                dict(name='memory', value='3.75', unit='GB'),
                dict(name='storage', value='410', unit='GB'),
             ]),
        dict(name='m1.large',
             capacities=[
                dict(name='cpu', value='2', unit='count'),
                dict(name='memory', value='7.5', unit='GB'),
                dict(name='storage', value='840', unit='GB'),
             ]),
        dict(name='m1.xlarge',
             capacities=[
                dict(name='cpu', value='4', unit='count'),
                dict(name='memory', value='15', unit='GB'),
                dict(name='storage', value='1680', unit='GB'),
             ]),
    ]
    for flavor in flavors:
        create_flavor(conn, base_url, m1_url, flavor)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import ConfigParser
import sys
import os
import multiprocessing
from pprint import pprint
from benchmark.benchmarker import Benchmarker
from setup.linux.unbuffered import Unbuffered

###################################################################################################
# Main
###################################################################################################
def main(argv=None):
    # Do argv default this way, as doing it in the functional declaration sets it at compile time
    if argv is None:
        argv = sys.argv

    # Enable unbuffered output so messages will appear in the proper order with subprocess output.
    sys.stdout=Unbuffered(sys.stdout)

    # Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
    sys.path.append('.')

    # Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
    sys.path.append('toolset/setup/linux')

    conf_parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        add_help=False)
    conf_parser.add_argument('--conf_file', default='benchmark.cfg', metavar='FILE', help='Optional configuration file to provide argument defaults. All config options can be overridden using the command line.')
    args, remaining_argv = conf_parser.parse_known_args()

    try:
        with open (args.conf_file):
            config = ConfigParser.SafeConfigParser()
            config.read([os.getcwd() + '/' + args.conf_file])
            defaults = dict(config.items("Defaults"))
    except IOError:
        if args.conf_file != 'benchmark.cfg':
            print 'Configuration file not found!'
        defaults = { "client-host":"localhost"}

    ##########################################################
    # Set up default values
    ##########################################################
    serverHost = os.environ.get('TFB_SERVER_HOST')
    clientHost = os.environ.get('TFB_CLIENT_HOST')
    clientUser = os.environ.get('TFB_CLIENT_USER')
    clientIden = os.environ.get('TFB_CLIENT_IDENTITY_FILE')
    databaHost = os.getenv('TFB_DATABASE_HOST', clientHost)
    databaUser = os.getenv('TFB_DATABASE_USER', clientUser)
    dbIdenFile = os.getenv('TFB_DATABASE_IDENTITY_FILE', clientIden)
    maxThreads = 8
    try:
        maxThreads = multiprocessing.cpu_count()
    except:
        pass

    ##########################################################
    # Set up argument parser
    ##########################################################
    parser = argparse.ArgumentParser(description='Run the Framework Benchmarking test suite.',
        parents=[conf_parser])
    parser.add_argument('-s', '--server-host', default=serverHost, help='The application server.')
    parser.add_argument('-c' ,'--client-host', default=clientHost, help='The client / load generation server.')
    parser.add_argument('-u', '--client-user', default=clientUser, help='The username to use for SSH to the client instance.')
    parser.add_argument('-i', '--client-identity-file', default=clientIden, help='The key to use for SSH to the client instance.')
    parser.add_argument('-d', '--database-host', default=databaHost, help='The database server.  If not provided, defaults to the value of --client-host.')
    parser.add_argument('--database-user', default=databaUser, help='The username to use for SSH to the database instance.  If not provided, defaults to the value of --client-user.')
    parser.add_argument('--database-identity-file', default=dbIdenFile, dest='database_identity_file', help='The key to use for SSH to the database instance.  If not provided, defaults to the value of --client-identity-file.')
    parser.add_argument('-p', dest='password_prompt', action='store_true')
    parser.add_argument('--install-software', action='store_true', help='runs the installation script before running the rest of the commands')
    parser.add_argument('--install', choices=['client', 'database', 'server', 'all'], default='all', help='Allows you to only install the server, client, or database software')
    parser.add_argument('--install-error-action', choices=['abort', 'continue'], default='continue', help='action to take in case of error during installation')
    parser.add_argument('--test', nargs='+', help='names of tests to run')
    parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
    parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
    parser.add_argument('-m', '--mode', choices=['benchmark', 'verify'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown')
    parser.add_argument('--list-tests', action='store_true', default=False, help='lists all the known tests that can run')
    parser.add_argument('--list-test-metadata', action='store_true', default=False, help='writes all the test metadata as a JSON file in the results directory')
    parser.add_argument('--max-concurrency', default=256, help='the maximum concurrency that the tests will run at. The query tests will run at this concurrency', type=int)
    parser.add_argument('--max-queries', default=20, help='The maximum number of queries to run during the query test', type=int)
    parser.add_argument('--query-interval', default=5, type=int)
    parser.add_argument('--max-threads', default=maxThreads, help='The max number of threads to run weight at, this should be set to the number of cores for your system.', type=int)
    parser.add_argument('--duration', default=15, help='Time in seconds that each test should run for.')
    parser.add_argument('--starting-concurrency', default=8, type=int)
    parser.add_argument('--sleep', type=int, default=60, help='the amount of time to sleep after starting each test to allow the server to start up.')
    parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')
    parser.add_argument('--name', default="ec2", help='The name to give this test. Results will be placed in a folder using this name.')
    parser.add_argument('--os', choices=['linux', 'windows'], default='linux', help='The operating system of the application server.')
    parser.add_argument('--database-os', choices=['linux', 'windows'], default='linux', help='The operating system of the database server.')
    parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Causes the configuration to print before any other commands are executed.')
    parser.set_defaults(**defaults) # Must do this after add, or each option's default will override the configuration file default
    args = parser.parse_args(remaining_argv)

    if args.verbose:
        print 'Configuration options: '
        pprint(args)
    benchmarker = Benchmarker(vars(args))

    # Run the benchmarker in the specified mode
    if benchmarker.list_tests:
      benchmarker.run_list_tests()
    elif benchmarker.list_test_metadata:
      benchmarker.run_list_test_metadata()
    elif benchmarker.parse != None:
      benchmarker.parse_timestamp()
    else:
      benchmarker.run()

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Cleaner syntax for code describing program flags<commit_after>#!/usr/bin/env python
import argparse
import ConfigParser
import sys
import os
import multiprocessing
from pprint import pprint
from benchmark.benchmarker import Benchmarker
from setup.linux.unbuffered import Unbuffered

###################################################################################################
# Main
###################################################################################################
def main(argv=None):
    # Do argv default this way, as doing it in the functional declaration sets it at compile time
    if argv is None:
        argv = sys.argv

    # Enable unbuffered output so messages will appear in the proper order with subprocess output.
    sys.stdout=Unbuffered(sys.stdout)

    # Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
    sys.path.append('.')

    # Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
    sys.path.append('toolset/setup/linux')

    conf_parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        add_help=False)
    conf_parser.add_argument('--conf_file', default='benchmark.cfg', metavar='FILE', help='Optional configuration file to provide argument defaults. All config options can be overridden using the command line.')
    args, remaining_argv = conf_parser.parse_known_args()

    try:
        with open (args.conf_file):
            config = ConfigParser.SafeConfigParser()
            config.read([os.getcwd() + '/' + args.conf_file])
            defaults = dict(config.items("Defaults"))
    except IOError:
        if args.conf_file != 'benchmark.cfg':
            print 'Configuration file not found!'
        defaults = { "client-host":"localhost"}

    ##########################################################
    # Set up default values
    ##########################################################
    serverHost = os.environ.get('TFB_SERVER_HOST')
    clientHost = os.environ.get('TFB_CLIENT_HOST')
    clientUser = os.environ.get('TFB_CLIENT_USER')
    clientIden = os.environ.get('TFB_CLIENT_IDENTITY_FILE')
    databaHost = os.getenv('TFB_DATABASE_HOST', clientHost)
    databaUser = os.getenv('TFB_DATABASE_USER', clientUser)
    dbIdenFile = os.getenv('TFB_DATABASE_IDENTITY_FILE', clientIden)
    maxThreads = 8
    try:
        maxThreads = multiprocessing.cpu_count()
    except:
        pass

    ##########################################################
    # Set up argument parser
    ##########################################################
    parser = argparse.ArgumentParser(description='Run the Framework Benchmarking test suite.',
        parents=[conf_parser],
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # SSH options
    parser.add_argument('-s', '--server-host', default=serverHost, help='The application server.')
    parser.add_argument('-c', '--client-host', default=clientHost, help='The client / load generation server.')
    parser.add_argument('-u', '--client-user', default=clientUser, help='The username to use for SSH to the client instance.')
    parser.add_argument('-i', '--client-identity-file', dest='client_identity_file', default=clientIden,
                        help='The key to use for SSH to the client instance.')
    parser.add_argument('-d', '--database-host', default=databaHost,
                        help='The database server.  If not provided, defaults to the value of --client-host.')
    parser.add_argument('--database-user', default=databaUser,
                        help='The username to use for SSH to the database instance.  If not provided, defaults to the value of --client-user.')
    parser.add_argument('--database-identity-file', default=dbIdenFile, dest='database_identity_file',
                        help='The key to use for SSH to the database instance.  If not provided, defaults to the value of --client-identity-file.')
    parser.add_argument('-p', dest='password_prompt', action='store_true', help='Prompt for password')

    # Install options
    parser.add_argument('--install', choices=['client', 'database', 'server', 'all'], default=None,
                        help='Runs installation script(s) before continuing on to execute the tests.')
    parser.add_argument('--install-error-action', choices=['abort', 'continue'], default='continue', help='action to take in case of error during installation')

    # Test options
    parser.add_argument('--test', nargs='+', help='names of tests to run')
    parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
    parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
    parser.add_argument('-m', '--mode', choices=['benchmark', 'verify'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown')
    parser.add_argument('--list-tests', action='store_true', default=False, help='lists all the known tests that can run')
    parser.add_argument('--list-test-metadata', action='store_true', default=False, help='writes all the test metadata as a JSON file in the results directory')
    parser.add_argument('--name', default="ec2", help='The name to give this test. Results will be placed in a folder using this name.')
    parser.add_argument('--os', choices=['linux', 'windows'], default='linux', help='The operating system of the application/framework server (the one running' +
                        'this binary')
    parser.add_argument('--database-os', choices=['linux', 'windows'], default='linux', help='The operating system of the database server.')

    # Benchmark options
    parser.add_argument('--max-concurrency', default=256, help='the maximum number of HTTP connections that wrk will keep open. The query tests will run at this maximum', type=int)
    parser.add_argument('--max-queries', default=20, help='The maximum number of queries to run during the query test', type=int)
    parser.add_argument('--query-interval', default=5, type=int, help='Query tests will go from 1 query to max queries in increments of interval queries')
    parser.add_argument('--max-threads', default=maxThreads, help='The max number of threads to run wrk at. This should be set to the number of cores for your client system.', type=int)
    parser.add_argument('--duration', default=15, help='Time in seconds that each test should run for.')
    parser.add_argument('--starting-concurrency', default=8, type=int)
    parser.add_argument('--sleep', type=int, default=60, help='the amount of time to sleep after starting each test to allow the server to start up.')

    # Misc Options
    parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')
    parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Causes the configuration to print before any other commands are executed.')
    parser.set_defaults(**defaults) # Must do this after add, or each option's default will override the configuration file default
    args = parser.parse_args(remaining_argv)

    # Verify and massage options
    if args.client_user is None:
      print 'Usernames (e.g. --client-user and --database-user) are required!'
      print 'The system will SSH into the client and the database for the install stage'
      print 'Aborting'
      exit(1)

    if args.database_user is None:
      args.database_user = args.client_user

    if args.database_host is None:
      args.database_host = args.client_host

    if args.verbose:
        print 'Configuration options: '
        pprint(args)
    benchmarker = Benchmarker(vars(args))

    # Run the benchmarker in the specified mode
    if benchmarker.list_tests:
      benchmarker.run_list_tests()
    elif benchmarker.list_test_metadata:
      benchmarker.run_list_test_metadata()
    elif benchmarker.parse != None:
      benchmarker.parse_timestamp()
    else:
      benchmarker.run()

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before># Calculo de Coordenadas (x,y,z)

## Profesor Revisor:
### Dr Ramon Solano Barajas

## Elaborado por:
### Jairo Avalos Velazquez
### Alan Mauricio Cruz Otero

def calcoor(csv):
    """
    Esta es una funcion para realizar el calculo de coordenadas en la biblioteca topgeo.
    """

     # Agregar los modulos numpy y pandas
    import numpy as np
    import pandas as pd
    from Tkinter import *
    from tkFileDialog import askopenfilename

    def llamada():
        nombre = askopenfilename()
        print nombre

    errmsg = "Error!"
    Button(text='Abrir Archivo', command=llamada).pack(fill=mainloop()

    llamada()

    #Almacenamos en la variable pts el archivo csv
    pts = pd.read_csv(llamada)

    #Se almacenan las variables en las columnas que queremos del csv
    DH = pts['Dist_H']
    AH = pts['Ang_H']
    AV = pts['Ang_V']

    #Se crean nuevas columnas en nuestra tabla y se hace el calculo de ellas
    pts ['Pr_x'] = (np.sin(np.radians(AH))* DH)
    pts ['Pr_y'] = (np.cos(np.radians(AH))* DH)
    pts ['Dist_V'] = (np.tan(np.radians(AV))* DH)

    pts.head()

    #Se almacenan las nuevas variables en las columnas que queremos de la tabla
    Px = pts['Pr_x']
    Py = pts['Pr_y']
    DV = pts['Dist_V']

    # Escribir las coordenadas arbitrarias donde se inicio el levantamiento con Estacion Total(X,Y,Z)
    xx = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento x: "))
    yy = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento y: "))
    zz = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento Z: "))

    # Escribir el altura del aparato y altura del prisma
    Alt_A = float(input("Ingresar altura del Aparato Estación Total: "))
    Alt_P = float(input("Ingresar altura del prisma: "))

    #Se crean nuevas columnas en nuestra tabla y se hace el calculo de ellas
    pts ['Coo_x'] = (xx + Px)
    pts ['Coo_y'] = (yy + Py)
    pts ['Coo_z'] = (zz + Alt_A + DV - Alt_P)

    pts.head()

    x = pts['Coo_x']
    y = pts['Coo_y']
    z = pts['Coo_z']

    #Se guarda la tabla terminada con los calculos realizados en un csv
    pts.to_csv('Lev_canal_Terminado.csv')

    # Se muestran los datos calculados en el archivo csv se muestran en pantalla
    return pts

calcoor('Lev_canal.csv')
<commit_msg>Update coordenada.py<commit_after># Calculo de Coordenadas (x,y,z)

## Profesor Revisor:
### Dr Ramon Solano Barajas

## Elaborado por:
### Jairo Avalos Velazquez
### Alan Mauricio Cruz Otero

def calcoor(csv):
    """
    Esta es una funcion para realizar el calculo de coordenadas en la biblioteca topgeo.
    """

     # Agregar los modulos numpy y pandas
    import numpy as np
    import pandas as pd
    from Tkinter import *
    from tkFileDialog import askopenfilename

    def llamada():
        nombre = askopenfilename()
        print nombre

    errmsg = "Error!"
    Button(text='Abrir Archivo', command=llamada).pack(fill=mainloop()

    #Almacenamos en la variable pts el archivo csv
    pts = pd.read_csv(llamada)

    #Se almacenan las variables en las columnas que queremos del csv
    DH = pts['Dist_H']
    AH = pts['Ang_H']
    AV = pts['Ang_V']

    #Se crean nuevas columnas en nuestra tabla y se hace el calculo de ellas
    pts ['Pr_x'] = (np.sin(np.radians(AH))* DH)
    pts ['Pr_y'] = (np.cos(np.radians(AH))* DH)
    pts ['Dist_V'] = (np.tan(np.radians(AV))* DH)

    pts.head()

    #Se almacenan las nuevas variables en las columnas que queremos de la tabla
    Px = pts['Pr_x']
    Py = pts['Pr_y']
    DV = pts['Dist_V']

    # Escribir las coordenadas arbitrarias donde se inicio el levantamiento con Estacion Total(X,Y,Z)
    xx = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento x: "))
    yy = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento y: "))
    zz = float(input("Ingresar coordenada arbitraria donde se inicio el levantamiento Z: "))

    # Escribir el altura del aparato y altura del prisma
    Alt_A = float(input("Ingresar altura del Aparato Estación Total: "))
    Alt_P = float(input("Ingresar altura del prisma: "))

    #Se crean nuevas columnas en nuestra tabla y se hace el calculo de ellas
    pts ['Coo_x'] = (xx + Px)
    pts ['Coo_y'] = (yy + Py)
    pts ['Coo_z'] = (zz + Alt_A + DV - Alt_P)

    pts.head()

    x = pts['Coo_x']
    y = pts['Coo_y']
    z = pts['Coo_z']

    #Se guarda la tabla terminada con los calculos realizados en un csv
    pts.to_csv('Lev_canal_Terminado.csv')

    # Se muestran los datos calculados en el archivo csv se muestran en pantalla
    return pts

calcoor('Lev_canal.csv')
<|endoftext|>


<commit_before>import logging
import os
import zipfile
from os import path

from toscaparser.prereq.csar import CSAR

class CONST:
    # DF_HOST = 'http://131.114.2.77/search'
    # DF_HOST = 'http://127.0.0.1:3000/search'
    DF_HOST = 'http://black.di.unipi.it:3000'
    DF_SEARCH = '/search'

    # CONTAINER_TYPE = 'tosker.nodes.Container'
    SOFTWARE_TYPE = 'tosker.nodes.Software'
    # IMAGE_TYPE = 'tosker.artifacts.Image'
    # DOCKERFILE_TYPE = 'tosker.artifacts.Dockerfile'

    PROPERTY_SW = 'supported_sw'
    PROPERTY_OS = 'os_distribution'
    REQUIREMENT_HOST = 'host'

    POLICY_TOP = 'top_rated'
    POLICY_SIZE = 'size'
    POLICY_USED = 'most_used'

class Logger:
    _ch = None

    @staticmethod
    def set_logger(level=logging.DEBUG):
        # global Logger._ch
        Logger._ch = logging.StreamHandler()
        Logger._ch.setLevel(level)
        formatter = logging.Formatter((
            '%(levelname) -3s %(asctime)s %(name)'
            '-3s %(funcName)'
            '-1s %(lineno) -0s: %(message)s'
        ))
        Logger._ch.setFormatter(formatter)

    @staticmethod
    def get(name, level=logging.DEBUG):
        log = logging.getLogger(name)
        log.setLevel(level)
        if Logger._ch is not None:
            log.addHandler(Logger._ch)
        return log

def unpack_csar(file_path):
    # Work around bug validation csar of toscaparser
    csar = CSAR(file_path)
    try:
        csar.validate()
    except ValueError as e:
        # _log.debug(e)
        if not str(e).startswith("The resource") or \
           not str(e).endswith("does not exist."):
            raise e

    csar.decompress()

    yaml_path = path.join(csar.temp_dir, csar.get_main_template())
    return (csar.temp_dir, yaml_path)

def pack_csar(csar_tmp_path, new_path):
    with zipfile.ZipFile(new_path, "w", zipfile.ZIP_DEFLATED) as zf:
        for dirname, subdirs, files in os.walk(csar_tmp_path):
            # zf.write(dirname, root)
            for filename in files:
                zf.write(path.join(dirname, filename),
                         path.relpath(path.join(dirname, filename),
                                      path.join(csar_tmp_path, '.')))

def get_host_requirements(node):
    if hasattr(node, 'entity_tpl'):
        node = node.entity_tpl
    if 'requirements' in node:
        for r in node['requirements']:
            (k, v), = r.items()
            if CONST.REQUIREMENT_HOST == k:
                if v is None:
                    r[k] = {}
                return r[k]
    return None

def get_host_node_filter(node):
    requirement = get_host_requirements(node)

    try:
        properties = requirement['node_filter']['properties'] or []
    except (TypeError,  KeyError):
        properties = []
    return properties

def get_node_from_tpl(tosca, str_node):
    return tosca.tpl['topology_template']['node_templates'][str_node]
<commit_msg>fix bug on helper function<commit_after>import logging
import os
import zipfile
from os import path

from toscaparser.prereq.csar import CSAR

class CONST:
    # DF_HOST = 'http://131.114.2.77/search'
    # DF_HOST = 'http://127.0.0.1:3000/search'
    DF_HOST = 'http://black.di.unipi.it:3000'
    DF_SEARCH = '/search'

    # CONTAINER_TYPE = 'tosker.nodes.Container'
    SOFTWARE_TYPE = 'tosker.nodes.Software'
    # IMAGE_TYPE = 'tosker.artifacts.Image'
    # DOCKERFILE_TYPE = 'tosker.artifacts.Dockerfile'

    PROPERTY_SW = 'supported_sw'
    PROPERTY_OS = 'os_distribution'
    REQUIREMENT_HOST = 'host'

    POLICY_TOP = 'top_rated'
    POLICY_SIZE = 'size'
    POLICY_USED = 'most_used'

class Logger:
    _ch = None

    @staticmethod
    def set_logger(level=logging.DEBUG):
        # global Logger._ch
        Logger._ch = logging.StreamHandler()
        Logger._ch.setLevel(level)
        formatter = logging.Formatter((
            '%(levelname) -3s %(asctime)s %(name)'
            '-3s %(funcName)'
            '-1s %(lineno) -0s: %(message)s'
        ))
        Logger._ch.setFormatter(formatter)

    @staticmethod
    def get(name, level=logging.DEBUG):
        log = logging.getLogger(name)
        log.setLevel(level)
        if Logger._ch is not None:
            log.addHandler(Logger._ch)
        return log

def unpack_csar(file_path):
    # Work around bug validation csar of toscaparser
    csar = CSAR(file_path)
    try:
        csar.validate()
    except ValueError as e:
        # _log.debug(e)
        if not str(e).startswith("The resource") or \
           not str(e).endswith("does not exist."):
            raise e

    csar.decompress()

    yaml_path = path.join(csar.temp_dir, csar.get_main_template())
    return (csar.temp_dir, yaml_path)

def pack_csar(csar_tmp_path, new_path):
    with zipfile.ZipFile(new_path, "w", zipfile.ZIP_DEFLATED) as zf:
        for dirname, subdirs, files in os.walk(csar_tmp_path):
            # zf.write(dirname, root)
            for filename in files:
                zf.write(path.join(dirname, filename),
                         path.relpath(path.join(dirname, filename),
                                      path.join(csar_tmp_path, '.')))

def get_host_requirements(node):
    if hasattr(node, 'entity_tpl'):
        node = node.entity_tpl
    if 'requirements' in node:
        for r in node['requirements']:
            (k, v), = r.items()
            if CONST.REQUIREMENT_HOST == k:
                if v is None:
                    r[k] = {}
                return r[k]
    return None

def get_host_node_filter(node):
    requirement = get_host_requirements(node)

    try:
        properties = requirement['node_filter']['properties'] or []
    except (TypeError,  KeyError):
        properties = []
    return properties

def get_node_from_tpl(tosca, str_node):
    for node in tosca.nodetemplates:
        if node.name == str_node:
            return node
    return None
    # return tosca.tpl['topology_template']['node_templates'][str_node]
<|endoftext|>


<commit_before>
__version__ = '0.1.2'

<commit_msg>0.1.3: no commit message<commit_after>
__version__ = '0.1.3'

<|endoftext|>


<commit_before>
version = (0, 2, 4)

# Add the git version if in a git tree...
import os, os.path
__path__ = os.path.realpath(os.path.dirname(__file__))
installpath = os.path.split(os.path.split(__path__)[0])[0]

# Get the git version this tree is based on
if os.path.exists(os.path.join(installpath, '.git')):
	# Read in git's 'HEAD' file which points to the correct reff to look at
	h = open(os.path.join(installpath, '.git', 'HEAD')).readline().strip()
	try:
		# Read in the ref
		ref = h.split(': ', 1)[1]
		# This file has the SHA1
		p = open(os.path.join(installpath, '.git', ref))
		del ref

		version_git = p.read().strip()
	except IndexError:
		version_git = h

	# What version are we trying to get too?
	import time
	if version[2] >= 99:
		version_target = (version[0], version[1]+1, 0)
	else:
		version_target = (version[0], version[1], version[2]+1)

	version_target_str = "%i.%i.%i" % version_target

version_str = "%i.%i.%i" % version[:3]

if __name__ == "__main__":
	import sys
	if len(sys.argv) > 1 and sys.argv[1] == '--fix':
		print """
import os, os.path
__path__ = os.path.realpath(os.path.dirname(__file__))
installpath = os.path.split(os.path.split(__path__)[0])[0]
"""
		for value in dir():
			if value.startswith('__') or value in ('installpath',):
				continue
			if isinstance(eval(value), (tuple, str, unicode)):
				exec("print '%s =', repr(%s)" % (value, value))

		print """
try:
	print version_str+'+'+version_target_str, "(git %s)" % version_git, "(installed at %s)" % installpath
except ValueError:
	print version_str, "(installed at %s)" % installpath
"""
		sys.exit(0)

	if os.path.exists(os.path.join(installpath, '.git')):
		print version_str+'+'+version_target_str, "(git %s)" % version_git, "(installed at %s)" % installpath
	else:
		print version_str, "(installed at %s)" % installpath
<commit_msg>Version bump.<commit_after>
version = (0, 2, 5)

# Add the git version if in a git tree...
import os, os.path
__path__ = os.path.realpath(os.path.dirname(__file__))
installpath = os.path.split(os.path.split(__path__)[0])[0]

# Get the git version this tree is based on
if os.path.exists(os.path.join(installpath, '.git')):
	# Read in git's 'HEAD' file which points to the correct reff to look at
	h = open(os.path.join(installpath, '.git', 'HEAD')).readline().strip()
	try:
		# Read in the ref
		ref = h.split(': ', 1)[1]
		# This file has the SHA1
		p = open(os.path.join(installpath, '.git', ref))
		del ref

		version_git = p.read().strip()
	except IndexError:
		version_git = h

	# What version are we trying to get too?
	import time
	if version[2] >= 99:
		version_target = (version[0], version[1]+1, 0)
	else:
		version_target = (version[0], version[1], version[2]+1)

	version_target_str = "%i.%i.%i" % version_target

version_str = "%i.%i.%i" % version[:3]

if __name__ == "__main__":
	import sys
	if len(sys.argv) > 1 and sys.argv[1] == '--fix':
		print """
import os, os.path
__path__ = os.path.realpath(os.path.dirname(__file__))
installpath = os.path.split(os.path.split(__path__)[0])[0]
"""
		for value in dir():
			if value.startswith('__') or value in ('installpath',):
				continue
			if isinstance(eval(value), (tuple, str, unicode)):
				exec("print '%s =', repr(%s)" % (value, value))

		print """
try:
	print version_str+'+'+version_target_str, "(git %s)" % version_git, "(installed at %s)" % installpath
except ValueError:
	print version_str, "(installed at %s)" % installpath
"""
		sys.exit(0)

	if os.path.exists(os.path.join(installpath, '.git')):
		print version_str+'+'+version_target_str, "(git %s)" % version_git, "(installed at %s)" % installpath
	else:
		print version_str, "(installed at %s)" % installpath
<|endoftext|>