<commit_before>try:
    import json
except ImportError:
    import simplejson as json

import hashlib
import logging
import optparse
import re
import socket
import time
from getpass import getpass

import requests
from pegasus.metrics import db
from repoze.lru import lru_cache

log = logging.getLogger("pegasus.metrics.loader")

def reprocess_raw_data():
    db.delete_processed_data()

    i = 0
    for data in db.each_raw_data():
        i += 1
        process_raw_data(data)

    return i

def reprocess_invalid_data():
    ids = db.get_invalid_ids()

    if len(ids) == 0:
        return 0

    db.delete_invalid_data()

    i = 0
    for data in db.each_raw_data(ids):
        i += 1
        process_raw_data(data)

    return i

@lru_cache(1024, timeout=3600)
def get_hostname_domain(ipaddr):
    if ipaddr is None:
        return None, None

    if ipaddr == "127.0.0.1":
        return "localhost", "localdomain"

    try:
        log.debug("Looking up %s" % ipaddr)
        hostname = socket.gethostbyaddr(ipaddr)[0]
        log.debug("%s is %s" % (ipaddr, hostname))

        # Count the number of dots in the hostname
        dots = len([x for x in hostname if x == "."])

        # This is a list of tlds for which a second-level domain
        # has no meaning. This is not all of them, just the ones
        # we are likely to encounter.
        tlds_wo_2nd_level = [".uk", ".il", ".nz", ".au", ".edu.pk", ".ac.in", ".edu.in"]

        # Require 2 dots for tlds without second-level, 1 otherwise
        if any(hostname.endswith(tld) for tld in tlds_wo_2nd_level):
            log.debug("Requiring 2 dots for host %s" % hostname)
            mindots = 2
        else:
            mindots = 1

        if dots <= mindots:
            domain = hostname
        else:
            # The domain is everything after the first dot
            domain = hostname[hostname.find(".") + 1 :]

        return hostname, domain
    except Exception as e:
        log.warning("%s: %s" % (e, ipaddr))
        return ipaddr, ipaddr

def process_raw_data(data):
    try:
        # Decode mappings
        for key in data:
            if type(data[key]) == str:
                data[key] = data[key].encode("utf-8")
        # Get the hostname and domain
        # print data["id"]
        ipaddr = data["remote_addr"]
        hostname, domain = get_hostname_domain(ipaddr)
        data["hostname"] = hostname
        data["domain"] = domain

        # Process metrics according to type
        client = data["client"]
        dtype = data["type"]
        if (client, dtype) == ("pegasus-plan", "metrics"):
            process_planner_metrics(data)
        elif (client, dtype) == ("pegasus-plan", "error"):
            process_planner_error(data)
            # Planner errors also contain planner metrics
            process_planner_metrics(data)
        elif (client, dtype) == ("condor_dagman", "metrics"):
            process_dagman_metrics(data)
        elif dtype == "download":
            process_download(data)
        else:
            error = "Unknown client/data type: %s/%s" % (client, dtype)
            log.warn(error)
            db.store_invalid_data(data["id"], error)
    except Exception as e:
        log.warn(data["id"])
        log.exception(e)
        db.store_invalid_data(data["id"], repr(e))

def _geolocate(host, ipaddr, timeout):
    try:
        r = requests.get("http://%s/json/%s" % (host, ipaddr), timeout=timeout)

        if r.status_code < 200 or r.status_code >= 300:
            log.warn(
                "Request to %s for %s returned failure: %s"
                % (host, ipaddr, r.status_code)
            )
            return None

        r.encoding = "utf-8"
        location = json.loads(r.text)
        for key in location:
            if type(location[key]) == str:
                location[key] = location[key].encode("utf-8")
        return location
    except Exception as e:
        log.exception(e)
        log.warn("Error getting location for %s from %s" % (ipaddr, host))

    return None

def geolocate(ipaddr):
    if (
        ipaddr.startswith("192.168.")
        or ipaddr.startswith("10.")
        or ipaddr == "127.0.0.1"
    ):
        log.warning("Skipping local address %s" % ipaddr)
        return None

    if ipaddr.startswith("172."):
        i = ipaddr.split(".")
        j = int(i[1])
        if 16 <= j <= 31:
            log.warning("Skipping local address %s" % ipaddr)
            return None

    hosts = [("gaul.isi.edu:8192", 0.5), ("freegeoip.net", 30)]
    for host, timeout in hosts:
        location = _geolocate(host, ipaddr, timeout)
        if location is not None:
            return location

    log.error("Unable to get location for %s" % ipaddr)
    return None

def process_location(ipaddr):
    if ipaddr is None or ipaddr == "":
        return

    if db.get_location(ipaddr):
        return

    location = geolocate(ipaddr)
    if location is not None:
        db.store_location(location)

def process_download(data):
    def nullify(key):
        if not key in data:
            data[key] = None
            return

        if len(data[key].strip()) == 0:
            data[key] = None

    # Convert missing and empty mappings to None
    nullify("name")
    nullify("email")
    nullify("organization")
    nullify("app_domain")
    nullify("app_description")
    nullify("howheard")
    nullify("howhelp")
    nullify("oldfeatures")
    nullify("newfeatures")

    # If the filename has a version string, then extract it
    # Always extract the longest string that looks like a version
    filename = data["filename"]
    ver = None
    for m in re.finditer(r"\d+(\.\d+)+", filename):
        newver = m.group(0)
        if ver is None or len(newver) > len(ver):
            ver = newver
    data["version"] = ver

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    db.store_download(data)

def process_planner_metrics(data):
    if "wf_uuid" not in data:
        raise Exception("wf_uuid missing")

    # Remove the nested structure the planner sends
    metrics = data["wf_metrics"]
    del data["wf_metrics"]
    data.update(metrics)
    # Change start_time and end_time into timestamps if they
    # are using the old string formats
    datefmt = "%b %d, %Y %H:%M:%S %p"
    start_time = data["start_time"]
    if isinstance(start_time, str):
        if start_time[0] in "0123456789":
            data["start_time"] = float(start_time)
        else:
            log.debug("Using old string format for planner start_time")
            ts = time.mktime(time.strptime(start_time, datefmt))
            data["start_time"] = ts

    end_time = data["end_time"]
    if isinstance(end_time, str):
        if end_time[0] in "0123456789":
            data["end_time"] = float(end_time)
        else:
            log.debug("Using old string format for planner end_time")
            ts = time.mktime(time.strptime(end_time, datefmt))
            data["end_time"] = ts

    if "data_config" not in data:
        data["data_config"] = None

    if "app_metrics" in data:
        app_metrics = data["app_metrics"]
        if "name" in app_metrics:
            data["application"] = app_metrics["name"]

    if "application" not in data:
        data["application"] = None

    if "dax_input_files" not in data:
        data["dax_input_files"] = None

    if "dax_inter_files" not in data:
        data["dax_inter_files"] = None

    if "dax_output_files" not in data:
        data["dax_output_files"] = None

    if "dax_total_files" not in data:
        data["dax_total_files"] = None

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    if "uses_pmc" not in data:
        data["uses_pmc"] = None

    if "planner_args" not in data:
        data["planner_args"] = None

    if "deleted_tasks" not in data:
        data["deleted_tasks"] = None

    data["dax_api"] = data.get("wf_api", data.get("dax_api", None))

    db.store_planner_metrics(data)

def process_dagman_metrics(data):
    if "parent_dagman_id" not in data or len(data["parent_dagman_id"]) == 0:
        data["parent_dagman_id"] = None

    if "planner" not in data or len(data["planner"]) == 0:
        data["planner"] = None

    if "planner_version" not in data or len(data["planner_version"]) == 0:
        data["planner_version"] = None

    if "total_job_time" not in data:
        data["total_job_time"] = None

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    db.store_dagman_metrics(data)

def hash_error(error):
    # XXX We assume it is a Java stacktrace for now
    # When we get more data we can revise our processing

    # Use MD5 to create a hash
    md = hashlib.md5()

    # Each stack trace has several lines
    lines = error.split("\n")

    # Line 0 should start with the exception type followed by a colon
    # If there is no colon, then just assume there was no message
    colon = lines[0].find(":")
    if colon > 0:
        md.update(lines[0][0:colon])
    else:
        md.update(lines[0])

    # The line where the exception was thrown should be the first one starting with "at"
    location_found = False
    for l in lines[1:]:
        if l.lstrip().startswith("at "):
            md.update(l)
            location_found = True
            break
    if not location_found:
        log.warn("Stack trace does not appear to have a location:\n%s" % error)

    # If the exception was caused by another, then take that into account
    for i in range(0, len(lines)):
        if lines[i].startswith("Caused by: "):
            cause = lines[i].split(":")[1]
            md.update(cause)
            if i + 1 < len(lines):
                md.update(lines[i + 1])
            else:
                log.warn("No location for cause in stack trace:\n%s" % error)

    errhash = md.hexdigest()

    return errhash

def process_planner_error(data):
    objid = data["id"]
    message = data["error"]
    errhash = hash_error(message)

    error = {"id": objid, "error": message, "hash": errhash}

    db.store_planner_errors(error)
<commit_msg>call  on bytes to converyt to  before comparing the values with str literals.<commit_after>try:
    import json
except ImportError:
    import simplejson as json

import hashlib
import logging
import optparse
import re
import socket
import time
from getpass import getpass

import requests
from pegasus.metrics import db
from repoze.lru import lru_cache

log = logging.getLogger("pegasus.metrics.loader")

def reprocess_raw_data():
    db.delete_processed_data()

    i = 0
    for data in db.each_raw_data():
        i += 1
        process_raw_data(data)

    return i

def reprocess_invalid_data():
    ids = db.get_invalid_ids()

    if len(ids) == 0:
        return 0

    db.delete_invalid_data()

    i = 0
    for data in db.each_raw_data(ids):
        i += 1
        process_raw_data(data)

    return i

@lru_cache(1024, timeout=3600)
def get_hostname_domain(ipaddr):
    if ipaddr is None:
        return None, None

    if ipaddr == "127.0.0.1":
        return "localhost", "localdomain"

    try:
        log.debug("Looking up %s" % ipaddr)
        hostname = socket.gethostbyaddr(ipaddr)[0]
        log.debug("%s is %s" % (ipaddr, hostname))

        # Count the number of dots in the hostname
        dots = len([x for x in hostname if x == "."])

        # This is a list of tlds for which a second-level domain
        # has no meaning. This is not all of them, just the ones
        # we are likely to encounter.
        tlds_wo_2nd_level = [".uk", ".il", ".nz", ".au", ".edu.pk", ".ac.in", ".edu.in"]

        # Require 2 dots for tlds without second-level, 1 otherwise
        if any(hostname.endswith(tld) for tld in tlds_wo_2nd_level):
            log.debug("Requiring 2 dots for host %s" % hostname)
            mindots = 2
        else:
            mindots = 1

        if dots <= mindots:
            domain = hostname
        else:
            # The domain is everything after the first dot
            domain = hostname[hostname.find(".") + 1 :]

        return hostname, domain
    except Exception as e:
        log.warning("%s: %s" % (e, ipaddr))
        return ipaddr, ipaddr

def process_raw_data(data):
    try:
        # Decode mappings
        for key in data:
            if type(data[key]) == str:
                data[key] = data[key].encode("utf-8")
        # Get the hostname and domain
        # print data["id"]
        ipaddr = data["remote_addr"]
        hostname, domain = get_hostname_domain(ipaddr)
        data["hostname"] = hostname
        data["domain"] = domain

        # Process metrics according to type
        client = data["client"].decode()
        dtype = data["type"].decode()
        if (client, dtype) == ("pegasus-plan", "metrics"):
            process_planner_metrics(data)
        elif (client, dtype) == ("pegasus-plan", "error"):
            process_planner_error(data)
            # Planner errors also contain planner metrics
            process_planner_metrics(data)
        elif (client, dtype) == ("condor_dagman", "metrics"):
            process_dagman_metrics(data)
        elif dtype == "download":
            process_download(data)
        else:
            error = "Unknown client/data type: %s/%s" % (client, dtype)
            log.warn(error)
            db.store_invalid_data(data["id"], error)
    except Exception as e:
        log.warn(data["id"])
        log.exception(e)
        db.store_invalid_data(data["id"], repr(e))

def _geolocate(host, ipaddr, timeout):
    try:
        r = requests.get("http://%s/json/%s" % (host, ipaddr), timeout=timeout)

        if r.status_code < 200 or r.status_code >= 300:
            log.warn(
                "Request to %s for %s returned failure: %s"
                % (host, ipaddr, r.status_code)
            )
            return None

        r.encoding = "utf-8"
        location = json.loads(r.text)
        for key in location:
            if type(location[key]) == str:
                location[key] = location[key].encode("utf-8")
        return location
    except Exception as e:
        log.exception(e)
        log.warn("Error getting location for %s from %s" % (ipaddr, host))

    return None

def geolocate(ipaddr):
    ipaddr = ipaddr.decode()
    if (
        ipaddr.startswith("192.168.")
        or ipaddr.startswith("10.")
        or ipaddr == "127.0.0.1"
    ):
        log.warning("Skipping local address %s" % ipaddr)
        return None

    if ipaddr.startswith("172."):
        i = ipaddr.split(".")
        j = int(i[1])
        if 16 <= j <= 31:
            log.warning("Skipping local address %s" % ipaddr)
            return None

    hosts = [("gaul.isi.edu:8192", 0.5), ("freegeoip.net", 30)]
    for host, timeout in hosts:
        location = _geolocate(host, ipaddr, timeout)
        if location is not None:
            return location

    log.error("Unable to get location for %s" % ipaddr)
    return None

def process_location(ipaddr):
    if ipaddr is None or ipaddr == "":
        return

    if db.get_location(ipaddr):
        return

    location = geolocate(ipaddr)
    if location is not None:
        db.store_location(location)

def process_download(data):
    def nullify(key):
        if not key in data:
            data[key] = None
            return

        if len(data[key].strip()) == 0:
            data[key] = None

    # Convert missing and empty mappings to None
    nullify("name")
    nullify("email")
    nullify("organization")
    nullify("app_domain")
    nullify("app_description")
    nullify("howheard")
    nullify("howhelp")
    nullify("oldfeatures")
    nullify("newfeatures")

    # If the filename has a version string, then extract it
    # Always extract the longest string that looks like a version
    filename = data["filename"]
    ver = None
    for m in re.finditer(r"\d+(\.\d+)+", filename):
        newver = m.group(0)
        if ver is None or len(newver) > len(ver):
            ver = newver
    data["version"] = ver

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    db.store_download(data)

def process_planner_metrics(data):
    if "wf_uuid" not in data:
        raise Exception("wf_uuid missing")

    # Remove the nested structure the planner sends
    metrics = data["wf_metrics"]
    del data["wf_metrics"]
    data.update(metrics)
    # Change start_time and end_time into timestamps if they
    # are using the old string formats
    datefmt = "%b %d, %Y %H:%M:%S %p"
    start_time = data["start_time"]
    if isinstance(start_time, str):
        if start_time[0] in "0123456789":
            data["start_time"] = float(start_time)
        else:
            log.debug("Using old string format for planner start_time")
            ts = time.mktime(time.strptime(start_time, datefmt))
            data["start_time"] = ts

    end_time = data["end_time"]
    if isinstance(end_time, str):
        if end_time[0] in "0123456789":
            data["end_time"] = float(end_time)
        else:
            log.debug("Using old string format for planner end_time")
            ts = time.mktime(time.strptime(end_time, datefmt))
            data["end_time"] = ts

    if "data_config" not in data:
        data["data_config"] = None

    if "app_metrics" in data:
        app_metrics = data["app_metrics"]
        if "name" in app_metrics:
            data["application"] = app_metrics["name"]

    if "application" not in data:
        data["application"] = None

    if "dax_input_files" not in data:
        data["dax_input_files"] = None

    if "dax_inter_files" not in data:
        data["dax_inter_files"] = None

    if "dax_output_files" not in data:
        data["dax_output_files"] = None

    if "dax_total_files" not in data:
        data["dax_total_files"] = None

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    if "uses_pmc" not in data:
        data["uses_pmc"] = None

    if "planner_args" not in data:
        data["planner_args"] = None

    if "deleted_tasks" not in data:
        data["deleted_tasks"] = None

    data["dax_api"] = data.get("wf_api", data.get("dax_api", None))

    db.store_planner_metrics(data)

def process_dagman_metrics(data):
    if "parent_dagman_id" not in data or len(data["parent_dagman_id"]) == 0:
        data["parent_dagman_id"] = None

    if "planner" not in data or len(data["planner"]) == 0:
        data["planner"] = None

    if "planner_version" not in data or len(data["planner_version"]) == 0:
        data["planner_version"] = None

    if "total_job_time" not in data:
        data["total_job_time"] = None

    if "remote_addr" in data:
        process_location(data["remote_addr"])

    db.store_dagman_metrics(data)

def hash_error(error):
    # XXX We assume it is a Java stacktrace for now
    # When we get more data we can revise our processing

    # Use MD5 to create a hash
    md = hashlib.md5()

    # Each stack trace has several lines
    lines = error.split("\n")

    # Line 0 should start with the exception type followed by a colon
    # If there is no colon, then just assume there was no message
    colon = lines[0].find(":")
    if colon > 0:
        md.update(lines[0][0:colon])
    else:
        md.update(lines[0])

    # The line where the exception was thrown should be the first one starting with "at"
    location_found = False
    for l in lines[1:]:
        if l.lstrip().startswith("at "):
            md.update(l)
            location_found = True
            break
    if not location_found:
        log.warn("Stack trace does not appear to have a location:\n%s" % error)

    # If the exception was caused by another, then take that into account
    for i in range(0, len(lines)):
        if lines[i].startswith("Caused by: "):
            cause = lines[i].split(":")[1]
            md.update(cause)
            if i + 1 < len(lines):
                md.update(lines[i + 1])
            else:
                log.warn("No location for cause in stack trace:\n%s" % error)

    errhash = md.hexdigest()

    return errhash

def process_planner_error(data):
    objid = data["id"]
    message = data["error"]
    errhash = hash_error(message)

    error = {"id": objid, "error": message, "hash": errhash}

    db.store_planner_errors(error)
<|endoftext|>


<commit_before>from kivy.uix.screenmanager import Screen

class HelpScreen(Screen):
    def on_pre_enter(self):
        self.set_text()

<commit_msg>Dependencies for help screen in .kv<commit_after>from kivy.uix.screenmanager import Screen

from pentai.gui.spacer import *
from pentai.gui.scrollable_label import *

class HelpScreen(Screen):
    def on_pre_enter(self):
        self.set_text()

<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import

import collections
import datetime
import os
import platform
import re
import socket
import subprocess
import sys
import time
try:
    import resource
except ImportError:
    resource = None

import six
try:
    # Optional dependency
    import psutil
except ImportError:
    psutil = None

import perf
from perf._utils import (format_timedelta, format_cpu_list,
                         parse_cpu_list,
                         get_isolated_cpus, MS_WINDOWS)
if MS_WINDOWS:
    from perf._win_memory import check_tracking_memory, get_peak_pagefile_usage

def normalize_text(text):
    text = str(text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()

def collect_python_metadata(metadata):
    # Implementation
    impl = perf.python_implementation()
    metadata['python_implementation'] = impl

    # Version
    version = platform.python_version()

    match = re.search(r'\[(PyPy [^ ]+)', sys.version)
    if match:
        version = '%s (Python %s)' % (match.group(1), version)

    bits = platform.architecture()[0]
    if bits:
        if bits == '64bit':
            bits = '64-bit'
        elif bits == '32bit':
            bits = '32-bit'
        version = '%s (%s)' % (version, bits)

    # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
    match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
    if match:
        revision = match.group(1)
    else:
        # 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
        # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
        match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
        if match:
            revision = match.group(1)
        else:
            revision = None
    if revision:
        version = '%s revision %s' % (version, revision)
    metadata['python_version'] = version

    if sys.executable:
        metadata['python_executable'] = sys.executable

    # Before PEP 393 (Python 3.3)
    if sys.version_info < (3, 3):
        if sys.maxunicode == 0xffff:
            unicode_impl = 'UTF-16'
        else:
            unicode_impl = 'UCS-4'
        metadata['python_unicode'] = unicode_impl

    # timer
    if (hasattr(time, 'perf_counter')
       and perf.perf_counter == time.perf_counter):

        info = time.get_clock_info('perf_counter')
        metadata['timer'] = ('%s, resolution: %s'
                             % (info.implementation,
                                format_timedelta(info.resolution)))
    elif perf.perf_counter == time.clock:
        metadata['timer'] = 'time.clock()'
    elif perf.perf_counter == time.time:
        metadata['timer'] = 'time.time()'

    # PYTHONHASHSEED
    if os.environ.get('PYTHONHASHSEED'):
        hash_seed = os.environ['PYTHONHASHSEED']
        try:
            if hash_seed != "random":
                hash_seed = int(hash_seed)
        except ValueError:
            pass
        else:
            metadata['python_hash_seed'] = hash_seed

    # CFLAGS
    try:
        import sysconfig
    except ImportError:
        pass
    else:
        cflags = sysconfig.get_config_var('CFLAGS')
        if cflags:
            cflags = normalize_text(cflags)
            metadata['python_cflags'] = cflags

def open_text(path):
    if six.PY3:
        return open(path, encoding="utf-8")
    else:
        return open(path)

def first_line(path, default=None):
    try:
        fp = open_text(path)
        try:
            line = fp.readline()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
        return line.rstrip()
    except IOError:
        if default is not None:
            return default
        raise

def read_proc(path):
    path = os.path.join('/proc', path)
    try:
        fp = open_text(path)
        try:
            for line in fp:
                yield line.rstrip()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
    except (OSError, IOError):
        return

def sys_path(path):
    return os.path.join("/sys", path)

def collect_linux_metadata(metadata):
    # ASLR
    for line in read_proc('sys/kernel/randomize_va_space'):
        if line == '0':
            metadata['aslr'] = 'No randomization'
        elif line == '1':
            metadata['aslr'] = 'Conservative randomization'
        elif line == '2':
            metadata['aslr'] = 'Full randomization'
        break

def get_cpu_affinity():
    if hasattr(os, 'sched_getaffinity'):
        return os.sched_getaffinity(0)

    if psutil is not None:
        proc = psutil.Process()
        # cpu_affinity() is only available on Linux, Windows and FreeBSD
        if hasattr(proc, 'cpu_affinity'):
            return proc.cpu_affinity()

    return None

def get_logical_cpu_count():
    if psutil is not None:
        # Number of logical CPUs
        cpu_count = psutil.cpu_count()
    elif hasattr(os, 'cpu_count'):
        # Python 3.4
        cpu_count = os.cpu_count()
    else:
        cpu_count = None
        try:
            import multiprocessing
        except ImportError:
            pass
        else:
            try:
                cpu_count = multiprocessing.cpu_count()
            except NotImplementedError:
                pass
    if cpu_count is not None and cpu_count < 1:
        return None
    return cpu_count

def collect_system_metadata(metadata):
    metadata['platform'] = platform.platform(True, False)
    if sys.platform.startswith('linux'):
        collect_linux_metadata(metadata)

    # on linux, load average over 1 minute
    for line in read_proc("loadavg"):
        fields = line.split()
        loadavg = fields[0]
        metadata['load_avg_1min'] = float(loadavg)

        if len(fields) >= 4 and '/' in fields[3]:
            runnable_threads = fields[3].split('/', 1)[0]
            runnable_threads = int(runnable_threads)
            metadata['runnable_threads'] = runnable_threads

    # Hostname
    hostname = socket.gethostname()
    if hostname:
        metadata['hostname'] = hostname

def collect_memory_metadata(metadata):
    if resource is not None:
        usage = resource.getrusage(resource.RUSAGE_SELF)
        max_rss = usage.ru_maxrss
        if max_rss:
            metadata['mem_max_rss'] = max_rss * 1024

    # On Windows, use GetProcessMemoryInfo() if available
    if MS_WINDOWS and not check_tracking_memory():
        usage = get_peak_pagefile_usage()
        if usage:
            metadata['mem_peak_pagefile_usage'] = usage

def get_cpu_boost(cpu):
    if not get_cpu_boost.working:
        return

    env = dict(os.environ, LC_ALL='C')
    args = ['cpupower', '-c', str(cpu), 'frequency-info']
    try:
        proc = subprocess.Popen(args,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                universal_newlines=True,
                                env=env)
        stdout = proc.communicate()[0]
        if proc.returncode != 0:
            # if the command failed once, never try it again
            # (consider that the command is not installed or does not work)
            get_cpu_boost.working = False
            return None
    except OSError:
        get_cpu_boost.working = False
        return None

    # cpupower doesn't seem to work on PPC64LE:
    # see https://github.com/haypo/perf/issues/11
    if 'no or unknown cpufreq driver is active on this CPU' in stdout:
        get_cpu_boost.working = False
        return None

    boost = False
    for line in stdout.splitlines():
        if boost:
            if 'Supported:' in line:
                value = line.split(':', 1)[-1].strip()
                if value == 'no':
                    return False
                if value == 'yes':
                    return True
                raise ValueError("unable to parse: %r" % line)
        elif 'boost state support' in line:
            boost = True

    raise ValueError("unable to parse cpupower output: %r" % stdout)
get_cpu_boost.working = True

def format_cpu_infos(infos):
    groups = collections.defaultdict(list)
    for cpu, info in infos.items():
        groups[info].append(cpu)

    items = [(cpus, info) for info, cpus in  groups.items()]
    items.sort()
    text = []
    for cpus, info in items:
        cpus = format_cpu_list(cpus)
        text.append('%s=%s' % (cpus, info))
    return ', '.join(text)

def collect_cpu_freq(metadata, cpus):
    # Parse /proc/cpuinfo: search for 'cpu MHz' (Intel) or 'clock' (Power8)
    cpu_set = set(cpus)
    cpu_freq = {}
    cpu = None
    for line in read_proc('cpuinfo'):
        if line.startswith('processor'):
            value = line.split(':', 1)[-1].strip()
            cpu = int(value)
            if cpu not in cpu_set:
                # skip this CPU
                cpu = None

        elif line.startswith('cpu MHz') and cpu is not None:
            # Intel: 'cpu MHz : 1261.613'
            mhz = line.split(':', 1)[-1].strip()
            mhz = float(mhz)
            mhz = int(round(mhz))
            cpu_freq[cpu] = '%s MHz' % mhz

        elif line.startswith('clock') and line.endswith('MHz') and cpu is not None:
            # Power8: 'clock : 3425.000000MHz'
            mhz = line[:-3].split(':', 1)[-1].strip()
            mhz = float(mhz)
            mhz = int(round(mhz))
            cpu_freq[cpu] = '%s MHz' % mhz

    if not cpu_freq:
        return

    metadata['cpu_freq'] = format_cpu_infos(cpu_freq)

def get_cpu_config(cpu):
    sys_cpu_path = sys_path("devices/system/cpu")
    info = []

    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_driver" % cpu)
    scaling_driver = first_line(path, default='')
    if scaling_driver:
        info.append('driver:%s' % scaling_driver)

    if scaling_driver == 'intel_pstate':
        path = os.path.join(sys_cpu_path, "intel_pstate/no_turbo")
        no_turbo = first_line(path, default='')
        if no_turbo == '1':
            info.append('intel_pstate:no turbo')
        elif no_turbo == '0':
            info.append('intel_pstate:turbo')
    else:
        boost = get_cpu_boost(cpu)
        if boost is not None:
            if boost:
                info.append('boost:supported')
            else:
                info.append('boost:not suppported')

    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_governor" % cpu)
    scaling_governor = first_line(path, default='')
    if scaling_governor:
        info.append('governor:%s' % scaling_governor)

    return info

def collect_cpu_config(metadata, cpus):
    nohz_full = first_line('/sys/devices/system/cpu/nohz_full')
    if nohz_full:
        nohz_full = parse_cpu_list(nohz_full)

    isolated = get_isolated_cpus()
    if isolated:
        isolated = set(isolated)

    configs = {}
    for cpu in cpus:
        config = get_cpu_config(cpu)
        if nohz_full and cpu in nohz_full:
            config.append('nohz_full')
        if isolated and cpu in isolated:
            config.append('isolated')
        if config:
            configs[cpu] = ', '.join(config)
    if not configs:
        return
    metadata['cpu_config'] = format_cpu_infos(configs)

def get_cpu_temperature(path, cpu_temp):
    hwmon_name = first_line(os.path.join(path, 'name'), default='')
    if not hwmon_name.startswith('coretemp'):
        return

    index = 1
    while True:
        template = os.path.join(path, "temp%s_%%s" % index)

        try:
            temp_label = first_line(template % 'label')
        except IOError:
            break

        temp_input = first_line(template % 'input')
        temp_input = float(temp_input) / 1000
        # FIXME: On Python 2, u"%.0f\xb0C" introduces unicode errors if the
        # locale encoding is ASCII
        temp_input = "%.0f C" % temp_input

        item = '%s:%s=%s' % (hwmon_name, temp_label, temp_input)
        cpu_temp.append(item)

        index += 1

def collect_cpu_temperatures(metadata):
    path = sys_path("class/hwmon")
    try:
        names = os.listdir(path)
    except OSError:
        return None

    cpu_temp = []
    for name in names:
        hwmon = os.path.join(path, name)
        get_cpu_temperature(hwmon, cpu_temp)
    if not cpu_temp:
        return None

    metadata['cpu_temp'] = ', '.join(cpu_temp)

def collect_cpu_affinity(metadata, cpu_affinity, cpu_count):
    if not cpu_affinity:
        return
    if not cpu_count:
        return

    # CPU affinity
    if set(cpu_affinity) == set(range(cpu_count)):
        return

    metadata['cpu_affinity'] = format_cpu_list(cpu_affinity)

def collect_cpu_model(metadata):
    for line in read_proc("cpuinfo"):
        if line.startswith('model name'):
            model_name = line.split(':', 1)[1].strip()
            if model_name:
                metadata['cpu_model_name'] = model_name
            break

        if line.startswith('machine'):
            machine = line.split(':', 1)[1].strip()
            if machine:
                metadata['cpu_machine'] = machine
            break

def collect_cpu_metadata(metadata):
    collect_cpu_model(metadata)

    # CPU count
    cpu_count = get_logical_cpu_count()
    if cpu_count:
        metadata['cpu_count'] = cpu_count

    cpu_affinity = get_cpu_affinity()
    collect_cpu_affinity(metadata, cpu_affinity, cpu_count)

    all_cpus = cpu_affinity
    if not all_cpus and cpu_count:
        all_cpus = tuple(range(cpu_count))

    if all_cpus:
        collect_cpu_freq(metadata, all_cpus)
        collect_cpu_config(metadata, all_cpus)

    collect_cpu_temperatures(metadata)

def collect_metadata(metadata):
    metadata['perf_version'] = perf.__version__

    date = datetime.datetime.now().isoformat()
    # FIXME: Move date to a regular run attribute with type datetime.datetime?
    metadata['date'] = date.split('.', 1)[0]

    collect_python_metadata(metadata)
    collect_system_metadata(metadata)
    collect_memory_metadata(metadata)
    collect_cpu_metadata(metadata)

    # Note: Don't collect VmPeak of /proc/self/status on Linux because it is
    # not accurate. See perf._memory for more accurate memory metrics.
<commit_msg>Fix cpu_config metadata<commit_after>from __future__ import division, print_function, absolute_import

import collections
import datetime
import os
import platform
import re
import socket
import subprocess
import sys
import time
try:
    import resource
except ImportError:
    resource = None

import six
try:
    # Optional dependency
    import psutil
except ImportError:
    psutil = None

import perf
from perf._utils import (format_timedelta, format_cpu_list,
                         parse_cpu_list,
                         get_isolated_cpus, MS_WINDOWS)
if MS_WINDOWS:
    from perf._win_memory import check_tracking_memory, get_peak_pagefile_usage

def normalize_text(text):
    text = str(text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()

def collect_python_metadata(metadata):
    # Implementation
    impl = perf.python_implementation()
    metadata['python_implementation'] = impl

    # Version
    version = platform.python_version()

    match = re.search(r'\[(PyPy [^ ]+)', sys.version)
    if match:
        version = '%s (Python %s)' % (match.group(1), version)

    bits = platform.architecture()[0]
    if bits:
        if bits == '64bit':
            bits = '64-bit'
        elif bits == '32bit':
            bits = '32-bit'
        version = '%s (%s)' % (version, bits)

    # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
    match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
    if match:
        revision = match.group(1)
    else:
        # 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
        # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
        match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
        if match:
            revision = match.group(1)
        else:
            revision = None
    if revision:
        version = '%s revision %s' % (version, revision)
    metadata['python_version'] = version

    if sys.executable:
        metadata['python_executable'] = sys.executable

    # Before PEP 393 (Python 3.3)
    if sys.version_info < (3, 3):
        if sys.maxunicode == 0xffff:
            unicode_impl = 'UTF-16'
        else:
            unicode_impl = 'UCS-4'
        metadata['python_unicode'] = unicode_impl

    # timer
    if (hasattr(time, 'perf_counter')
       and perf.perf_counter == time.perf_counter):

        info = time.get_clock_info('perf_counter')
        metadata['timer'] = ('%s, resolution: %s'
                             % (info.implementation,
                                format_timedelta(info.resolution)))
    elif perf.perf_counter == time.clock:
        metadata['timer'] = 'time.clock()'
    elif perf.perf_counter == time.time:
        metadata['timer'] = 'time.time()'

    # PYTHONHASHSEED
    if os.environ.get('PYTHONHASHSEED'):
        hash_seed = os.environ['PYTHONHASHSEED']
        try:
            if hash_seed != "random":
                hash_seed = int(hash_seed)
        except ValueError:
            pass
        else:
            metadata['python_hash_seed'] = hash_seed

    # CFLAGS
    try:
        import sysconfig
    except ImportError:
        pass
    else:
        cflags = sysconfig.get_config_var('CFLAGS')
        if cflags:
            cflags = normalize_text(cflags)
            metadata['python_cflags'] = cflags

def open_text(path):
    if six.PY3:
        return open(path, encoding="utf-8")
    else:
        return open(path)

def first_line(path, default=None):
    try:
        fp = open_text(path)
        try:
            line = fp.readline()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
        return line.rstrip()
    except IOError:
        if default is not None:
            return default
        raise

def read_proc(path):
    path = os.path.join('/proc', path)
    try:
        fp = open_text(path)
        try:
            for line in fp:
                yield line.rstrip()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
    except (OSError, IOError):
        return

def sys_path(path):
    return os.path.join("/sys", path)

def collect_linux_metadata(metadata):
    # ASLR
    for line in read_proc('sys/kernel/randomize_va_space'):
        if line == '0':
            metadata['aslr'] = 'No randomization'
        elif line == '1':
            metadata['aslr'] = 'Conservative randomization'
        elif line == '2':
            metadata['aslr'] = 'Full randomization'
        break

def get_cpu_affinity():
    if hasattr(os, 'sched_getaffinity'):
        return os.sched_getaffinity(0)

    if psutil is not None:
        proc = psutil.Process()
        # cpu_affinity() is only available on Linux, Windows and FreeBSD
        if hasattr(proc, 'cpu_affinity'):
            return proc.cpu_affinity()

    return None

def get_logical_cpu_count():
    if psutil is not None:
        # Number of logical CPUs
        cpu_count = psutil.cpu_count()
    elif hasattr(os, 'cpu_count'):
        # Python 3.4
        cpu_count = os.cpu_count()
    else:
        cpu_count = None
        try:
            import multiprocessing
        except ImportError:
            pass
        else:
            try:
                cpu_count = multiprocessing.cpu_count()
            except NotImplementedError:
                pass
    if cpu_count is not None and cpu_count < 1:
        return None
    return cpu_count

def collect_system_metadata(metadata):
    metadata['platform'] = platform.platform(True, False)
    if sys.platform.startswith('linux'):
        collect_linux_metadata(metadata)

    # on linux, load average over 1 minute
    for line in read_proc("loadavg"):
        fields = line.split()
        loadavg = fields[0]
        metadata['load_avg_1min'] = float(loadavg)

        if len(fields) >= 4 and '/' in fields[3]:
            runnable_threads = fields[3].split('/', 1)[0]
            runnable_threads = int(runnable_threads)
            metadata['runnable_threads'] = runnable_threads

    # Hostname
    hostname = socket.gethostname()
    if hostname:
        metadata['hostname'] = hostname

def collect_memory_metadata(metadata):
    if resource is not None:
        usage = resource.getrusage(resource.RUSAGE_SELF)
        max_rss = usage.ru_maxrss
        if max_rss:
            metadata['mem_max_rss'] = max_rss * 1024

    # On Windows, use GetProcessMemoryInfo() if available
    if MS_WINDOWS and not check_tracking_memory():
        usage = get_peak_pagefile_usage()
        if usage:
            metadata['mem_peak_pagefile_usage'] = usage

def get_cpu_boost(cpu):
    if not get_cpu_boost.working:
        return

    env = dict(os.environ, LC_ALL='C')
    args = ['cpupower', '-c', str(cpu), 'frequency-info']
    try:
        proc = subprocess.Popen(args,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                universal_newlines=True,
                                env=env)
        stdout = proc.communicate()[0]
        if proc.returncode != 0:
            # if the command failed once, never try it again
            # (consider that the command is not installed or does not work)
            get_cpu_boost.working = False
            return None
    except OSError:
        get_cpu_boost.working = False
        return None

    # cpupower doesn't seem to work on PPC64LE:
    # see https://github.com/haypo/perf/issues/11
    if 'no or unknown cpufreq driver is active on this CPU' in stdout:
        get_cpu_boost.working = False
        return None

    boost = False
    for line in stdout.splitlines():
        if boost:
            if 'Supported:' in line:
                value = line.split(':', 1)[-1].strip()
                if value == 'no':
                    return False
                if value == 'yes':
                    return True
                raise ValueError("unable to parse: %r" % line)
        elif 'boost state support' in line:
            boost = True

    raise ValueError("unable to parse cpupower output: %r" % stdout)
get_cpu_boost.working = True

def format_cpu_infos(infos):
    groups = collections.defaultdict(list)
    for cpu, info in infos.items():
        groups[info].append(cpu)

    items = [(cpus, info) for info, cpus in  groups.items()]
    items.sort()
    text = []
    for cpus, info in items:
        cpus = format_cpu_list(cpus)
        text.append('%s=%s' % (cpus, info))
    return ', '.join(text)

def collect_cpu_freq(metadata, cpus):
    # Parse /proc/cpuinfo: search for 'cpu MHz' (Intel) or 'clock' (Power8)
    cpu_set = set(cpus)
    cpu_freq = {}
    cpu = None
    for line in read_proc('cpuinfo'):
        if line.startswith('processor'):
            value = line.split(':', 1)[-1].strip()
            cpu = int(value)
            if cpu not in cpu_set:
                # skip this CPU
                cpu = None

        elif line.startswith('cpu MHz') and cpu is not None:
            # Intel: 'cpu MHz : 1261.613'
            mhz = line.split(':', 1)[-1].strip()
            mhz = float(mhz)
            mhz = int(round(mhz))
            cpu_freq[cpu] = '%s MHz' % mhz

        elif line.startswith('clock') and line.endswith('MHz') and cpu is not None:
            # Power8: 'clock : 3425.000000MHz'
            mhz = line[:-3].split(':', 1)[-1].strip()
            mhz = float(mhz)
            mhz = int(round(mhz))
            cpu_freq[cpu] = '%s MHz' % mhz

    if not cpu_freq:
        return

    metadata['cpu_freq'] = format_cpu_infos(cpu_freq)

def get_cpu_config(cpu):
    sys_cpu_path = sys_path("devices/system/cpu")
    info = []

    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_driver" % cpu)
    scaling_driver = first_line(path, default='')
    if scaling_driver:
        info.append('driver:%s' % scaling_driver)

    if scaling_driver == 'intel_pstate':
        path = os.path.join(sys_cpu_path, "intel_pstate/no_turbo")
        no_turbo = first_line(path, default='')
        if no_turbo == '1':
            info.append('intel_pstate:no turbo')
        elif no_turbo == '0':
            info.append('intel_pstate:turbo')
    else:
        boost = get_cpu_boost(cpu)
        if boost is not None:
            if boost:
                info.append('boost:supported')
            else:
                info.append('boost:not suppported')

    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_governor" % cpu)
    scaling_governor = first_line(path, default='')
    if scaling_governor:
        info.append('governor:%s' % scaling_governor)

    return info

def collect_cpu_config(metadata, cpus):
    nohz_full = first_line('/sys/devices/system/cpu/nohz_full', default='')
    if nohz_full:
        nohz_full = parse_cpu_list(nohz_full)

    isolated = get_isolated_cpus()
    if isolated:
        isolated = set(isolated)

    configs = {}
    for cpu in cpus:
        config = get_cpu_config(cpu)
        if nohz_full and cpu in nohz_full:
            config.append('nohz_full')
        if isolated and cpu in isolated:
            config.append('isolated')
        if config:
            configs[cpu] = ', '.join(config)
    if not configs:
        return
    metadata['cpu_config'] = format_cpu_infos(configs)

def get_cpu_temperature(path, cpu_temp):
    hwmon_name = first_line(os.path.join(path, 'name'), default='')
    if not hwmon_name.startswith('coretemp'):
        return

    index = 1
    while True:
        template = os.path.join(path, "temp%s_%%s" % index)

        try:
            temp_label = first_line(template % 'label')
        except IOError:
            break

        temp_input = first_line(template % 'input')
        temp_input = float(temp_input) / 1000
        # FIXME: On Python 2, u"%.0f\xb0C" introduces unicode errors if the
        # locale encoding is ASCII
        temp_input = "%.0f C" % temp_input

        item = '%s:%s=%s' % (hwmon_name, temp_label, temp_input)
        cpu_temp.append(item)

        index += 1

def collect_cpu_temperatures(metadata):
    path = sys_path("class/hwmon")
    try:
        names = os.listdir(path)
    except OSError:
        return None

    cpu_temp = []
    for name in names:
        hwmon = os.path.join(path, name)
        get_cpu_temperature(hwmon, cpu_temp)
    if not cpu_temp:
        return None

    metadata['cpu_temp'] = ', '.join(cpu_temp)

def collect_cpu_affinity(metadata, cpu_affinity, cpu_count):
    if not cpu_affinity:
        return
    if not cpu_count:
        return

    # CPU affinity
    if set(cpu_affinity) == set(range(cpu_count)):
        return

    metadata['cpu_affinity'] = format_cpu_list(cpu_affinity)

def collect_cpu_model(metadata):
    for line in read_proc("cpuinfo"):
        if line.startswith('model name'):
            model_name = line.split(':', 1)[1].strip()
            if model_name:
                metadata['cpu_model_name'] = model_name
            break

        if line.startswith('machine'):
            machine = line.split(':', 1)[1].strip()
            if machine:
                metadata['cpu_machine'] = machine
            break

def collect_cpu_metadata(metadata):
    collect_cpu_model(metadata)

    # CPU count
    cpu_count = get_logical_cpu_count()
    if cpu_count:
        metadata['cpu_count'] = cpu_count

    cpu_affinity = get_cpu_affinity()
    collect_cpu_affinity(metadata, cpu_affinity, cpu_count)

    all_cpus = cpu_affinity
    if not all_cpus and cpu_count:
        all_cpus = tuple(range(cpu_count))

    if all_cpus:
        collect_cpu_freq(metadata, all_cpus)
        collect_cpu_config(metadata, all_cpus)

    collect_cpu_temperatures(metadata)

def collect_metadata(metadata):
    metadata['perf_version'] = perf.__version__

    date = datetime.datetime.now().isoformat()
    # FIXME: Move date to a regular run attribute with type datetime.datetime?
    metadata['date'] = date.split('.', 1)[0]

    collect_python_metadata(metadata)
    collect_system_metadata(metadata)
    collect_memory_metadata(metadata)
    collect_cpu_metadata(metadata)

    # Note: Don't collect VmPeak of /proc/self/status on Linux because it is
    # not accurate. See perf._memory for more accurate memory metrics.
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs all benchmarks in PerfKitBenchmarker.

All benchmarks in PerfKitBenchmarker export the following interface:

GetInfo: this returns, the name of the benchmark, the number of mahcines
          required to run one instance of the benchmark, a detailed description
          of the benchmark, and if the benchmark requires a scratch disk.
Prepare: this function takes a list of VMs as an input parameter. The benchmark
         will then get all binaries required to run the benchmark and, if
         required, create data files.
Run: this function takes a list of VMs as an input parameter. The benchmark will
     then run the benchmark upon the machines specified. The function will
     return a dictonary containing the results of the benchmark.
Cleanup: this function takes a list of VMs as an input parameter. The benchmark
         will then return the machine to the state it was at before Prepare
         was called.

PerfKitBenchmarker has following run stages: prepare, run, cleanup and all.
prepare: PerfKitBenchmarker will read command-line flags, decide which
benchmarks to run
         and create necessary resources for each benchmark, including networks,
         VMs, disks, keys and execute the Prepare function of each benchmark to
         install necessary softwares, upload datafiles, etc and generate a
         run_uri, which can be used to run benchmark multiple times.
run: PerfKitBenchmarker execute the Run function of each benchmark and collect
samples
     generated. Publisher may publish these samples accourding to settings. Run
     stage can be called multiple times with the run_uri generated by prepare
     stage.
cleanup: PerfKitBenchmarker will run Cleanup function of each benchmark to
uninstall
         softwares and delete data files. Then it will delete VMs, key files,
         networks, disks generated in prepare stage.
all: PerfKitBenchmarker will run all above stages (prepare, run, cleanup). Any
resources
     generated in prepare will be automatically deleted at last.
     PerfKitBenchmarker won't
     be able to rerun with exactly same VMs, networks, disks with the same
     run_uri.
"""

import getpass
import logging
import sys
import time
import uuid

from perfkitbenchmarker import benchmarks
from perfkitbenchmarker import benchmark_spec
from perfkitbenchmarker import flags
from perfkitbenchmarker import static_virtual_machine
from perfkitbenchmarker import version
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.publisher import SampleCollector

STAGE_ALL = 'all'
STAGE_PREPARE = 'prepare'
STAGE_RUN = 'run'
STAGE_CLEANUP = 'cleanup'
DEBUG = 'debug'
INFO = 'info'
LOG_LEVELS = {
    DEBUG: logging.DEBUG,
    INFO: logging.INFO
}
REQUIRED_INFO = ['scratch_disk', 'num_machines']
# List of functions taking a benchmark_spec. Will be called before benchmark.Run
# with two parameters: the benchmark and benchmark_spec.
BEFORE_RUN_HOOKS = []
FLAGS = flags.FLAGS

flags.DEFINE_list('ssh_options', [], 'Additional options to pass to ssh.')
flags.DEFINE_integer('parallelism', 1,
                     'The number of benchmarks to run in parallel.')
flags.DEFINE_list('benchmarks', [], 'Benchmarks that should be run, '
                  'default is all.')
flags.DEFINE_string('project', None, 'GCP project ID under which '
                    'to create the virtual machines')
flags.DEFINE_list(
    'zones', None,
    'A list of zones within which to run PerfKitBenchmarker.'
    ' This is specific to the cloud provider you are running on. '
    'If multiple zones are given, PerfKitBenchmarker will create 1 VM in '
    'zone, until enough VMs are created as specified in each '
    'benchmark.')
# TODO(user): note that this is currently very GCE specific. Need to create a
#    module which can traslate from some generic types to provider specific
#    nomenclature.
flags.DEFINE_string('machine_type', None, 'Machine '
                    'types that will be created for benchmarks that don\'t '
                    'require a particular type.')
flags.DEFINE_integer('num_vms', 1, 'For benchmarks which can make use of a '
                     'variable number of machines, the number of VMs to use.')
flags.DEFINE_string('image', None, 'Default image that will be '
                    'linked to the VM')
flags.DEFINE_integer('scratch_disk_size', 500, 'Size, in gb, for all scratch '
                     'disks, default is 500')
flags.DEFINE_string('run_uri', None, 'Name of the Run. If provided, this '
                    'should be alphanumeric and less than or equal to 10 '
                    'characters in length.')
flags.DEFINE_string('owner', getpass.getuser(), 'Owner name. '
                    'Used to tag created resources and performance records.')
flags.DEFINE_enum('log_level', INFO, [DEBUG, INFO], 'The log level to run at.')
flags.DEFINE_enum(
    'run_stage', STAGE_ALL,
    [STAGE_ALL, STAGE_PREPARE, STAGE_RUN, STAGE_CLEANUP],
    'The stage of perfkitbenchmarker to run. By default it runs all stages.')
flags.DEFINE_list('benchmark_config_pair', None,
                  'Benchmark and its config file pair, separated by :.')
flags.DEFINE_integer('duration_in_seconds', None,
                     'duration of benchmarks. '
                     '(only valid for mesh_benchmark)')
flags.DEFINE_string('static_vm_file', None,
                    'The file path for the Static Machine file. See '
                    'static_virtual_machine.py for a description of this file.')
flags.DEFINE_boolean('version', False, 'Display the version and exit.')
flags.DEFINE_enum(
    'scratch_disk_type', benchmark_spec.STANDARD,
    [benchmark_spec.STANDARD, benchmark_spec.SSD, benchmark_spec.IOPS],
    'Type for all scratch disks. The default is standard')
flags.DEFINE_boolean('use_local_disk', False, 'For benchmarks that use disks, '
                     'this indicates that they should run against '
                     'the local disk(s) instead of remote storage.')

MAX_RUN_URI_LENGTH = 10

def ConfigureLogging():
  """Configure logging.

  Note that this will destroy existing logging configuration!

  This configures python logging with a pair of handlers:
  * Messages at FLAGS.log_level and above are emitted to stderr.
  * Messages at DEBUG and above are emitted to 'pkb.log' under
    vm_util.GetTempDir().
  """
  logger = logging.getLogger()
  logger.handlers = []
  logger.setLevel(logging.DEBUG)

  stream_handler = logging.StreamHandler()
  stream_handler.setLevel(LOG_LEVELS[FLAGS.log_level])
  formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
  stream_handler.setFormatter(formatter)
  logger.addHandler(stream_handler)

  vm_util.GenTempDir()
  log_path = vm_util.PrependTempDir('pkb.log')
  logging.info('Verbose logging to: %s', log_path)
  file_handler = logging.FileHandler(filename=log_path)
  file_handler.setLevel(logging.DEBUG)
  formatter = logging.Formatter(
      '%(asctime)s %(filename)s:%(lineno)d %(levelname)-8s %(message)s')
  file_handler.setFormatter(formatter)
  logger.addHandler(file_handler)

def ShouldRunBenchmark(benchmark):
  if not FLAGS.benchmarks:
    return True
  return benchmark['name'] in FLAGS.benchmarks

# TODO(user): Consider moving to benchmark_spec.
def ValidateBenchmarkInfo(benchmark_info):
  for required_key in REQUIRED_INFO:
    if required_key not in benchmark_info:
      logging.error('Benchmark information %s is corrupt. It does not contain'
                    'the key %s. Please add the specified key to the benchmark'
                    'info. Skipping benchmark.', benchmark_info, required_key)
      # TODO(user): Raise error with info about the validation failure
      return False
  return True

def ListUnknownBenchmarks():
  """Identify invalid benchmark names specified in the command line flags."""
  valid_benchmark_names = frozenset(benchmark.GetInfo()['name']
                                    for benchmark in benchmarks.BENCHMARKS)
  specified_benchmark_names = frozenset(FLAGS.benchmarks)

  return sorted(specified_benchmark_names - valid_benchmark_names)

def RunBenchmark(benchmark, collector):
  """Runs a single benchmark and adds the results to the collector.

  Args:
    benchmark: The benchmark module to be run.
    collector: The SampleCollector object to add samples to.
  """
  benchmark_info = benchmark.GetInfo()
  benchmark_specification = None
  if not ShouldRunBenchmark(benchmark_info):
    return
  if not ValidateBenchmarkInfo(benchmark_info):
    return

  # Optional prerequisite checking.
  check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
  if check_prereqs:
    check_prereqs()

  start_time = time.time()
  try:
    if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
      logging.info('Preparing benchmark %s', benchmark_info['name'])
      benchmark_specification = benchmark_spec.BenchmarkSpec(benchmark_info)
      benchmark_specification.Prepare()
      benchmark.Prepare(benchmark_specification)
    else:
      benchmark_specification = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
          benchmark_info['name'])
    if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
      logging.info('Running benchmark %s', benchmark_info['name'])
      for before_run_hook in BEFORE_RUN_HOOKS:
        before_run_hook(benchmark=benchmark,
                        benchmark_spec=benchmark_specification)
      samples = benchmark.Run(benchmark_specification)
      collector.AddSamples(samples, benchmark_info['name'],
                           benchmark_specification)

    if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
      logging.info('Cleaning up benchmark %s', benchmark_info['name'])
      benchmark.Cleanup(benchmark_specification)
      benchmark_specification.Delete()
      if FLAGS.run_stage == STAGE_ALL:
        end_time = time.time()
        end_to_end_sample = ['End to End Runtime',
                             end_time - start_time,
                             'seconds', {}]
        collector.AddSamples([end_to_end_sample], benchmark_info['name'],
                             benchmark_specification)
  except Exception:
    # Resource cleanup (below) can take a long time. Log the error to give
    # immediate feedback, then re-throw.
    logging.exception('Error during benchmark %s', benchmark_info['name'])
    raise
  finally:
    if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
      if benchmark_specification:
        benchmark_specification.Delete()
    else:
      benchmark_specification.PickleSpec()

def RunBenchmarks(publish=True):
  """Runs all benchmarks in PerfKitBenchmarker.

  Args:
    publish: A boolean indicating whether results should be published.

  Returns:
    Exit status for the process.
  """
  if FLAGS.version:
    print version.VERSION
    return

  if FLAGS.run_uri is None:
    if FLAGS.run_stage not in [STAGE_ALL, STAGE_PREPARE]:
      logging.error(
          'Cannot run "%s" with unspecified run_uri.', FLAGS.run_stage)
      return 1
    else:
      FLAGS.run_uri = str(uuid.uuid4())[-8:]
  elif not FLAGS.run_uri.isalnum() or len(FLAGS.run_uri) > MAX_RUN_URI_LENGTH:
    logging.error('run_uri must be alphanumeric and less than or equal '
                  'to 10 characters in length.')
    return 1

  ConfigureLogging()

  unknown_benchmarks = ListUnknownBenchmarks()
  if unknown_benchmarks:
    logging.error('Unknown benchmark(s) provided: %s',
                  ', '.join(unknown_benchmarks))
    return 1

  vm_util.SSHKeyGen()
  collector = SampleCollector()

  if FLAGS.static_vm_file:
    with open(FLAGS.static_vm_file) as fp:
      static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
          fp)

  if FLAGS.benchmark_config_pair:
    # Convert benchmark_config_pair into a {benchmark_name: file_name}
    # dictionary.
    tmp_dict = {}
    for config_pair in FLAGS.benchmark_config_pair:
      pair = config_pair.split(':')
      tmp_dict[pair[0]] = pair[1]
    FLAGS.benchmark_config_pair = tmp_dict

  try:
    if FLAGS.parallelism > 1:
      args = [((benchmark, collector), {})
              for benchmark in benchmarks.BENCHMARKS]
      vm_util.RunThreaded(
          RunBenchmark, args, max_concurrent_threads=FLAGS.parallelism)
    else:
      for benchmark in benchmarks.BENCHMARKS:
        RunBenchmark(benchmark, collector)
  finally:
    if collector.samples:
      collector.PublishSamples()

  if FLAGS.run_stage not in [STAGE_ALL, STAGE_CLEANUP]:
    logging.info(
        'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri)

def Main(argv=sys.argv):
  logging.basicConfig(level=logging.INFO)
  # TODO: Verify if there is other way of appending additional help
  # message.
  # Inject more help documentation
  benchmark_list = ['%s:  %s' % (benchmark_module.GetInfo()['name'],
                                 benchmark_module.GetInfo()['description'])
                    for benchmark_module in benchmarks.BENCHMARKS]
  sys.modules['__main__'].__doc__ = __doc__ + (
      '\nBenchmarks:\n\t%s') % '\n\t'.join(benchmark_list)
  try:
    argv = FLAGS(argv)  # parse flags
  except flags.FlagsError as e:
    logging.error(
        '%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
    sys.exit(1)
  return RunBenchmarks()
<commit_msg>Log exception with benchmark name<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Runs all benchmarks in PerfKitBenchmarker.

All benchmarks in PerfKitBenchmarker export the following interface:

GetInfo: this returns, the name of the benchmark, the number of mahcines
          required to run one instance of the benchmark, a detailed description
          of the benchmark, and if the benchmark requires a scratch disk.
Prepare: this function takes a list of VMs as an input parameter. The benchmark
         will then get all binaries required to run the benchmark and, if
         required, create data files.
Run: this function takes a list of VMs as an input parameter. The benchmark will
     then run the benchmark upon the machines specified. The function will
     return a dictonary containing the results of the benchmark.
Cleanup: this function takes a list of VMs as an input parameter. The benchmark
         will then return the machine to the state it was at before Prepare
         was called.

PerfKitBenchmarker has following run stages: prepare, run, cleanup and all.
prepare: PerfKitBenchmarker will read command-line flags, decide which
benchmarks to run
         and create necessary resources for each benchmark, including networks,
         VMs, disks, keys and execute the Prepare function of each benchmark to
         install necessary softwares, upload datafiles, etc and generate a
         run_uri, which can be used to run benchmark multiple times.
run: PerfKitBenchmarker execute the Run function of each benchmark and collect
samples
     generated. Publisher may publish these samples accourding to settings. Run
     stage can be called multiple times with the run_uri generated by prepare
     stage.
cleanup: PerfKitBenchmarker will run Cleanup function of each benchmark to
uninstall
         softwares and delete data files. Then it will delete VMs, key files,
         networks, disks generated in prepare stage.
all: PerfKitBenchmarker will run all above stages (prepare, run, cleanup). Any
resources
     generated in prepare will be automatically deleted at last.
     PerfKitBenchmarker won't
     be able to rerun with exactly same VMs, networks, disks with the same
     run_uri.
"""

import getpass
import logging
import sys
import time
import uuid

from perfkitbenchmarker import benchmarks
from perfkitbenchmarker import benchmark_spec
from perfkitbenchmarker import flags
from perfkitbenchmarker import static_virtual_machine
from perfkitbenchmarker import version
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.publisher import SampleCollector

STAGE_ALL = 'all'
STAGE_PREPARE = 'prepare'
STAGE_RUN = 'run'
STAGE_CLEANUP = 'cleanup'
DEBUG = 'debug'
INFO = 'info'
LOG_LEVELS = {
    DEBUG: logging.DEBUG,
    INFO: logging.INFO
}
REQUIRED_INFO = ['scratch_disk', 'num_machines']
# List of functions taking a benchmark_spec. Will be called before benchmark.Run
# with two parameters: the benchmark and benchmark_spec.
BEFORE_RUN_HOOKS = []
FLAGS = flags.FLAGS

flags.DEFINE_list('ssh_options', [], 'Additional options to pass to ssh.')
flags.DEFINE_integer('parallelism', 1,
                     'The number of benchmarks to run in parallel.')
flags.DEFINE_list('benchmarks', [], 'Benchmarks that should be run, '
                  'default is all.')
flags.DEFINE_string('project', None, 'GCP project ID under which '
                    'to create the virtual machines')
flags.DEFINE_list(
    'zones', None,
    'A list of zones within which to run PerfKitBenchmarker.'
    ' This is specific to the cloud provider you are running on. '
    'If multiple zones are given, PerfKitBenchmarker will create 1 VM in '
    'zone, until enough VMs are created as specified in each '
    'benchmark.')
# TODO(user): note that this is currently very GCE specific. Need to create a
#    module which can traslate from some generic types to provider specific
#    nomenclature.
flags.DEFINE_string('machine_type', None, 'Machine '
                    'types that will be created for benchmarks that don\'t '
                    'require a particular type.')
flags.DEFINE_integer('num_vms', 1, 'For benchmarks which can make use of a '
                     'variable number of machines, the number of VMs to use.')
flags.DEFINE_string('image', None, 'Default image that will be '
                    'linked to the VM')
flags.DEFINE_integer('scratch_disk_size', 500, 'Size, in gb, for all scratch '
                     'disks, default is 500')
flags.DEFINE_string('run_uri', None, 'Name of the Run. If provided, this '
                    'should be alphanumeric and less than or equal to 10 '
                    'characters in length.')
flags.DEFINE_string('owner', getpass.getuser(), 'Owner name. '
                    'Used to tag created resources and performance records.')
flags.DEFINE_enum('log_level', INFO, [DEBUG, INFO], 'The log level to run at.')
flags.DEFINE_enum(
    'run_stage', STAGE_ALL,
    [STAGE_ALL, STAGE_PREPARE, STAGE_RUN, STAGE_CLEANUP],
    'The stage of perfkitbenchmarker to run. By default it runs all stages.')
flags.DEFINE_list('benchmark_config_pair', None,
                  'Benchmark and its config file pair, separated by :.')
flags.DEFINE_integer('duration_in_seconds', None,
                     'duration of benchmarks. '
                     '(only valid for mesh_benchmark)')
flags.DEFINE_string('static_vm_file', None,
                    'The file path for the Static Machine file. See '
                    'static_virtual_machine.py for a description of this file.')
flags.DEFINE_boolean('version', False, 'Display the version and exit.')
flags.DEFINE_enum(
    'scratch_disk_type', benchmark_spec.STANDARD,
    [benchmark_spec.STANDARD, benchmark_spec.SSD, benchmark_spec.IOPS],
    'Type for all scratch disks. The default is standard')
flags.DEFINE_boolean('use_local_disk', False, 'For benchmarks that use disks, '
                     'this indicates that they should run against '
                     'the local disk(s) instead of remote storage.')

MAX_RUN_URI_LENGTH = 10

def ConfigureLogging():
  """Configure logging.

  Note that this will destroy existing logging configuration!

  This configures python logging with a pair of handlers:
  * Messages at FLAGS.log_level and above are emitted to stderr.
  * Messages at DEBUG and above are emitted to 'pkb.log' under
    vm_util.GetTempDir().
  """
  logger = logging.getLogger()
  logger.handlers = []
  logger.setLevel(logging.DEBUG)

  stream_handler = logging.StreamHandler()
  stream_handler.setLevel(LOG_LEVELS[FLAGS.log_level])
  formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
  stream_handler.setFormatter(formatter)
  logger.addHandler(stream_handler)

  vm_util.GenTempDir()
  log_path = vm_util.PrependTempDir('pkb.log')
  logging.info('Verbose logging to: %s', log_path)
  file_handler = logging.FileHandler(filename=log_path)
  file_handler.setLevel(logging.DEBUG)
  formatter = logging.Formatter(
      '%(asctime)s %(filename)s:%(lineno)d %(levelname)-8s %(message)s')
  file_handler.setFormatter(formatter)
  logger.addHandler(file_handler)

def ShouldRunBenchmark(benchmark):
  if not FLAGS.benchmarks:
    return True
  return benchmark['name'] in FLAGS.benchmarks

# TODO(user): Consider moving to benchmark_spec.
def ValidateBenchmarkInfo(benchmark_info):
  for required_key in REQUIRED_INFO:
    if required_key not in benchmark_info:
      logging.error('Benchmark information %s is corrupt. It does not contain'
                    'the key %s. Please add the specified key to the benchmark'
                    'info. Skipping benchmark.', benchmark_info, required_key)
      # TODO(user): Raise error with info about the validation failure
      return False
  return True

def ListUnknownBenchmarks():
  """Identify invalid benchmark names specified in the command line flags."""
  valid_benchmark_names = frozenset(benchmark.GetInfo()['name']
                                    for benchmark in benchmarks.BENCHMARKS)
  specified_benchmark_names = frozenset(FLAGS.benchmarks)

  return sorted(specified_benchmark_names - valid_benchmark_names)

def RunBenchmark(benchmark, collector):
  """Runs a single benchmark and adds the results to the collector.

  Args:
    benchmark: The benchmark module to be run.
    collector: The SampleCollector object to add samples to.
  """
  benchmark_info = benchmark.GetInfo()
  benchmark_specification = None
  if not ShouldRunBenchmark(benchmark_info):
    return
  if not ValidateBenchmarkInfo(benchmark_info):
    return

  # Optional prerequisite checking.
  check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
  if check_prereqs:
    try:
      check_prereqs()
    except:
      logging.exception('Prerequisite check failed for %s',
                        benchmark_info['name'])
      raise

  start_time = time.time()
  try:
    if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
      logging.info('Preparing benchmark %s', benchmark_info['name'])
      benchmark_specification = benchmark_spec.BenchmarkSpec(benchmark_info)
      benchmark_specification.Prepare()
      benchmark.Prepare(benchmark_specification)
    else:
      benchmark_specification = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
          benchmark_info['name'])
    if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
      logging.info('Running benchmark %s', benchmark_info['name'])
      for before_run_hook in BEFORE_RUN_HOOKS:
        before_run_hook(benchmark=benchmark,
                        benchmark_spec=benchmark_specification)
      samples = benchmark.Run(benchmark_specification)
      collector.AddSamples(samples, benchmark_info['name'],
                           benchmark_specification)

    if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
      logging.info('Cleaning up benchmark %s', benchmark_info['name'])
      benchmark.Cleanup(benchmark_specification)
      benchmark_specification.Delete()
      if FLAGS.run_stage == STAGE_ALL:
        end_time = time.time()
        end_to_end_sample = ['End to End Runtime',
                             end_time - start_time,
                             'seconds', {}]
        collector.AddSamples([end_to_end_sample], benchmark_info['name'],
                             benchmark_specification)
  except Exception:
    # Resource cleanup (below) can take a long time. Log the error to give
    # immediate feedback, then re-throw.
    logging.exception('Error during benchmark %s', benchmark_info['name'])
    raise
  finally:
    if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
      if benchmark_specification:
        benchmark_specification.Delete()
    else:
      benchmark_specification.PickleSpec()

def RunBenchmarks(publish=True):
  """Runs all benchmarks in PerfKitBenchmarker.

  Args:
    publish: A boolean indicating whether results should be published.

  Returns:
    Exit status for the process.
  """
  if FLAGS.version:
    print version.VERSION
    return

  if FLAGS.run_uri is None:
    if FLAGS.run_stage not in [STAGE_ALL, STAGE_PREPARE]:
      logging.error(
          'Cannot run "%s" with unspecified run_uri.', FLAGS.run_stage)
      return 1
    else:
      FLAGS.run_uri = str(uuid.uuid4())[-8:]
  elif not FLAGS.run_uri.isalnum() or len(FLAGS.run_uri) > MAX_RUN_URI_LENGTH:
    logging.error('run_uri must be alphanumeric and less than or equal '
                  'to 10 characters in length.')
    return 1

  ConfigureLogging()

  unknown_benchmarks = ListUnknownBenchmarks()
  if unknown_benchmarks:
    logging.error('Unknown benchmark(s) provided: %s',
                  ', '.join(unknown_benchmarks))
    return 1

  vm_util.SSHKeyGen()
  collector = SampleCollector()

  if FLAGS.static_vm_file:
    with open(FLAGS.static_vm_file) as fp:
      static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
          fp)

  if FLAGS.benchmark_config_pair:
    # Convert benchmark_config_pair into a {benchmark_name: file_name}
    # dictionary.
    tmp_dict = {}
    for config_pair in FLAGS.benchmark_config_pair:
      pair = config_pair.split(':')
      tmp_dict[pair[0]] = pair[1]
    FLAGS.benchmark_config_pair = tmp_dict

  try:
    if FLAGS.parallelism > 1:
      args = [((benchmark, collector), {})
              for benchmark in benchmarks.BENCHMARKS]
      vm_util.RunThreaded(
          RunBenchmark, args, max_concurrent_threads=FLAGS.parallelism)
    else:
      for benchmark in benchmarks.BENCHMARKS:
        RunBenchmark(benchmark, collector)
  finally:
    if collector.samples:
      collector.PublishSamples()

  if FLAGS.run_stage not in [STAGE_ALL, STAGE_CLEANUP]:
    logging.info(
        'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri)

def Main(argv=sys.argv):
  logging.basicConfig(level=logging.INFO)
  # TODO: Verify if there is other way of appending additional help
  # message.
  # Inject more help documentation
  benchmark_list = ['%s:  %s' % (benchmark_module.GetInfo()['name'],
                                 benchmark_module.GetInfo()['description'])
                    for benchmark_module in benchmarks.BENCHMARKS]
  sys.modules['__main__'].__doc__ = __doc__ + (
      '\nBenchmarks:\n\t%s') % '\n\t'.join(benchmark_list)
  try:
    argv = FLAGS(argv)  # parse flags
  except flags.FlagsError as e:
    logging.error(
        '%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
    sys.exit(1)
  return RunBenchmarks()
<|endoftext|>


<commit_before>import os

from perfrunner.helpers import local
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.tests import PerfTest

class BackupRestoreTest(PerfTest):

    def extract_tools(self):
        local.extract_cb(filename='couchbase.rpm')

    def flush_buckets(self):
        for i in range(self.test_config.cluster.num_buckets):
            bucket = 'bucket-{}'.format(i + 1)
            self.rest.flush_bucket(self.master_node, bucket)

    def backup(self, mode=None):
        local.backup(
            master_node=self.master_node,
            cluster_spec=self.cluster_spec,
            threads=self.test_config.backup_settings.threads,
            wrapper=self.rest.is_community(self.master_node),
            mode=mode,
            compression=self.test_config.backup_settings.compression,
            storage_type=self.test_config.backup_settings.storage_type,
            sink_type=self.test_config.backup_settings.sink_type,
            shards=self.test_config.backup_settings.shards
        )

    def compact(self):
        snapshots = local.get_backup_snapshots(self.cluster_spec)
        local.compact(self.cluster_spec,
                      snapshots,
                      self.rest.is_community(self.master_node)
                      )

    def restore(self):
        local.drop_caches()

        local.restore(cluster_spec=self.cluster_spec,
                      master_node=self.master_node,
                      threads=self.test_config.restore_settings.threads,
                      wrapper=self.rest.is_community(self.master_node))

    def run(self):
        self.extract_tools()

        self.load()
        self.wait_for_persistence()

class BackupTest(BackupRestoreTest):

    @with_stats
    @timeit
    def backup(self, mode=None):
        super().backup(mode)

    def _report_kpi(self, time_elapsed):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'
        backup_size = local.calc_backup_size(self.cluster_spec)

        backing_store = self.test_config.backup_settings.storage_type
        sink_type = self.test_config.backup_settings.sink_type

        tool = 'backup'
        if backing_store:
            tool += '-' + backing_store
        elif sink_type:
            tool += '-' + sink_type

        self.reporter.post(
            *self.metrics.bnr_throughput(time_elapsed,
                                         edition,
                                         tool=tool)
        )

        self.reporter.post(
            *self.metrics.backup_size(backup_size,
                                      edition,
                                      tool=tool if backing_store or sink_type
                                      else None)
        )

    def run(self):
        super().run()

        time_elapsed = self.backup()

        self.report_kpi(time_elapsed)

class BackupSizeTest(BackupTest):

    def _report_kpi(self, *args):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'
        backup_size = local.calc_backup_size(self.cluster_spec)

        self.reporter.post(
            *self.metrics.backup_size(backup_size, edition)
        )

class BackupTestWithCompact(BackupTest):

    @with_stats
    @timeit
    def backup(self, mode=None):
        super().backup(mode)
        super().compact()

class BackupUnderLoadTest(BackupTest):

    def run(self):
        super(BackupTest, self).run()

        self.hot_load()

        self.access_bg()

        time_elapsed = self.backup()

        self.report_kpi(time_elapsed)

class MergeTest(BackupRestoreTest):

    @with_stats
    @timeit
    def merge(self):
        snapshots = local.get_backup_snapshots(self.cluster_spec)

        local.drop_caches()

        local.cbbackupmgr_merge(self.cluster_spec, snapshots)

    def _report_kpi(self, time_elapsed):
        self.reporter.post(
            *self.metrics.merge_throughput(time_elapsed)
        )

    def run(self):
        self.extract_tools()

        self.load()
        self.wait_for_persistence()
        self.backup()  # 1st snapshot

        self.flush_buckets()

        self.load()
        self.wait_for_persistence()
        self.backup(mode=True)  # 2nd snapshot

        time_elapsed = self.merge()

        self.report_kpi(time_elapsed)

class RestoreTest(BackupRestoreTest):

    @with_stats
    @timeit
    def restore(self):
        super().restore()

    def _report_kpi(self, time_elapsed):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'

        backing_store = self.test_config.backup_settings.storage_type
        sink_type = self.test_config.backup_settings.sink_type

        tool = 'restore'
        if backing_store:
            tool += '-' + backing_store
        elif sink_type:
            tool += '-' + sink_type

        self.reporter.post(
            *self.metrics.bnr_throughput(time_elapsed,
                                         edition,
                                         tool=tool)
        )

    def run(self):
        super().run()

        self.backup()

        self.flush_buckets()

        time_elapsed = self.restore()

        self.report_kpi(time_elapsed)

class ExportImportTest(BackupRestoreTest):

    def export(self):
        local.cbexport(master_node=self.master_node,
                       cluster_spec=self.cluster_spec,
                       bucket=self.test_config.buckets[0],
                       data_format=self.test_config.export_settings.format,
                       threads=self.test_config.restore_settings.threads)

    def import_data(self):
        import_file = self.test_config.export_settings.import_file
        if import_file is None:
            import_file = 'data.{}'.format(self.test_config.export_settings.type)
            import_file = os.path.join(self.cluster_spec.backup, import_file)
        if self.test_config.export_settings.format != 'sample':
            import_file = 'file://{}'.format(import_file)

        local.drop_caches()

        local.cbimport(master_node=self.master_node,
                       cluster_spec=self.cluster_spec,
                       data_type=self.test_config.export_settings.type,
                       data_format=self.test_config.export_settings.format,
                       bucket=self.test_config.buckets[0],
                       import_file=import_file,
                       threads=self.test_config.backup_settings.threads)

    def _report_kpi(self, time_elapsed: float):
        self.reporter.post(
            *self.metrics.import_and_export_throughput(time_elapsed)
        )

class ExportTest(ExportImportTest):

    @with_stats
    @timeit
    def export(self):
        super().export()

    def run(self):
        super().run()

        time_elapsed = self.export()

        self.report_kpi(time_elapsed)

class ImportTest(ExportImportTest):

    @with_stats
    @timeit
    def import_data(self):
        super().import_data()

    def run(self):
        super().run()

        self.export()

        self.flush_buckets()

        time_elapsed = self.import_data()

        self.report_kpi(time_elapsed)

class ImportSampleDataTest(ImportTest):

    def _report_kpi(self, time_elapsed: float):
        self.reporter.post(
            *self.metrics.import_file_throughput(time_elapsed)
        )

    def run(self):
        self.extract_tools()

        time_elapsed = self.import_data()

        self.report_kpi(time_elapsed)
<commit_msg>CBPS-634 Remove backup size KPI from blackhole backups<commit_after>import os

from perfrunner.helpers import local
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.tests import PerfTest

class BackupRestoreTest(PerfTest):

    def extract_tools(self):
        local.extract_cb(filename='couchbase.rpm')

    def flush_buckets(self):
        for i in range(self.test_config.cluster.num_buckets):
            bucket = 'bucket-{}'.format(i + 1)
            self.rest.flush_bucket(self.master_node, bucket)

    def backup(self, mode=None):
        local.backup(
            master_node=self.master_node,
            cluster_spec=self.cluster_spec,
            threads=self.test_config.backup_settings.threads,
            wrapper=self.rest.is_community(self.master_node),
            mode=mode,
            compression=self.test_config.backup_settings.compression,
            storage_type=self.test_config.backup_settings.storage_type,
            sink_type=self.test_config.backup_settings.sink_type,
            shards=self.test_config.backup_settings.shards
        )

    def compact(self):
        snapshots = local.get_backup_snapshots(self.cluster_spec)
        local.compact(self.cluster_spec,
                      snapshots,
                      self.rest.is_community(self.master_node)
                      )

    def restore(self):
        local.drop_caches()

        local.restore(cluster_spec=self.cluster_spec,
                      master_node=self.master_node,
                      threads=self.test_config.restore_settings.threads,
                      wrapper=self.rest.is_community(self.master_node))

    def run(self):
        self.extract_tools()

        self.load()
        self.wait_for_persistence()

class BackupTest(BackupRestoreTest):

    @with_stats
    @timeit
    def backup(self, mode=None):
        super().backup(mode)

    def _report_kpi(self, time_elapsed):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'
        backup_size = local.calc_backup_size(self.cluster_spec)

        backing_store = self.test_config.backup_settings.storage_type
        sink_type = self.test_config.backup_settings.sink_type

        tool = 'backup'
        if backing_store:
            tool += '-' + backing_store
        elif sink_type:
            tool += '-' + sink_type

        self.reporter.post(
            *self.metrics.bnr_throughput(time_elapsed,
                                         edition,
                                         tool=tool)
        )

        if sink_type != 'blackhole':
            self.reporter.post(
                *self.metrics.backup_size(
                    backup_size,
                    edition,
                    tool=tool if backing_store or sink_type else None)
            )

    def run(self):
        super().run()

        time_elapsed = self.backup()

        self.report_kpi(time_elapsed)

class BackupSizeTest(BackupTest):

    def _report_kpi(self, *args):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'
        backup_size = local.calc_backup_size(self.cluster_spec)

        self.reporter.post(
            *self.metrics.backup_size(backup_size, edition)
        )

class BackupTestWithCompact(BackupTest):

    @with_stats
    @timeit
    def backup(self, mode=None):
        super().backup(mode)
        super().compact()

class BackupUnderLoadTest(BackupTest):

    def run(self):
        super(BackupTest, self).run()

        self.hot_load()

        self.access_bg()

        time_elapsed = self.backup()

        self.report_kpi(time_elapsed)

class MergeTest(BackupRestoreTest):

    @with_stats
    @timeit
    def merge(self):
        snapshots = local.get_backup_snapshots(self.cluster_spec)

        local.drop_caches()

        local.cbbackupmgr_merge(self.cluster_spec, snapshots)

    def _report_kpi(self, time_elapsed):
        self.reporter.post(
            *self.metrics.merge_throughput(time_elapsed)
        )

    def run(self):
        self.extract_tools()

        self.load()
        self.wait_for_persistence()
        self.backup()  # 1st snapshot

        self.flush_buckets()

        self.load()
        self.wait_for_persistence()
        self.backup(mode=True)  # 2nd snapshot

        time_elapsed = self.merge()

        self.report_kpi(time_elapsed)

class RestoreTest(BackupRestoreTest):

    @with_stats
    @timeit
    def restore(self):
        super().restore()

    def _report_kpi(self, time_elapsed):
        edition = self.rest.is_community(self.master_node) and 'CE' or 'EE'

        backing_store = self.test_config.backup_settings.storage_type
        sink_type = self.test_config.backup_settings.sink_type

        tool = 'restore'
        if backing_store:
            tool += '-' + backing_store
        elif sink_type:
            tool += '-' + sink_type

        self.reporter.post(
            *self.metrics.bnr_throughput(time_elapsed,
                                         edition,
                                         tool=tool)
        )

    def run(self):
        super().run()

        self.backup()

        self.flush_buckets()

        time_elapsed = self.restore()

        self.report_kpi(time_elapsed)

class ExportImportTest(BackupRestoreTest):

    def export(self):
        local.cbexport(master_node=self.master_node,
                       cluster_spec=self.cluster_spec,
                       bucket=self.test_config.buckets[0],
                       data_format=self.test_config.export_settings.format,
                       threads=self.test_config.restore_settings.threads)

    def import_data(self):
        import_file = self.test_config.export_settings.import_file
        if import_file is None:
            import_file = 'data.{}'.format(self.test_config.export_settings.type)
            import_file = os.path.join(self.cluster_spec.backup, import_file)
        if self.test_config.export_settings.format != 'sample':
            import_file = 'file://{}'.format(import_file)

        local.drop_caches()

        local.cbimport(master_node=self.master_node,
                       cluster_spec=self.cluster_spec,
                       data_type=self.test_config.export_settings.type,
                       data_format=self.test_config.export_settings.format,
                       bucket=self.test_config.buckets[0],
                       import_file=import_file,
                       threads=self.test_config.backup_settings.threads)

    def _report_kpi(self, time_elapsed: float):
        self.reporter.post(
            *self.metrics.import_and_export_throughput(time_elapsed)
        )

class ExportTest(ExportImportTest):

    @with_stats
    @timeit
    def export(self):
        super().export()

    def run(self):
        super().run()

        time_elapsed = self.export()

        self.report_kpi(time_elapsed)

class ImportTest(ExportImportTest):

    @with_stats
    @timeit
    def import_data(self):
        super().import_data()

    def run(self):
        super().run()

        self.export()

        self.flush_buckets()

        time_elapsed = self.import_data()

        self.report_kpi(time_elapsed)

class ImportSampleDataTest(ImportTest):

    def _report_kpi(self, time_elapsed: float):
        self.reporter.post(
            *self.metrics.import_file_throughput(time_elapsed)
        )

    def run(self):
        self.extract_tools()

        time_elapsed = self.import_data()

        self.report_kpi(time_elapsed)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin

from apps.internal import views as internal_views
from apps.authentication import views as authentication_views
from apps.dashboard import views as dashboard_views
from apps.genome import views as genome_views
from apps.riskreport import views as riskreport_views
from apps.faq import views as faq_views
from lib.utils import clogging
log = clogging.getColorLogger(__name__)

admin.autodiscover()

urlpatterns = [
    # Internal
    url(r'^internal/admin/', include(admin.site.urls)),
    url(r'^internal/health-check/$', internal_views.health_check),

    # Authentication
    url(r'^logout/$',   authentication_views.logout),
    url(r'^trydemo/$',  authentication_views.trydemo),

    # Dashboard
    url(r'^dashboard/$', dashboard_views.index, name='dashboard'),

    # Genome upload/delete/status
    url(r'^genome/upload/$', genome_views.upload, name='genome-upload'),
    url(r'^genome/delete/$', genome_views.delete, name='genome-delete'),
    url(r'^genome/status/$', genome_views.status, name='genome-status'),

    # Risk Report
    url(r'^riskreport/$', riskreport_views.index, name='riskreport'),
    url(r'^riskreport/(?P<display_id>[^/]{8})/(?P<phenotype_id>\d*)/$', 'apps.riskreport.views.phenotype', name='riskreport-phenotype'),

    # TODO:
    # url(r'^riskreport/(?P<display_id>[^/]{8})/$', 'apps.riskreport.views.show', name='riskreport-show'),
    # url(r'^riskreport/export/$', 'apps.riskreport.views.export'),

    # TODO:
    # url(r'^library/$', 'apps.library.views.index'),
    # url(r'^library/trait/$', 'apps.library.views.trait_index'),
    # url(r'^library/trait/(?P<trait>.*?)/$', 'apps.library.views.trait'),
    # url(r'^library/snps/$', 'apps.library.views.snps_index'),
    # url(r'^library/snps/rs/(?P<rs>.*?)/$', 'apps.library.views.snps'),
    # url(r'^library/summary/$', 'apps.library.views.summary_index'),
    # url(r'^library/summary/(?P<field_name>.*?)/$', 'apps.library.views.summary'),

    # FAQ
    url(r'^faq/$', faq_views.index, name='faq'),

] + static(settings.STATIC_URL)

if settings.DEBUG:
    import debug_toolbar

    urlpatterns += [
        url(r'^__debug__/$', include(debug_toolbar.urls)),
    ]

if not settings.DEMO_MODE_ONLY:
    urlpatterns += [
        url(r'^register/$', authentication_views.register),
        url(r'^about/$',    authentication_views.about, name='about'),
        url(r'^activation/(?P<activation_key>\w{{{activation_key_length}}})/$'.format(activation_key_length=settings.ACCOUNT_ACTIVATION_KEY_LENGTH),
            authentication_views.activation),
        url(r'^login/$',    authentication_views.login),
    ]

# Include extra apps defined in settings.py
extra_apps = [x for x in list(set(settings.INSTALLED_APPS) - set(settings.DEFAULT_APPS)) if x.startswith('apps.')]

for extra_app in extra_apps:
    name = extra_app[5:]
    urlpatterns += [
        url('^{name}/'.format(name=name), include('apps.{name}.urls'.format(name=name))),
    ]

# Set root (/)
if 'apps.landing' in extra_apps:
    from apps.landing import views as landing_views
    root_view = landing_views.index
else:
    log.info('apps.landing is not installed. So /login is set as root (/) url.')
    root_view = authentication_views.login

urlpatterns += [
    url(r'^$', root_view),
]
<commit_msg>disable /faq<commit_after>from django.conf.urls import patterns, include, url
from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin

from apps.internal import views as internal_views
from apps.authentication import views as authentication_views
from apps.dashboard import views as dashboard_views
from apps.genome import views as genome_views
from apps.riskreport import views as riskreport_views
from apps.faq import views as faq_views
from lib.utils import clogging
log = clogging.getColorLogger(__name__)

admin.autodiscover()

urlpatterns = [
    # Internal
    url(r'^internal/admin/', include(admin.site.urls)),
    url(r'^internal/health-check/$', internal_views.health_check),

    # Authentication
    url(r'^logout/$',   authentication_views.logout),
    url(r'^trydemo/$',  authentication_views.trydemo),

    # Dashboard
    url(r'^dashboard/$', dashboard_views.index, name='dashboard'),

    # Genome upload/delete/status
    url(r'^genome/upload/$', genome_views.upload, name='genome-upload'),
    url(r'^genome/delete/$', genome_views.delete, name='genome-delete'),
    url(r'^genome/status/$', genome_views.status, name='genome-status'),

    # Risk Report
    url(r'^riskreport/$', riskreport_views.index, name='riskreport'),
    url(r'^riskreport/(?P<display_id>[^/]{8})/(?P<phenotype_id>\d*)/$', 'apps.riskreport.views.phenotype', name='riskreport-phenotype'),

    # TODO:
    # url(r'^riskreport/(?P<display_id>[^/]{8})/$', 'apps.riskreport.views.show', name='riskreport-show'),
    # url(r'^riskreport/export/$', 'apps.riskreport.views.export'),

    # TODO:
    # url(r'^library/$', 'apps.library.views.index'),
    # url(r'^library/trait/$', 'apps.library.views.trait_index'),
    # url(r'^library/trait/(?P<trait>.*?)/$', 'apps.library.views.trait'),
    # url(r'^library/snps/$', 'apps.library.views.snps_index'),
    # url(r'^library/snps/rs/(?P<rs>.*?)/$', 'apps.library.views.snps'),
    # url(r'^library/summary/$', 'apps.library.views.summary_index'),
    # url(r'^library/summary/(?P<field_name>.*?)/$', 'apps.library.views.summary'),

    # FAQ
    # url(r'^faq/$', faq_views.index, name='faq'),

] + static(settings.STATIC_URL)

if settings.DEBUG:
    import debug_toolbar

    urlpatterns += [
        url(r'^__debug__/$', include(debug_toolbar.urls)),
    ]

if not settings.DEMO_MODE_ONLY:
    urlpatterns += [
        url(r'^register/$', authentication_views.register),
        url(r'^about/$',    authentication_views.about, name='about'),
        url(r'^activation/(?P<activation_key>\w{{{activation_key_length}}})/$'.format(activation_key_length=settings.ACCOUNT_ACTIVATION_KEY_LENGTH),
            authentication_views.activation),
        url(r'^login/$',    authentication_views.login),
    ]

# Include extra apps defined in settings.py
extra_apps = [x for x in list(set(settings.INSTALLED_APPS) - set(settings.DEFAULT_APPS)) if x.startswith('apps.')]

for extra_app in extra_apps:
    name = extra_app[5:]
    urlpatterns += [
        url('^{name}/'.format(name=name), include('apps.{name}.urls'.format(name=name))),
    ]

# Set root (/)
if 'apps.landing' in extra_apps:
    from apps.landing import views as landing_views
    root_view = landing_views.index
else:
    log.info('apps.landing is not installed. So /login is set as root (/) url.')
    root_view = authentication_views.login

urlpatterns += [
    url(r'^$', root_view),
]
<|endoftext|>