<commit_before># Based on Drake's drake.bzl file,
# https://github.com/RobotLocomotion/drake/blob/master/tools/drake.bzl.

DREAL_VERSION = "4.18.10.1"

DREAL_PREFIX = "opt/dreal/%s" % DREAL_VERSION

PYTHON_VERSION_STRING = select({
    "@dreal//tools:py3config": "3",
    "//conditions:default": "2.7",
})

PYTHON_PACKAGE_DIR = "lib/python" + PYTHON_VERSION_STRING + "/site-packages"

# The CXX_FLAGS will be enabled for all C++ rules in the project
# building with any compiler.
CXX_FLAGS = [
    "-Werror=all",
    "-Werror=attributes",
    "-Werror=ignored-qualifiers",
    "-Werror=overloaded-virtual",
    "-Werror=old-style-cast",
    "-Werror=deprecated",
    "-Werror=deprecated-declarations",
]

# The CLANG_FLAGS will be enabled for all C++ rules in the project when
# building with clang.
CLANG_FLAGS = CXX_FLAGS + [
    "-Werror=shadow",
    "-Werror=inconsistent-missing-override",
    "-Werror=sign-compare",
    "-Werror=return-stack-address",
    "-Werror=non-virtual-dtor",
]

# The GCC_FLAGS will be enabled for all C++ rules in the project when
# building with gcc.
GCC_FLAGS = CXX_FLAGS + [
    "-Werror=extra",
    "-Werror=return-local-addr",
    "-Werror=non-virtual-dtor",
    "-Werror=unused-but-set-parameter",
    "-Werror=logical-op",
]

# The GCC_CC_TEST_FLAGS will be enabled for all cc_test rules in the project
# when building with gcc.
GCC_CC_TEST_FLAGS = [
    "-Wno-unused-parameter",
]

def _platform_copts(rule_copts, cc_test = 0):
    """Returns both the rule_copts, and platform-specific copts.

    When cc_test=1, the GCC_CC_TEST_FLAGS will be added.  It should only be set
    to 1 from cc_test rules or rules that are boil down to cc_test rules.
    """
    extra_gcc_flags = []
    if cc_test:
        extra_gcc_flags = GCC_CC_TEST_FLAGS
    return select({
        "//tools:gcc": GCC_FLAGS + extra_gcc_flags + rule_copts,
        "//tools:clang": CLANG_FLAGS + rule_copts,
        "//tools:apple": CLANG_FLAGS + rule_copts,
        "//conditions:default": rule_copts,
    })

def _check_library_deps_blacklist(name, deps):
    """Report an error if a library should not use something from deps."""
    if not deps:
        return
    if type(deps) != "list":
        # We can't handle select() yet.
        return
    for dep in deps:
        if dep.endswith(":main"):
            fail("The cc_library '" + name + "' must not depend on a :main " +
                 "function from a cc_library; only cc_binary program should " +
                 "have a main function")

def dreal_cc_library(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ library.
    """
    _check_library_deps_blacklist(name, deps)
    native.cc_library(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        **kwargs
    )

def _make_search_paths(prefix, levels_to_root):
    return ",".join(
        [
            "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level))
            for search_level in range(levels_to_root + 1)
        ],
    )

def dreal_pybind_library(
        name,
        py_srcs = [],
        py_deps = [],
        cc_srcs = [],
        cc_deps = []):
    """Creates a rule to declare a pybind library.

    Note that `cc_deps` should includes header-only dependencies.
    """
    cc_so_name = "_" + name + ".so"

    # The last +3 is for "lib/python*/site-packages".
    levels_to_root = native.package_name().count("/") + name.count("/") + 3
    dreal_cc_binary(
        name = cc_so_name,
        srcs = cc_srcs + [
            "//:libdreal.so",
        ],
        linkshared = 1,
        linkstatic = 1,
        deps = cc_deps + [
            "//:dreal_shared_library",
            "@pybind11",
        ],
        copts = [
            # "-fvisibility=hidden",
        ],
        linkopts = select({
            "@dreal//tools:linux": [
                "-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
            ],
            "@//conditions:default": [],
        }),
    )
    native.py_library(
        name = name,
        srcs = py_srcs,
        data = [
            cc_so_name,
        ],
        deps = py_deps,
        srcs_version = "PY2AND3",
        visibility = ["//dreal:__subpackages__"],
    )

def dreal_cc_binary(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        testonly = 0,
        add_test_rule = 0,
        test_rule_args = [],
        test_rule_size = None,
        **kwargs):
    """Creates a rule to declare a C++ binary.

    If you wish to create a smoke-test demonstrating that your binary runs
    without crashing, supply add_test_rule=1. Note that if you wish to do
    this, you should consider suppressing that urge, and instead writing real
    tests. The smoke-test will be named <name>_test. You may override cc_test
    defaults using test_rule_args=["-f", "--bar=42"] or test_rule_size="baz".
    """
    native.cc_binary(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        testonly = testonly,
        **kwargs
    )

    if add_test_rule:
        dreal_cc_test(
            name = name + "_test",
            hdrs = hdrs,
            srcs = srcs,
            deps = deps,
            copts = copts,
            size = test_rule_size,
            testonly = testonly,
            args = test_rule_args,
            **kwargs
        )

def dreal_cc_test(
        name,
        size = None,
        srcs = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ unit test.

    Note that for almost all cases, dreal_cc_googletest should be
    used, instead of this rule.

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename
    convention.

    """
    if size == None:
        size = "small"
    if srcs == None:
        srcs = ["test/%s.cc" % name]
    native.cc_test(
        name = name,
        size = size,
        srcs = srcs,
        copts = _platform_copts(copts, cc_test = 1),
        **kwargs
    )

def dreal_cc_googletest(
        name,
        deps = None,
        use_default_main = True,
        **kwargs):
    """Creates a rule to declare a C++ unit test using googletest.

    Always adds a deps= entry for googletest main (@gtest//:main).

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename convention.
    By default, sets use_default_main=True to use GTest's main, via
    @gtest//:main. Otherwise, it will depend on @gtest//:without_main.
    """
    if deps == None:
        deps = []
    if use_default_main:
        deps.append("@gtest//:main")
    else:
        deps.append("@gtest//:without_main")
    dreal_cc_test(
        name = name,
        deps = deps,
        **kwargs
    )

def smt2_test(
        name,
        tags = [],
        **kwargs):
    """Create smt2 test."""
    smt2 = name + ".smt2"
    expected = smt2 + ".expected"
    data_files = native.glob([
        smt2 + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % smt2,
            "$(location %s)" % expected,
        ],
        tags = tags + ["smt2"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

def dr_test(
        name,
        args = [],
        **kwargs):
    """Create dr test."""
    dr = name + ".dr"
    expected = dr + ".expected"
    data_files = native.glob([
        dr + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % dr,
            "$(location %s)" % expected,
        ] + args,
        tags = ["dr"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

# Generate a file with specified content
def _generate_file_impl(ctx):
    ctx.actions.write(output = ctx.outputs.out, content = ctx.attr.content)

dreal_generate_file = rule(
    attrs = {
        "content": attr.string(),
        "out": attr.output(mandatory = True),
    },
    output_to_genfiles = True,
    implementation = _generate_file_impl,
)
<commit_msg>bazel(tools/dreal.bzl): Sort/Clean up CXX flags<commit_after># Based on Drake's drake.bzl file,
# https://github.com/RobotLocomotion/drake/blob/master/tools/drake.bzl.

DREAL_VERSION = "4.18.10.1"

DREAL_PREFIX = "opt/dreal/%s" % DREAL_VERSION

PYTHON_VERSION_STRING = select({
    "@dreal//tools:py3config": "3",
    "//conditions:default": "2.7",
})

PYTHON_PACKAGE_DIR = "lib/python" + PYTHON_VERSION_STRING + "/site-packages"

# The CXX_FLAGS will be enabled for all C++ rules in the project
# building with any compiler.
CXX_FLAGS = [
    "-Werror=all",
    "-Werror=attributes",
    "-Werror=deprecated",
    "-Werror=deprecated-declarations",
    "-Werror=ignored-qualifiers",
    "-Werror=old-style-cast",
    "-Werror=overloaded-virtual",
    "-Werror=shadow",
]

# The CLANG_FLAGS will be enabled for all C++ rules in the project when
# building with clang.
CLANG_FLAGS = CXX_FLAGS + [
    "-Werror=absolute-value",
    "-Werror=inconsistent-missing-override",
    "-Werror=non-virtual-dtor",
    "-Werror=return-stack-address",
    "-Werror=sign-compare",
]

# The GCC_FLAGS will be enabled for all C++ rules in the project when
# building with gcc.
GCC_FLAGS = CXX_FLAGS + [
    "-Werror=extra",
    "-Werror=logical-op",
    "-Werror=non-virtual-dtor",
    "-Werror=return-local-addr",
    "-Werror=unused-but-set-parameter",
]

# The GCC_CC_TEST_FLAGS will be enabled for all cc_test rules in the project
# when building with gcc.
GCC_CC_TEST_FLAGS = [
    "-Wno-unused-parameter",
]

def _platform_copts(rule_copts, cc_test = 0):
    """Returns both the rule_copts, and platform-specific copts.

    When cc_test=1, the GCC_CC_TEST_FLAGS will be added.  It should only be set
    to 1 from cc_test rules or rules that are boil down to cc_test rules.
    """
    extra_gcc_flags = []
    if cc_test:
        extra_gcc_flags = GCC_CC_TEST_FLAGS
    return select({
        "//tools:gcc": GCC_FLAGS + extra_gcc_flags + rule_copts,
        "//tools:clang": CLANG_FLAGS + rule_copts,
        "//tools:apple": CLANG_FLAGS + rule_copts,
        "//conditions:default": rule_copts,
    })

def _check_library_deps_blacklist(name, deps):
    """Report an error if a library should not use something from deps."""
    if not deps:
        return
    if type(deps) != "list":
        # We can't handle select() yet.
        return
    for dep in deps:
        if dep.endswith(":main"):
            fail("The cc_library '" + name + "' must not depend on a :main " +
                 "function from a cc_library; only cc_binary program should " +
                 "have a main function")

def dreal_cc_library(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ library.
    """
    _check_library_deps_blacklist(name, deps)
    native.cc_library(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        **kwargs
    )

def _make_search_paths(prefix, levels_to_root):
    return ",".join(
        [
            "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level))
            for search_level in range(levels_to_root + 1)
        ],
    )

def dreal_pybind_library(
        name,
        py_srcs = [],
        py_deps = [],
        cc_srcs = [],
        cc_deps = []):
    """Creates a rule to declare a pybind library.

    Note that `cc_deps` should includes header-only dependencies.
    """
    cc_so_name = "_" + name + ".so"

    # The last +3 is for "lib/python*/site-packages".
    levels_to_root = native.package_name().count("/") + name.count("/") + 3
    dreal_cc_binary(
        name = cc_so_name,
        srcs = cc_srcs + [
            "//:libdreal.so",
        ],
        linkshared = 1,
        linkstatic = 1,
        deps = cc_deps + [
            "//:dreal_shared_library",
            "@pybind11",
        ],
        copts = [
            # "-fvisibility=hidden",
        ],
        linkopts = select({
            "@dreal//tools:linux": [
                "-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
            ],
            "@//conditions:default": [],
        }),
    )
    native.py_library(
        name = name,
        srcs = py_srcs,
        data = [
            cc_so_name,
        ],
        deps = py_deps,
        srcs_version = "PY2AND3",
        visibility = ["//dreal:__subpackages__"],
    )

def dreal_cc_binary(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        testonly = 0,
        add_test_rule = 0,
        test_rule_args = [],
        test_rule_size = None,
        **kwargs):
    """Creates a rule to declare a C++ binary.

    If you wish to create a smoke-test demonstrating that your binary runs
    without crashing, supply add_test_rule=1. Note that if you wish to do
    this, you should consider suppressing that urge, and instead writing real
    tests. The smoke-test will be named <name>_test. You may override cc_test
    defaults using test_rule_args=["-f", "--bar=42"] or test_rule_size="baz".
    """
    native.cc_binary(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        testonly = testonly,
        **kwargs
    )

    if add_test_rule:
        dreal_cc_test(
            name = name + "_test",
            hdrs = hdrs,
            srcs = srcs,
            deps = deps,
            copts = copts,
            size = test_rule_size,
            testonly = testonly,
            args = test_rule_args,
            **kwargs
        )

def dreal_cc_test(
        name,
        size = None,
        srcs = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ unit test.

    Note that for almost all cases, dreal_cc_googletest should be
    used, instead of this rule.

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename
    convention.

    """
    if size == None:
        size = "small"
    if srcs == None:
        srcs = ["test/%s.cc" % name]
    native.cc_test(
        name = name,
        size = size,
        srcs = srcs,
        copts = _platform_copts(copts, cc_test = 1),
        **kwargs
    )

def dreal_cc_googletest(
        name,
        deps = None,
        use_default_main = True,
        **kwargs):
    """Creates a rule to declare a C++ unit test using googletest.

    Always adds a deps= entry for googletest main (@gtest//:main).

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename convention.
    By default, sets use_default_main=True to use GTest's main, via
    @gtest//:main. Otherwise, it will depend on @gtest//:without_main.
    """
    if deps == None:
        deps = []
    if use_default_main:
        deps.append("@gtest//:main")
    else:
        deps.append("@gtest//:without_main")
    dreal_cc_test(
        name = name,
        deps = deps,
        **kwargs
    )

def smt2_test(
        name,
        tags = [],
        **kwargs):
    """Create smt2 test."""
    smt2 = name + ".smt2"
    expected = smt2 + ".expected"
    data_files = native.glob([
        smt2 + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % smt2,
            "$(location %s)" % expected,
        ],
        tags = tags + ["smt2"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

def dr_test(
        name,
        args = [],
        **kwargs):
    """Create dr test."""
    dr = name + ".dr"
    expected = dr + ".expected"
    data_files = native.glob([
        dr + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % dr,
            "$(location %s)" % expected,
        ] + args,
        tags = ["dr"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

# Generate a file with specified content
def _generate_file_impl(ctx):
    ctx.actions.write(output = ctx.outputs.out, content = ctx.attr.content)

dreal_generate_file = rule(
    attrs = {
        "content": attr.string(),
        "out": attr.output(mandatory = True),
    },
    output_to_genfiles = True,
    implementation = _generate_file_impl,
)
<|endoftext|>


<commit_before>#!/usr/bin/python

# Description: The filter is used for filtering part of useless data, since the
# addr2line is very slow, so we don't need to translate them, generate a pure
# data for next step

import sys
import os
import getopt
import string

trace_file = ""

process_start = False

# To understand the basic working flow, let's see an example, if there is a call
# graph like:
#a
#|--b
#|  |--c
#|--b
#|  |--d
#|  |--d
#|  |--e

# And, the data sturcture will be like this:
#{
#    func = "xx",
#    caller = "xx",
#    next = [],
#}
#
# So the full memory data will be:
#[
#    {
#        func = "a",
#        caller = "xx",
#        next = [
#            {
#                func = "b",
#                caller = "xx",
#                next = [
#                    {
#                        func = "c",
#                        caller = "xx",
#                        next = []
#                    }
#                ]
#            },
#            {
#                func = "b",
#                caller = "xx",
#                times = 1,
#                next = [
#                    {
#                        func = "d",
#                        caller = "xx",
#                        next = []
#                    },
#                    {
#                        func = "d",
#                        caller = "xx",
#                        next = []
#                    },
#                    {
#                        func = "e",
#                        caller = "xx",
#                        next = []
#                    }
#                ]
#            }
#        ]
#    }
#]

def load_trace_file():
    try:
        file = open(trace_file, "r")

        call_graph = []
        gen_report(file, call_graph)
        dump_graph(call_graph)
    finally:
        file.close()

def create_frame(func, caller):
    return {
        'func' : func,
        'caller' : caller,
        'next' : []
    }

def should_skip_line(type):
    global process_start
    if process_start == False:
        if type == "X":
            return True
        else:
            process_start = True
            return False
    else:
        return False

def gen_report(file, call_list):
    while True:
        line = file.readline()
        if not line:
            return

        value_list = line.strip('\n').split('|')

        # get the data from one line
        type = value_list[0]

        # We should pre-cut off the top of 'X' lines, or the data will be useless
        if should_skip_line(type):
            continue

        func = value_list[1]
        caller = value_list[2]

        # we entry into next call
        if type == "E":
            frame = create_frame(func, caller)
            call_list.append(frame)

            gen_report(file, frame['next'])

        # we exit from a call
        elif type == "X":
            return

def dump_graph(call_graph):
    for frame in call_graph:
        print "E|%s|%s" % (frame['func'],
                           frame['caller'])

        dump_graph(frame['next'])

        print "X|0x0|0x0"

def usage():
    print "usage: filter.py [-f trace.txt]"

if __name__ == "__main__":
    if len(sys.argv) == 1:
        usage()
        sys.exit(1)

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:")

        for op, value in opts:
            if op == "-h":
                sys.exit(0)
            elif op == "-f":
                trace_file = value
    except:
        usage()
        sys.exit(1)

    load_trace_file()
<commit_msg>optimize filter.py, no need to buffer all the data in phase 3<commit_after>#!/usr/bin/python

# Description: The filter is used for filtering part of useless data, since the
# addr2line is very slow, so we don't need to translate them, generate a pure
# data for next step

import sys
import os
import getopt
import string

trace_file = ""

process_start = False

def load_trace_file():
    try:
        file = open(trace_file, "r")

        filter_graph(file)
    finally:
        file.close()

def should_skip_line(type):
    global process_start
    if process_start == False:
        if type == "X":
            return True
        else:
            process_start = True
            return False
    else:
        return False

def filter_graph(file):
    while True:
        line = file.readline()
        if not line:
            return

        value_list = line.strip('\n').split('|')

        # get the data from one line
        type = value_list[0]

        # We should pre-cut off the top of 'X' lines, or the data will be useless
        if should_skip_line(type):
            continue

        func = value_list[1]
        caller = value_list[2]

        # we entry into next call
        if type == "E":
            print "E|%s|%s" % (func, caller)
            filter_graph(file)

        # we exit from a call
        elif type == "X":
            print "X|%s|%s" % (func, caller)
            return

def usage():
    print "usage: filter.py -f trace.txt"

if __name__ == "__main__":
    if len(sys.argv) == 1:
        usage()
        sys.exit(1)

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:")

        for op, value in opts:
            if op == "-h":
                usage()
                sys.exit(0)
            elif op == "-f":
                trace_file = value

    except Exception, e:
        print "Fatal: " + str(e)
        usage()
        sys.exit(1)

    load_trace_file()
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from disco.bot.command import CommandError

UNITS = {
    's': lambda v: v,
    'm': lambda v: v * 60,
    'h': lambda v: v * 60 * 60,
    'd': lambda v: v * 60 * 60 * 24,
    'w': lambda v: v * 60 * 60 * 24 * 7,
}

def parse_duration(raw, source=None, negative=False, safe=False):
    if not raw:
        if safe:
            return None
        raise CommandError('Invalid duration')

    value = 0
    digits = ''

    for char in raw:
        if char.isdigit():
            digits += char
            continue

        if char not in UNITS or not digits:
            if safe:
                return None
            raise CommandError('Invalid duration')

        value += UNITS[char](int(digits))
        digits = ''

    if negative:
        value = value * -1

    return (source or datetime.utcnow()) + timedelta(seconds=value + 1)

def humanize_duration(duration, format='full'):
    now = datetime.utcnow()
    if isinstance(duration, timedelta):
        duration = datetime.utcnow() - timedelta(seconds=duration.total_seconds())
    diff_delta = duration - now
    diff = int(diff_delta.total_seconds())

    minutes, seconds = divmod(diff, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)
    weeks, days = divmod(days, 7)
    units = [weeks, days, hours, minutes, seconds]

    if format == 'full':
        unit_strs = ['week', 'day', 'hour', 'minute', 'second']
    elif format == 'short':
        unit_strs = ['w', 'd', 'h', 'm', 's']

    expires = []
    for x in range(0, 5):
        if units[x] == 0:
            continue
        else:
            if format == 'short':
                expires.append('{}{}'.format(units[x], unit_strs[x]))
            elif units[x] > 1:
                expires.append('{} {}s'.format(units[x], unit_strs[x]))
            else:
                expires.append('{} {}'.format(units[x], unit_strs[x]))

    return ', '.join(expires)
<commit_msg>[util.input] actual...fix<commit_after>from datetime import datetime, timedelta
from disco.bot.command import CommandError

UNITS = {
    's': lambda v: v,
    'm': lambda v: v * 60,
    'h': lambda v: v * 60 * 60,
    'd': lambda v: v * 60 * 60 * 24,
    'w': lambda v: v * 60 * 60 * 24 * 7,
}

def parse_duration(raw, source=None, negative=False, safe=False):
    if not raw:
        if safe:
            return None
        raise CommandError('Invalid duration')

    value = 0
    digits = ''

    for char in raw:
        if char.isdigit():
            digits += char
            continue

        if char not in UNITS or not digits:
            if safe:
                return None
            raise CommandError('Invalid duration')

        value += UNITS[char](int(digits))
        digits = ''

    if negative:
        value = value * -1

    return (source or datetime.utcnow()) + timedelta(seconds=value + 1)

def humanize_duration(duration, format='full'):
    now = datetime.utcnow()
    if isinstance(duration, timedelta):
        if duration.total_seconds() > 0:
            duration = datetime.today() + duration
        else:
            duration = datetime.utcnow() - timedelta(seconds=duration.total_seconds())
    diff_delta = duration - now
    diff = int(diff_delta.total_seconds())

    minutes, seconds = divmod(diff, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)
    weeks, days = divmod(days, 7)
    units = [weeks, days, hours, minutes, seconds]

    if format == 'full':
        unit_strs = ['week', 'day', 'hour', 'minute', 'second']
    elif format == 'short':
        unit_strs = ['w', 'd', 'h', 'm', 's']

    expires = []
    for x in range(0, 5):
        if units[x] == 0:
            continue
        else:
            if format == 'short':
                expires.append('{}{}'.format(units[x], unit_strs[x]))
            elif units[x] > 1:
                expires.append('{} {}s'.format(units[x], unit_strs[x]))
            else:
                expires.append('{} {}'.format(units[x], unit_strs[x]))

    return ', '.join(expires)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Execute compiler with stats option for all benchmarks
# Parse output
# Draw graphs

help = """
graphs.py - Generate graphs from compiler output

Use:
	graphs.py [OPTION...]

Options:
	-h,--help
		Print this help.
	--drawall
		Draw all graphs. By default the script let the user choose the information to draw.
	--stdexec
		Use standard execution. Same as --exec="Standard;"?
	--exec="DESCRIPTION;COMPILER_OPTION1 COMPILER_OPTION2 ..."
		Add execution with given compiler options. All given executions are drawn

Example:

	graphs.py --exec="Standard exec;" --exec="With all tests;--all-tests" --drawall
		Draw all graphs for both executions (Standard, and with all-tests option).

	graphs.py --stdexec
		Let the user interactively choose the information to draw from only standard execution.
"""

import sys
import io
import glob
import os
import subprocess
from pylab import *
from copy import deepcopy
from matplotlib.backends.backend_pdf import PdfPages

# Constants
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) + '/' # Current script path
LC_PATH     = SCRIPT_PATH + '../'                               # Compiler path
LC_EXEC     = 'lazy-comp'                                       # Compiler exec name
PDF_OUTPUT  = SCRIPT_PATH + 'graphs.pdf'                        # PDF output file
BENCH_PATH  = LC_PATH + 'benchmarks/*.scm'                      # Benchmarks path
BAR_COLORS  = ["#444444","#666666","#888888","#AAAAAA","#CCCCCC","#EEEEEEn"]         # Bar colors
#BAR_COLORS = ["#222222", "#666666", "#AAAAAA", "#EEEEEE"] # Paper sw15

# Parser constants, must match compiler --stats output
CSV_INDICATOR  = '--'
STAT_SEPARATOR = ':'
CSV_SEPARATOR  = ';'

# Options
DRAW_ALL = '--drawall' # Draw all graphs
STD_EXEC = '--stdexec' # Add standard execution to executions list
REF_EXEC = '--refexec' # Set reference execution for scale
SORT_EXEC = '--sortexec' # Sort

OPT_REF = False
OPT_SORT = False

# Globals
execs     = {}
printhelp = False

# Set current working directory to compiler path
os.chdir(LC_PATH)

# Get all benchmarks full path sorted by name
files = sorted(glob.glob(BENCH_PATH))

#-------------------------------------------------------------------------------------
# Utils

def num(s):
    try:
        return int(s)
    except ValueError:
        return float(s)

def WARNING(s):
	print('WARNING: ' + s)

#-------------------------------------------------------------------------------------
# Main

def setargs():
	global printhelp
	global OPT_REF
	global OPT_SORT
	if '-h' in sys.argv or '--help' in sys.argv:
		printhelp = True
	if STD_EXEC in sys.argv:
		execs['Standard'] = ''
	if REF_EXEC in sys.argv:
		OPT_REF = sys.argv[sys.argv.index(REF_EXEC)+1]
	if SORT_EXEC in sys.argv:
		OPT_SORT = sys.argv[sys.argv.index(SORT_EXEC)+1]

	for arg in sys.argv:
		if arg.startswith('--exec='):
			pair = arg[7:].split(';')
			name = pair[0]
			lcargs = pair[1].split()
			execs[name] = lcargs

def go():
	if printhelp:
		print(help)
	else:
		# 1 - run benchmarks and parse compiler output
		benchs_data = {}
		keys = []
		for ex in execs:
			ks,data = runparse(execs[ex]) # TODO : donner arguments
			if keys == []:
				keys = ks
			else:
				if len(ks) != len(keys):
					raise Exception("Error")
			benchs_data[ex] = data

		# 2 - Draw all graphs
		drawGraphs(keys,benchs_data)
		print('Done!')

# Run compiler with 'opts', parse output and return keys and data
def runparse(opts):
	print("Running with options: '" + ' '.join(opts) + "'")
	data = {}

	# Get keys
	first = files[0]
	keys = []

	for file in files:

		file_name  = os.path.basename(file)
		print(file_name + '...')

		options = [LC_PATH + LC_EXEC, file, '--stats']
		options.extend(opts) # TODO : renommer 'options'
		output = subprocess.check_output(options).decode("utf-8")

		bench_data = parseOutput(output)

		data[file_name] = bench_data

		# Get keys on first result
		if file == first:
			for key in bench_data:
				keys.append(key)

	return keys,data

#-------------------------------------------------------------------------------------
# Parser: Read stats output from compiler and return python table representation

# Read 'KEY:VALUE' stat
def readStat(stream,data,line):
	stat = line.split(STAT_SEPARATOR)
	key = stat[0].strip()
	val = num(stat[1].strip())
	# Store key/value in global data
	data[key] = val
	line = stream.readline()
	return line

# Read CSV stat
def readCSV(stream,data):
	csv = []
	# Consume CSV indicator line
	line = stream.readline()
	# Read table title
	title = line.strip()
	line = stream.readline()
	# Read table header
	header = line.split(CSV_SEPARATOR)
	for el in header:
		csv.append([el.strip()])
	# Read CSV data
	line = stream.readline()
	while not line.startswith(CSV_INDICATOR):
		linecsv = line.split(CSV_SEPARATOR)
		for i in range(0,len(linecsv)):
			csv[i].extend([num(linecsv[i].strip())]) ## THIS IS NOT EFFICIENT (for large CSV outputs)
		line = stream.readline()
	# Store key/value (title/csv) in global data
	data[title] = csv
	# Consume CSV indicator line
	line = stream.readline()
	return line

# Return python table from compiler 'output'
def parseOutput(output):
	# Data for this benchmark
	data = {}
	# Stream
	stream = io.StringIO(output)
	# Parse
	line = stream.readline()
	while line:
		# CSV table
		if line.startswith(CSV_INDICATOR):
			line = readCSV(stream,data)
		# Key/Value line
		else:
			line = readStat(stream,data,line)
	return data

#-------------------------------------------------------------------------------------
# Draw

# Draw all graphs associated to keys using benchs_data
# benchs_data contains all information for all benchmarks for all executions
# ex. benchs_data['Standard']['array1.scm']['Closures'] to get the number of
#     closures created for benchmark array1.scm using standard exec
def drawGraphs(keys,benchs_data):

	# Let user choose the graph to draw (-1 or empty for all graphs)
	if not DRAW_ALL in sys.argv:
		sortedKeys = sorted(keys)
		print('Keys:')
		print('-1: ALL')
		for i in range(0,len(sortedKeys)):
			print(' ' + str(i) + ': ' + sortedKeys[i])
		inp = input('Key to draw (all) > ')
		if not inp == '':
			choice = num(inp)
			if choice >= 0:
				keys = [sortedKeys[choice]]

	firstExec = list(benchs_data.keys())[0]
	firstBenchmark = os.path.basename(files[0])

	# Gen pdf output file
	pdf = PdfPages(PDF_OUTPUT)

	# For each key
	for key in keys:
		# CSV, NYI
		if type(benchs_data[firstExec][firstBenchmark][key]) == list:
			drawCSV(pdf,key,benchs_data)
		# Key/Value, draw graph
		else:
			print("Drawing '" + key + "'...")
			drawKeyValueGraph(pdf,key,benchs_data)

	pdf.close()

## This is a specific implementation for #stubs/#versions
## TODO: Do something generic !
def drawCSV(pdf,key,benchs_data):
	fig = plt.figure(key)
	title = key
	res = {}

	for execution in benchs_data:
		for bench in benchs_data[execution]:
			for data in benchs_data[execution][bench][key]:
			   if data[0] == '#stubs':
			     for i in range(0,len(data)-1):
			     	index = i+1
			     	numvers = i
			     	if (numvers >= 5):
			     		numvers = -1
			     	if (numvers in res):
			     		res[numvers] += data[index]
			     	else:
			     		res[numvers] = data[index]

	xvals = []
	yvals = []
	labels = []

	keys = sorted(res.keys())

	for key in keys:
		if key != 0 and key != -1:
			xvals.append(key)
			yvals.append(res[key])
			labels.append(key)

	xvals.append(len(xvals)+1)
	yvals.append(res[-1])
	labels.append('>=5')

	sum = 0
	for val in yvals:
		sum += val
	for i in range(0,len(yvals)):
		p = (yvals[i] * 100) / sum
		yvals[i] = p

	plt.title(title + ' (total=' + str(sum) + ')')

	X = np.array(xvals)
	Y = np.array(yvals)

	bar(X, +Y, 1, facecolor=BAR_COLORS[0], edgecolor='white', label=key, zorder=10)

	axes = gca()
	axes.get_xaxis().set_visible(False)

	# Draw grid
	axes = gca()
	axes.grid(True, zorder=1, color="#707070")
	axes.set_axisbelow(True) # Keep grid under the axes

	for i in range(0,len(labels)):
		text(X[i]+0.25, -0.0, labels[i], ha='right', va='top')

	# print(xvals)
	# print(yvals)
	# print(labels)
	# print(res)
	pdf.savefig(fig)

# Draw graph for given key
# Y: values for this key
# X: benchmarks
def drawKeyValueGraph(pdf,key,benchs_data):
	fig = plt.figure(key,figsize=(20,7))
	#plt.title(key)

	exec_ref = ''

	# Number of benchmarks
	firstExec = list(benchs_data.keys())[0]
	n = len(benchs_data[firstExec])
	X = np.arange(n) # X set is [0, 1, ..., n-1]

	Ys = {}
	# pour chaque executions
	for d in benchs_data:
		Y = []
		# pour chaque benchmark
		for f in files:
			Y.extend([benchs_data[d][os.path.basename(f)][key]])
		# Transforme en tableau numpy
		Y = np.array(Y)
		Ys[d] = Y

	width = 1 / (len(Ys)+1)

	#----------
	# TODO: move to external fn
	# Use a reference execution. All values for this exec are 100%
	# Values for others executions are computed from this reference exec
	if OPT_REF:
		exec_ref = OPT_REF # Reference execution (100%)
		Y2 = deepcopy(Ys)      # Deep copy of Y values
		# Set all references to 100
		for v in range(0,len(Y2[exec_ref])):
			Y2[exec_ref][v] = '100'
		# For each exec which is not ref exec
		candraw = True # TODO : rename
		for ex in Y2:
			if ex != exec_ref:
				for i in range(0,len(Y2[ex])):
					ref = Ys[exec_ref][i]
					cur = Ys[ex][i]
					# We can't compute %, warning and stop
					if ref == 0:
						WARNING("Can't draw '" + key + "' using a reference execution.")
						return
					# Compute % and set
					else:
						Y2[ex][i] = (cur*100)/ref
		# Y2 are the new values to draw
		Ys = Y2
	#----------

	fileList = files
	Yvals = Ys

	# Sort Y values by a given execution
	# TODO: get the execution by command line option
	if OPT_SORT:
		fileList,Yvals = sortByExecution(Yvals,OPT_SORT)

	# Draw grid
	axes = gca()
	axes.grid(True, zorder=1, color="#707070")
	axes.set_axisbelow(True) # Keep grid under the axes

	keys = sorted(Yvals.keys())

	i = 0
	for key in keys: # todo : renommer key en exec?
		if key != exec_ref:
			Y = Yvals[key]
			color = BAR_COLORS[i];
			bar(X+(i*width), +Y, width, facecolor=color, edgecolor='white', label=key, zorder=10)
			i += 1

	# Hide X values
	axes.get_xaxis().set_visible(False)

	# # Set Y limit
	#l = len(str(max(Y2))) # number of digit of max value
	#ylim(0,max(Y2)+pow(10,l-1)) # Y is from 0 to (max + 10^i-1)
	# # Draw values for each bar
	# for x,y in zip(X,Y1):
	#     text(x+0.4, y+0.05, '%.2f' % y, ha='center', va= 'bottom')

	ylim(0,100);

	# Draw benchmark name
	for i in range(0,len(fileList)):
		text(X[i]+0.25, -1, os.path.basename(fileList[i])[:-4], rotation=90, ha='center', va='top', size='large')

	# Legend:
	# Shrink by 10% on the bottom
	box = axes.get_position()
	axes.set_position([box.x0, box.y0 + box.height * 0.15, box.width, box.height * 0.85])
	# Put a legend below axis
	legend(loc='upper center', bbox_to_anchor=(0., 0., 1., -0.21), prop={'size':12}, ncol=len(keys), mode='expand', borderaxespad=0.)

	# Save to pdf
	pdf.savefig(fig)

#-------------------------------------------------------------------------------------
# Manage Y values

# Sort Y values by values from a specific execution
def sortByExecution(Ys,execref):

	# Pseudo-decorate: Change data layout to allow the useof sort()
	decorated = []
	for fileIndex in range(0,len(files)):
		r = [] # List of results for current file
		for execution in Ys:
			r.extend([execution,Ys[execution][fileIndex]])
		r.append(files[fileIndex])
		decorated.append(r)

	# Sort
	i = decorated[0].index(execref)
	decorated = sorted(decorated,key=lambda el: el[i+1])
	# Pseudo-undecorate: Restore previous layout with sorted data
	undecorated = {}
	ordered_files = []
	i = 0;
	while not decorated[0][i] in files:
		execution = decorated[0][i]
		vals = []
		# For each data associated to file
		for el in decorated:
			vals.append(el[i+1])
			filepath = el[len(el)-1]
			if not filepath in ordered_files:
				ordered_files.append(filepath)
		undecorated[execution] = np.asarray(vals);
		i+=2

	return(ordered_files,undecorated)

#-------------------------------------------------------------------------------------

setargs()
go()<commit_msg>update graph script<commit_after>#!/usr/bin/env python

# Execute compiler with stats option for all benchmarks
# Parse output
# Draw graphs

help = """
graphs.py - Generate graphs from compiler output

Use:
	graphs.py [OPTION...]

Options:
	-h,--help
		Print this help.
	--drawall
		Draw all graphs. By default the script let the user choose the information to draw.
	--stdexec
		Use standard execution. Same as --exec="Standard;"?
	--exec="DESCRIPTION;COMPILER_OPTION1 COMPILER_OPTION2 ..."
		Add execution with given compiler options. All given executions are drawn

Example:

	graphs.py --exec="Standard exec;" --exec="With all tests;--all-tests" --drawall
		Draw all graphs for both executions (Standard, and with all-tests option).

	graphs.py --stdexec
		Let the user interactively choose the information to draw from only standard execution.
"""

import sys
import io
import glob
import os
import subprocess
from pylab import *
from copy import deepcopy
from matplotlib.backends.backend_pdf import PdfPages

# Constants
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) + '/' # Current script path
LC_PATH     = SCRIPT_PATH + '../'                               # Compiler path
LC_EXEC     = 'lazy-comp'                                       # Compiler exec name
PDF_OUTPUT  = SCRIPT_PATH + 'graphs.pdf'                        # PDF output file
BENCH_PATH  = LC_PATH + 'benchmarks/*.scm'                      # Benchmarks path
BAR_COLORS  = ["#444444","#666666","#888888","#AAAAAA","#CCCCCC","#EEEEEEn"]         # Bar colors
#BAR_COLORS = ["#222222", "#666666", "#AAAAAA", "#EEEEEE"] # Paper sw15

# Parser constants, must match compiler --stats output
CSV_INDICATOR  = '--'
STAT_SEPARATOR = ':'
CSV_SEPARATOR  = ';'

# Options
DRAW_ALL = '--drawall' # Draw all graphs
STD_EXEC = '--stdexec' # Add standard execution to executions list
REF_EXEC = '--refexec' # Set reference execution for scale
SORT_EXEC = '--sortexec' # Sort

OPT_REF = False
OPT_SORT = False

# Globals
execs     = {}
printhelp = False

# Set current working directory to compiler path
os.chdir(LC_PATH)

# Get all benchmarks full path sorted by name
files = sorted(glob.glob(BENCH_PATH))

#-------------------------------------------------------------------------------------
# Utils

def num(s):
    try:
        return int(s)
    except ValueError:
        return float(s)

def WARNING(s):
	print('WARNING: ' + s)

#-------------------------------------------------------------------------------------
# Main

def setargs():
	global printhelp
	global OPT_REF
	global OPT_SORT
	if '-h' in sys.argv or '--help' in sys.argv:
		printhelp = True
	if STD_EXEC in sys.argv:
		execs['Standard'] = ''
	if REF_EXEC in sys.argv:
		OPT_REF = sys.argv[sys.argv.index(REF_EXEC)+1]
	if SORT_EXEC in sys.argv:
		OPT_SORT = sys.argv[sys.argv.index(SORT_EXEC)+1]

	for arg in sys.argv:
		if arg.startswith('--exec='):
			pair = arg[7:].split(';')
			name = pair[0]
			lcargs = pair[1].split()
			execs[name] = lcargs

def go():
	if printhelp:
		print(help)
	else:
		# 1 - run benchmarks and parse compiler output
		benchs_data = {}
		keys = []
		for ex in execs:
			ks,data = runparse(execs[ex]) # TODO : donner arguments
			if keys == []:
				keys = ks
			else:
				if len(ks) != len(keys):
					raise Exception("Error")
			benchs_data[ex] = data

		# 2 - Draw all graphs
		drawGraphs(keys,benchs_data)
		print('Done!')

# Run compiler with 'opts', parse output and return keys and data
def runparse(opts):
	print("Running with options: '" + ' '.join(opts) + "'")
	data = {}

	# Get keys
	first = files[0]
	keys = []

	for file in files:

		file_name  = os.path.basename(file)
		print(file_name + '...')

		options = [LC_PATH + LC_EXEC, file, '--stats']
		options.extend(opts) # TODO : renommer 'options'
		output = subprocess.check_output(options).decode("utf-8")

		bench_data = parseOutput(output)

		data[file_name] = bench_data

		# Get keys on first result
		if file == first:
			for key in bench_data:
				keys.append(key)

	return keys,data

#-------------------------------------------------------------------------------------
# Parser: Read stats output from compiler and return python table representation

# Read 'KEY:VALUE' stat
def readStat(stream,data,line):
	stat = line.split(STAT_SEPARATOR)
	key = stat[0].strip()
	val = num(stat[1].strip())
	# Store key/value in global data
	data[key] = val
	line = stream.readline()
	return line

# Read CSV stat
def readCSV(stream,data):
	csv = []
	# Consume CSV indicator line
	line = stream.readline()
	# Read table title
	title = line.strip()
	line = stream.readline()
	# Read table header
	header = line.split(CSV_SEPARATOR)
	for el in header:
		csv.append([el.strip()])
	# Read CSV data
	line = stream.readline()
	while not line.startswith(CSV_INDICATOR):
		linecsv = line.split(CSV_SEPARATOR)
		for i in range(0,len(linecsv)):
			csv[i].extend([num(linecsv[i].strip())]) ## THIS IS NOT EFFICIENT (for large CSV outputs)
		line = stream.readline()
	# Store key/value (title/csv) in global data
	data[title] = csv
	# Consume CSV indicator line
	line = stream.readline()
	return line

# Return python table from compiler 'output'
def parseOutput(output):
	# Data for this benchmark
	data = {}
	# Stream
	stream = io.StringIO(output)
	# Parse
	line = stream.readline()
	while line:
		# CSV table
		if line.startswith(CSV_INDICATOR):
			line = readCSV(stream,data)
		# Key/Value line
		else:
			line = readStat(stream,data,line)
	return data

#-------------------------------------------------------------------------------------
# Draw

# Draw all graphs associated to keys using benchs_data
# benchs_data contains all information for all benchmarks for all executions
# ex. benchs_data['Standard']['array1.scm']['Closures'] to get the number of
#     closures created for benchmark array1.scm using standard exec
def drawGraphs(keys,benchs_data):

	# Let user choose the graph to draw (-1 or empty for all graphs)
	if not DRAW_ALL in sys.argv:
		sortedKeys = sorted(keys)
		print('Keys:')
		print('-1: ALL')
		for i in range(0,len(sortedKeys)):
			print(' ' + str(i) + ': ' + sortedKeys[i])
		inp = input('Key to draw (all) > ')
		if not inp == '':
			choice = num(inp)
			if choice >= 0:
				keys = [sortedKeys[choice]]

	firstExec = list(benchs_data.keys())[0]
	firstBenchmark = os.path.basename(files[0])

	# Gen pdf output file
	pdf = PdfPages(PDF_OUTPUT)

	# For each key
	for key in keys:
		# CSV, NYI
		if type(benchs_data[firstExec][firstBenchmark][key]) == list:
			drawCSV(pdf,key,benchs_data)
		# Key/Value, draw graph
		else:
			print("Drawing '" + key + "'...")
			drawKeyValueGraph(pdf,key,benchs_data)

	pdf.close()

## This is a specific implementation for #stubs/#versions
## TODO: Do something generic !
def drawCSV(pdf,key,benchs_data):
	fig = plt.figure(key)
	title = key
	res = {}

	for execution in benchs_data:
		for bench in benchs_data[execution]:
			for data in benchs_data[execution][bench][key]:
			   if data[0] == '#stubs':
			     for i in range(0,len(data)-1):
			     	index = i+1
			     	numvers = i
			     	if (numvers >= 5):
			     		numvers = -1
			     	if (numvers in res):
			     		res[numvers] += data[index]
			     	else:
			     		res[numvers] = data[index]

	xvals = []
	yvals = []
	labels = []

	keys = sorted(res.keys())

	for key in keys:
		if key != 0 and key != -1:
			xvals.append(key)
			yvals.append(res[key])
			labels.append(key)

	xvals.append(len(xvals)+1)
	yvals.append(res[-1])
	labels.append('>=5')

	sum = 0
	for val in yvals:
		sum += val
	for i in range(0,len(yvals)):
		p = (yvals[i] * 100) / sum
		yvals[i] = p

	plt.title(title + ' (total=' + str(sum) + ')')

	X = np.array(xvals)
	Y = np.array(yvals)

	bar(X, +Y, 1, facecolor=BAR_COLORS[0], edgecolor='white', label=key, zorder=10)

	axes = gca()
	axes.get_xaxis().set_visible(False)

	# Draw grid
	axes = gca()
	axes.grid(True, zorder=1, color="#707070")
	axes.set_axisbelow(True) # Keep grid under the axes

	for i in range(0,len(labels)):
		text(X[i]+0.25, -0.0, labels[i], ha='right', va='top')

	# print(xvals)
	# print(yvals)
	# print(labels)
	# print(res)
	pdf.savefig(fig)

# Draw graph for given key
# Y: values for this key
# X: benchmarks
def drawKeyValueGraph(pdf,key,benchs_data):
	fig = plt.figure(key,figsize=(20,7))
	plt.title(key)

	exec_ref = ''

	# Number of benchmarks
	firstExec = list(benchs_data.keys())[0]
	n = len(benchs_data[firstExec])
	X = np.arange(n) # X set is [0, 1, ..., n-1]

	Ys = {}
	# pour chaque executions
	for d in benchs_data:
		Y = []
		# pour chaque benchmark
		for f in files:
			Y.extend([benchs_data[d][os.path.basename(f)][key]])
		# Transforme en tableau numpy
		Y = np.array(Y)
		Ys[d] = Y

	width = 1 / (len(Ys)+1)

	#----------
	# TODO: move to external fn
	# Use a reference execution. All values for this exec are 100%
	# Values for others executions are computed from this reference exec
	if OPT_REF:
		exec_ref = OPT_REF # Reference execution (100%)
		Y2 = deepcopy(Ys)      # Deep copy of Y values
		# Set all references to 100
		for v in range(0,len(Y2[exec_ref])):
			Y2[exec_ref][v] = '100'
		# For each exec which is not ref exec
		candraw = True # TODO : rename
		for ex in Y2:
			if ex != exec_ref:
				for i in range(0,len(Y2[ex])):
					ref = Ys[exec_ref][i]
					cur = Ys[ex][i]
					# We can't compute %, warning and stop
					if ref == 0:
						WARNING("Can't draw '" + key + "' using a reference execution.")
						return
					# Compute % and set
					else:
						Y2[ex][i] = (cur*100)/ref
		# Y2 are the new values to draw
		Ys = Y2
	#----------

	fileList = files
	Yvals = Ys

	# Sort Y values by a given execution
	# TODO: get the execution by command line option
	if OPT_SORT:
		fileList,Yvals = sortByExecution(Yvals,OPT_SORT)

	# Draw grid
	axes = gca()
	axes.grid(True, zorder=1, color="#707070")
	axes.set_axisbelow(True) # Keep grid under the axes

	keys = sorted(Yvals.keys())

	i = 0
	for key in keys: # todo : renommer key en exec?
		if key != exec_ref:
			Y = Yvals[key]
			color = BAR_COLORS[i];
			bar(X+(i*width), +Y, width, facecolor=color, edgecolor='white', label=key, zorder=10)
			i += 1

	# Hide X values
	axes.get_xaxis().set_visible(False)

	# # Set Y limit
	#l = len(str(max(Y2))) # number of digit of max value
	#ylim(0,max(Y2)+pow(10,l-1)) # Y is from 0 to (max + 10^i-1)
	# # Draw values for each bar
	# for x,y in zip(X,Y1):
	#     text(x+0.4, y+0.05, '%.2f' % y, ha='center', va= 'bottom')
	#ylim(0,100);

	# Draw benchmark name
	for i in range(0,len(fileList)):
		text(X[i]+0.25, -1, os.path.basename(fileList[i])[:-4], rotation=90, ha='center', va='top', size='large')

	# Legend:
	# Shrink by 10% on the bottom
	box = axes.get_position()
	axes.set_position([box.x0, box.y0 + box.height * 0.15, box.width, box.height * 0.85])
	# Put a legend below axis
	legend(loc='upper center', bbox_to_anchor=(0., 0., 1., -0.21), prop={'size':12}, ncol=len(keys), mode='expand', borderaxespad=0.)

	# Save to pdf
	pdf.savefig(fig)

#-------------------------------------------------------------------------------------
# Manage Y values

# Sort Y values by values from a specific execution
def sortByExecution(Ys,execref):

	# Pseudo-decorate: Change data layout to allow the useof sort()
	decorated = []
	for fileIndex in range(0,len(files)):
		r = [] # List of results for current file
		for execution in Ys:
			r.extend([execution,Ys[execution][fileIndex]])
		r.append(files[fileIndex])
		decorated.append(r)

	# Sort
	i = decorated[0].index(execref)
	decorated = sorted(decorated,key=lambda el: el[i+1])
	# Pseudo-undecorate: Restore previous layout with sorted data
	undecorated = {}
	ordered_files = []
	i = 0;
	while not decorated[0][i] in files:
		execution = decorated[0][i]
		vals = []
		# For each data associated to file
		for el in decorated:
			vals.append(el[i+1])
			filepath = el[len(el)-1]
			if not filepath in ordered_files:
				ordered_files.append(filepath)
		undecorated[execution] = np.asarray(vals);
		i+=2

	return(ordered_files,undecorated)

#-------------------------------------------------------------------------------------

setargs()
go()<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
    A tool to automatically query/close bugs after a git merge
"""

from launchpadlib.launchpad import Launchpad
import argparse
import logging
import itertools
import os
import re
import shlex
import subprocess

# regexps
RE_BUGS = re.compile('\[f=(.*?)\]')
RE_REVIEWER = re.compile('\[r=(.*?)\]')

CACHE_DIR = os.path.expanduser("~/.launchpadlib/cache/")

def parse_and_login(time=None):
    """
        Convenience function to parse and login to launchpad
    """
    launchpad = None
    commits_output = None

    if time:
        commits_output = CommitsOutput.since(time)

        launchpad = Launchpad.login_with('OpenQuake Bug Bot', 'staging',
                CACHE_DIR)

    return launchpad, commits_output

class CommitsOutput(object):
    """
        Helper class for git commit output
    """

    @staticmethod
    def since(time):
        """
            Reads from git
            extracts the output from a pipe
            returns read lines
        """
        git_cmd = shlex.split("git log --merges --since='%s'" % time)
        git = subprocess.Popen(
            git_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        grep_cmd = shlex.split("grep '\[r='")
        grep = subprocess.Popen(grep_cmd, stdin=git.stdout,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        out, err = grep.communicate()

        if grep.returncode != 0 and len(out):
            error = ("%s terminated with exit code: %s\n%s"
                     % (' '.join(git_cmd) + '|' + ' '.join(grep_cmd),
                         grep.returncode, err))
            logging.error(error)
            raise Exception(error)

        return [line.strip() for line in out.splitlines()]

# A serie of ArgumentParser action(s) that are triggered by parse_args()
def fix_committed(launchpad, commit_line):
    class FixCommitted(argparse.Action):
        """ Changes the status of a bug to Fix Committed when it is in
            the master repository (i.e. merged)
        """

        def __call__(self, parser, namespace, values, option_string=None):

            print 'FixCommitted: %s %r %r %r' % (parser, namespace,
                    values, option_string)

            changed_bugs = []
            if namespace.time:
                bugs = launchpad_lookup(launchpad,
                        filter_bugs(commit_line))

                for bug in bugs:
                    if bug.bug_tasks[0].status != "Fix Committed":
                        bug.bug_tasks[0].status = 'Fix Committed'
                        changed_bugs.append(bug)
            return changed_bugs
    return FixCommitted

class FixReleased(argparse.Action):
    """
        Changes the status of a bug to Fix Released when the release packages
        are built
    """
    def __call__(self, parser, namespace, values, option_string=None):
        print 'FixReleased: %r %r %r' % (namespace, values, option_string)
        setattr(namespace, self.dest, values)

class ChangeLog(argparse.Action):
    """
        Prints on screen the ChangeLog since a time
    """
    def __call__(self, parser, namespace, values, option_string=None):
        print 'ChangeLog: %r %r %r' % (namespace, values, option_string)

        launchpad, commits_output = parse_and_login(namespace.time)

        for commit_line in commits_output:
            reviewers = filter_reviewers(commit_line)
            bugs = launchpad_lookup(launchpad,
                    filter_bugs(commit_line))
            for bug in bugs:
                entry = """Fix-committed:  %s
Summary:        %s
Url:            %s
Reviewed-by:    %s
Closed-by:      %s
""" % (
                    str(bug.bug_tasks[0].date_fix_committed),
                    str(bug.title),
                    str(bug.bug_tasks[0].web_link),
                    str(reviewers),
                    str(bug.bug_tasks[0].assignee.name))

                print entry
        setattr(namespace, self.dest, values)

def arg_parse():
    """
        Prepares ArgumentParser for argument checking/triggering
    """
    parser = argparse.ArgumentParser(description=__doc__, add_help=False)

    parser.add_argument('-t', '--time', dest="time",
            help="time: timeframe (i.e '1 month', '1 week', etc) \
                with quotes",
            metavar="TIME",
            )

    # pre-parse time to provide it to the next parser with custom methods in
    # custom argparse actions
    args, _ = parser.parse_known_args()

    # hack to make the option required after pre-parsing
    parser._actions[0].required = True

    # merges the two parsers and instantiate the second final parser
    action_parser = argparse.ArgumentParser(description=__doc__,
            add_help=True, parents=[parser])

    (launchpad, commits_output) = parse_and_login(args.time)
    action_parser.add_argument('-c', '--fix-committed',
            action=fix_committed(launchpad, commits_output),
            help="Invoked from the CI gets from a git repository every \
                bug and changes status on launchpad to Fix Committed",
            nargs=0,
            dest='fix_committed',
            required=False)

    action_parser.add_argument('-r', '--fix-released', action=FixReleased,
            help="Invoked from the ppa build fetches from a git repository \
                every with Fix Committed status and changes it to \
                Fix Released",
            nargs=0,
            required=False)

    action_parser.add_argument('-l', '--change-log', action=ChangeLog,
            help="Invoked from the CI gets from a git repository every \
                bug and changes status on launchpad to Fix Committed",
            nargs=0,
            required=False)

    if args.time:
        action_args = action_parser.parse_args()
        print action_args
    else:
        print action_parser.print_help()
        return (None, None)

    return action_parser, action_args

def filter_reviewers(reviewers):
    """
        Little helper function to filter reviewers
    """

    filtered_reviewers = [reviewer for reviewer in
            RE_REVIEWER.findall(reviewers).pop(0).split(',')]
    return ','.join(filtered_reviewers)

def launchpad_lookup(lp, bugs):
    try:
        return [lp.bugs[bug] for bug in bugs]
    # Sometimes launchpad does not fetch the correct bug, we have to handle
    # this situation
    except KeyError, e:
        error_message = 'Bug not found, maybe a launchpad api error or ' \
                'staging area? bug: %s' % e
        logging.error(error_message)
        raise Exception(error_message)

def filter_bugs(commit):
    """
        Little helper function to filter bugs
    """
    bugs = RE_BUGS.findall(commit)
    if len(bugs):
        return itertools.ifilterfalse(
            lambda bug: bug.startswith('*'),
            bugs.pop(0).split(','))
    return bugs

if __name__ == '__main__':
    (PARSER, ARGS) = arg_parse()
<commit_msg>converted changelog<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
    A tool to automatically query/close bugs after a git merge
"""

from launchpadlib.launchpad import Launchpad
import argparse
import logging
import itertools
import os
import re
import shlex
import subprocess

# regexps
RE_BUGS = re.compile('\[f=(.*?)\]')
RE_REVIEWER = re.compile('\[r=(.*?)\]')

CACHE_DIR = os.path.expanduser("~/.launchpadlib/cache/")

def parse_and_login(time=None):
    """
        Convenience function to parse and login to launchpad
    """
    launchpad = None
    commits_output = []

    if time:
        commits_output = CommitsOutput.since(time)

        launchpad = Launchpad.login_with('OpenQuake Bug Bot', 'production',
                CACHE_DIR)

    return launchpad, commits_output

class CommitsOutput(object):
    """
        Helper class for git commit output
    """

    @staticmethod
    def since(time):
        """
            Reads from git
            extracts the output from a pipe
            returns read lines
        """
        git_cmd = shlex.split("git log --merges --since='%s'" % time)
        git = subprocess.Popen(
            git_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        grep_cmd = shlex.split("grep '\[r='")
        grep = subprocess.Popen(grep_cmd, stdin=git.stdout,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        out, err = grep.communicate()

        if grep.returncode != 0 and len(out):
            error = ("%s terminated with exit code: %s\n%s"
                     % (' '.join(git_cmd) + '|' + ' '.join(grep_cmd),
                         grep.returncode, err))
            logging.error(error)
            raise Exception(error)

        return [line.strip() for line in out.splitlines()]

# A serie of ArgumentParser action(s) that are triggered by parse_args()
def fix_committed(launchpad, commit_lines):
    class FixCommitted(argparse.Action):
        """ Changes the status of a bug to Fix Committed when it is in
            the master repository (i.e. merged)
        """

        def __call__(self, parser, namespace, values, option_string=None):

            print 'FixCommitted: %s %r %r %r' % (parser, namespace,
                    values, option_string)

            changed_bugs = []
            for commit_line in commit_lines:
                if namespace.time:
                    bugs = launchpad_lookup(launchpad,
                            filter_bugs(commit_line))

                    for bug in bugs:
                        if bug.bug_tasks[0].status != "Fix Committed":
                            bug.bug_tasks[0].status = 'Fix Committed'
                            changed_bugs.append(bug)
            return changed_bugs
    return FixCommitted

class FixReleased(argparse.Action):
    """
        Changes the status of a bug to Fix Released when the release packages
        are built
    """
    def __call__(self, parser, namespace, values, option_string=None):
        print 'FixReleased: %r %r %r' % (namespace, values, option_string)
        setattr(namespace, self.dest, values)

def changelog(launchpad, commit_lines):
    class ChangeLog(argparse.Action):
        """
            Prints on screen the ChangeLog since a time
        """
        def __call__(self, parser, namespace, values, option_string=None):
            print 'ChangeLog: %r %r %r' % (namespace, values, option_string)

            for commit_line in commit_lines:
                reviewers = filter_reviewers(commit_line)
                bugs = launchpad_lookup(launchpad,
                        filter_bugs(commit_line))
                for bug in bugs:
                    entry = """Fix-committed:  %s
Summary:        %s
Url:            %s
Reviewed-by:    %s
Closed-by:      %s
""" % (
                        str(bug.bug_tasks[0].date_fix_committed),
                        str(bug.title),
                        str(bug.bug_tasks[0].web_link),
                        str(reviewers),
                        str(bug.bug_tasks[0].assignee.name))

                    print entry
    return ChangeLog

def arg_parse():
    """
        Prepares ArgumentParser for argument checking/triggering
    """

    launchpad = None
    commits_output = []
    parser = argparse.ArgumentParser(description=__doc__, add_help=False)

    parser.add_argument('-t', '--time', dest="time",
            help="time: timeframe (i.e '1 month', '1 week', etc) \
                with quotes",
            metavar="TIME"
            )

    # pre-parse time to provide it to the next parser with custom methods in
    # custom argparse actions
    args, remaining_argv = parser.parse_known_args()

    parser._actions[0].required = True

    if args.time:
        if len(remaining_argv):
            (launchpad, commits_output) = parse_and_login(args.time)
        remaining_argv.extend(['-t', args.time])

    # merges the two parsers and instantiate the second final parser
    action_parser = argparse.ArgumentParser(description=__doc__,
            parents=[parser],
            formatter_class=argparse.RawDescriptionHelpFormatter,
            add_help=True)

    # hack
    action_group = action_parser.add_mutually_exclusive_group()
    action_group.add_argument('-c', '--fix-committed',
            action=fix_committed(launchpad, commits_output),
            help="Invoked from the CI gets from a git repository every \
                bug and changes status on launchpad to Fix Committed",
            nargs=0,
            dest='fix_committed',
            required=False)

    action_group.add_argument('-r', '--fix-released', action=FixReleased,
            help="Invoked from the ppa build fetches from a git repository \
                every with Fix Committed status and changes it to \
                Fix Released",
            nargs=0,
            required=False)

    action_group.add_argument('-l', '--changelog',
            action=changelog(launchpad, commits_output),
            help="Invoked from the CI gets from a git repository every \
                bug and changes status on launchpad to Fix Committed",
            nargs=0,
            required=False)

    action_parser.parse_args(remaining_argv)

    return args

def filter_reviewers(reviewers):
    """
        Little helper function to filter reviewers
    """

    filtered_reviewers = [reviewer for reviewer in
            RE_REVIEWER.findall(reviewers).pop(0).split(',')]
    return ','.join(filtered_reviewers)

def launchpad_lookup(lp, bugs):
    try:
        return [lp.bugs[bug] for bug in bugs]
    # Sometimes launchpad does not fetch the correct bug, we have to handle
    # this situation
    except KeyError, e:
        error_message = 'Bug not found, maybe a launchpad api error or ' \
                'staging area? bug: %s' % e
        logging.error(error_message)
        raise Exception(error_message)

def filter_bugs(commit):
    """
        Little helper function to filter bugs
    """
    bugs = RE_BUGS.findall(commit)
    if len(bugs):
        return itertools.ifilterfalse(
            lambda bug: bug.startswith('*'),
            bugs.pop(0).split(','))
    return bugs

if __name__ == '__main__':
    arg_parse()
<|endoftext|>


<commit_before>'''
A single-queue global scheduler for CoCrawler

global QPS value used for all hosts

remember last deadline for every host

hand out work in order, increment deadlines

'''
import time
import asyncio
import pickle
from collections import defaultdict
from operator import itemgetter
import logging
import cachetools.ttl
import json

from . import config
from . import stats

LOGGER = logging.getLogger(__name__)

class Scheduler:
    def __init__(self):
        self.q = asyncio.PriorityQueue()
        self.ridealong = {}
        self.awaiting_work = 0
        self.maxhostqps = None
        self.delta_t = None
        self.next_fetch = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds good enough for QPS=0.1 and up
        self.frozen_until = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds is longer than our typical delay
        self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
        self.delta_t = 1./self.maxhostqps
        self.max_crawled_urls = int(config.read('Crawl', 'MaxCrawledUrls') or 0) or None  # 0 => None

    def global_crawl_quota_exceeded(self):
        if ((self.max_crawled_urls is not None and
             (stats.stat_value('fetch http code=200') or 0) >= self.max_crawled_urls)):
            return True

    async def get_work(self):
        '''
        This function is called separately by each worker. It's allowed to sleep and/or requeue
        if work can't be done immediately.
        '''
        while True:
            if self.global_crawl_quota_exceeded():
                raise asyncio.CancelledError  # cancel this one worker

            try:
                work = self.q.get_nowait()
            except asyncio.queues.QueueEmpty:
                # using awaiting_work to see if all workers are idle can race with sleeping s.q.get()
                # putting it in an except clause makes sure the race is only run when
                # the queue is actually empty.
                self.awaiting_work += 1
                with stats.coroutine_state('awaiting work'):
                    work = await self.q.get()
                self.awaiting_work -= 1

            now = time.time()
            surt = work[2]
            surt_host, _, _ = surt.partition(')')
            ridealong = self.get_ridealong(surt)

            recycle, why, dt = self.do_we_recycle(now, surt, surt_host, ridealong)

            # sleep then requeue
            if recycle:
                stats.stats_sum(why+' sum', dt)
                with stats.coroutine_state(why):
                    await asyncio.sleep(dt)
                    self.q.put_nowait(work)
                    self.q.task_done()
                    continue

            # Normal case: sleep if needed, and then return the work to the caller.
            self.next_fetch[surt_host] = now + dt + self.delta_t
            if dt > 0:
                stats.stats_sum(why+' sum', dt)
                with stats.coroutine_state(why):
                    await asyncio.sleep(dt)

            return work

    def do_we_recycle(self, now, surt, surt_host, ridealong):
        recycle = False
        why = None
        dt = 0

        # does host have cached dns? XXX
        # if not, and we're The One, fetch it
        # if not, and we aren't The One, recycle

        # does host have cached robots? XXX
        # if not, and we're The One, fetch it
        # if not, and we aren't The One, recycle

        # when's the next available rate limit slot?
        now = time.time()
        if surt_host in self.next_fetch:
            dt = max(self.next_fetch[surt_host] - now, 0.)
        else:
            dt = 0
        if dt > 3.0:
            recycle = True
            why = 'scheduler ratelimit recycle'
            dt = 3.0
        elif dt > 0:
            why = 'scheduler ratelimit short sleep'

        return recycle, why, dt

    def work_done(self):
        self.q.task_done()

    def requeue_work(self, work):
        '''
        When we requeue work after a failure, we add 0.5 to the rand;
        eventually do that in here
        '''
        self.q.put_nowait(work)

    def queue_work(self, work):
        self.q.put_nowait(work)

    def qsize(self):
        return self.q.qsize()

    def set_ridealong(self, ridealongid, work):
        self.ridealong[ridealongid] = work

    def get_ridealong(self, ridealongid):
        if ridealongid in self.ridealong:
            return self.ridealong[ridealongid]
        else:
            LOGGER.warning('ridealong data for surt %s not found', ridealongid)
            return {}

    def del_ridealong(self, ridealongid):
        if ridealongid in self.ridealong:
            del self.ridealong[ridealongid]

    def ridealong_size(self):
        return len(self.ridealong)

    def done(self, worker_count):
        return self.awaiting_work == worker_count and self.q.qsize() == 0

    async def close(self):
        await self.q.join()

    def save(self, crawler, f):
        # XXX make this more self-describing
        pickle.dump('Put the XXX header here', f)  # XXX date, conf file name, conf file checksum
        pickle.dump(self.ridealong, f)
        pickle.dump(crawler._seeds, f)
        count = self.q.qsize()
        pickle.dump(count, f)
        for _ in range(0, count):
            work = self.q.get_nowait()
            pickle.dump(work, f)

    def load(self, crawler, f):
        header = pickle.load(f)  # XXX check that this is a good header... log it
        self.ridealong = pickle.load(f)
        crawler._seeds = pickle.load(f)
        self.q = asyncio.PriorityQueue()
        count = pickle.load(f)
        for _ in range(0, count):
            work = pickle.load(f)
            self.q.put_nowait(work)

    def dump_frontier(self):
        while True:
            try:
                work = self.q.get_nowait()
            except asyncio.queues.QueueEmpty:
                break
            priority, rand, surt = work
            ridealong = self.get_ridealong(surt)
            print(json.dumps({'priority': priority, 'rand': rand, 'url': ridealong['url'].url}))

    def summarize(self):
        '''
        Print a human-readable summary of what's in the queues
        '''
        print('{} items in the crawl queue'.format(self.q.qsize()))
        print('{} items in the ridealong dict'.format(len(self.ridealong)))

        if self.q.qsize() != len(self.ridealong):
            LOGGER.error('Different counts for queue size and ridealong size')
            q_keys = set()
            try:
                while True:
                    priority, rand, surt = self.q.get_nowait()
                    q_keys.add(surt)
            except asyncio.queues.QueueEmpty:
                pass
            ridealong_keys = set(self.ridealong.keys())
            extra_q = q_keys.difference(ridealong_keys)
            extra_r = ridealong_keys.difference(q_keys)
            if extra_q:
                print('Extra urls in queues and not ridealong')
                print(extra_q)
            if extra_r:
                print('Extra urls in ridealong and not queues')
                print(extra_r)
                for r in extra_r:
                    print('  ', r, self.ridealong[r])
            raise ValueError('cannot continue, I just destroyed the queue')

        priority_count = defaultdict(int)
        netlocs = defaultdict(int)
        for k, v in self.ridealong.items():
            priority_count[v['priority']] += 1
            url = v['url']
            netlocs[url.urlsplit.netloc] += 1

        print('{} different hosts in the queue'.format(len(netlocs)))
        print('Queue counts by priority:')
        for p in sorted(list(priority_count.keys())):
            if priority_count[p] > 0:
                print('  {}: {}'.format(p, priority_count[p]))
        print('Queue counts for top 10 netlocs')
        netloc_order = sorted(netlocs.items(), key=itemgetter(1), reverse=True)[0:10]
        for k, v in netloc_order:
            print('  {}: {}'.format(k, v))

    def update_priority(self, priority, rand):
        '''
        When a fail occurs, we get requeued with a bigger 'rand'.
        This means that as we crawl in a given priority, we accumulate
        more and more repeatedly failing pages as we get close to the
        end of the queue. This function increments the priority if
        rand is too large, kicking the can down the road.
        '''
        while rand > 1.2:
            priority += 1
            rand -= 1.0
        return priority, rand
<commit_msg>bugfix, the q.get can succeed after quota exceeded<commit_after>'''
A single-queue global scheduler for CoCrawler

global QPS value used for all hosts

remember last deadline for every host

hand out work in order, increment deadlines

'''
import time
import asyncio
import pickle
from collections import defaultdict
from operator import itemgetter
import logging
import cachetools.ttl
import json

from . import config
from . import stats

LOGGER = logging.getLogger(__name__)

class Scheduler:
    def __init__(self):
        self.q = asyncio.PriorityQueue()
        self.ridealong = {}
        self.awaiting_work = 0
        self.maxhostqps = None
        self.delta_t = None
        self.next_fetch = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds good enough for QPS=0.1 and up
        self.frozen_until = cachetools.ttl.TTLCache(10000, 10)  # 10 seconds is longer than our typical delay
        self.maxhostqps = float(config.read('Crawl', 'MaxHostQPS'))
        self.delta_t = 1./self.maxhostqps
        self.max_crawled_urls = int(config.read('Crawl', 'MaxCrawledUrls') or 0) or None  # 0 => None

    def global_crawl_quota_exceeded(self):
        if ((self.max_crawled_urls is not None and
             (stats.stat_value('fetch http code=200') or 0) >= self.max_crawled_urls)):
            return True

    async def get_work(self):
        '''
        This function is called separately by each worker. It's allowed to sleep and/or requeue
        if work can't be done immediately.
        '''
        while True:
            if self.global_crawl_quota_exceeded():
                raise asyncio.CancelledError  # cancel this one worker

            try:
                work = self.q.get_nowait()
            except asyncio.queues.QueueEmpty:
                # using awaiting_work to see if all workers are idle can race with sleeping s.q.get()
                # putting it in an except clause makes sure the race is only run when
                # the queue is actually empty.
                self.awaiting_work += 1
                with stats.coroutine_state('awaiting work'):
                    work = await self.q.get()
                self.awaiting_work -= 1

            if self.global_crawl_quota_exceeded():
                self.q.put_nowait(work)
                self.q.task_done()
                raise asyncio.CancelledError  # cancel this one worker

            now = time.time()
            surt = work[2]
            surt_host, _, _ = surt.partition(')')
            ridealong = self.get_ridealong(surt)

            recycle, why, dt = self.do_we_recycle(now, surt, surt_host, ridealong)

            # sleep then requeue
            if recycle:
                stats.stats_sum(why+' sum', dt)
                with stats.coroutine_state(why):
                    await asyncio.sleep(dt)
                    self.q.put_nowait(work)
                    self.q.task_done()
                    continue

            # Normal case: sleep if needed, and then return the work to the caller.
            self.next_fetch[surt_host] = now + dt + self.delta_t
            if dt > 0:
                stats.stats_sum(why+' sum', dt)
                with stats.coroutine_state(why):
                    await asyncio.sleep(dt)

            return work

    def do_we_recycle(self, now, surt, surt_host, ridealong):
        recycle = False
        why = None
        dt = 0

        # does host have cached dns? XXX
        # if not, and we're The One, fetch it
        # if not, and we aren't The One, recycle

        # does host have cached robots? XXX
        # if not, and we're The One, fetch it
        # if not, and we aren't The One, recycle

        # when's the next available rate limit slot?
        now = time.time()
        if surt_host in self.next_fetch:
            dt = max(self.next_fetch[surt_host] - now, 0.)
        else:
            dt = 0
        if dt > 3.0:
            recycle = True
            why = 'scheduler ratelimit recycle'
            dt = 3.0
        elif dt > 0:
            why = 'scheduler ratelimit short sleep'

        return recycle, why, dt

    def work_done(self):
        self.q.task_done()

    def requeue_work(self, work):
        '''
        When we requeue work after a failure, we add 0.5 to the rand;
        eventually do that in here
        '''
        self.q.put_nowait(work)

    def queue_work(self, work):
        self.q.put_nowait(work)

    def qsize(self):
        return self.q.qsize()

    def set_ridealong(self, ridealongid, work):
        self.ridealong[ridealongid] = work

    def get_ridealong(self, ridealongid):
        if ridealongid in self.ridealong:
            return self.ridealong[ridealongid]
        else:
            LOGGER.warning('ridealong data for surt %s not found', ridealongid)
            return {}

    def del_ridealong(self, ridealongid):
        if ridealongid in self.ridealong:
            del self.ridealong[ridealongid]

    def ridealong_size(self):
        return len(self.ridealong)

    def done(self, worker_count):
        return self.awaiting_work == worker_count and self.q.qsize() == 0

    async def close(self):
        await self.q.join()

    def save(self, crawler, f):
        # XXX make this more self-describing
        pickle.dump('Put the XXX header here', f)  # XXX date, conf file name, conf file checksum
        pickle.dump(self.ridealong, f)
        pickle.dump(crawler._seeds, f)
        count = self.q.qsize()
        pickle.dump(count, f)
        for _ in range(0, count):
            work = self.q.get_nowait()
            pickle.dump(work, f)

    def load(self, crawler, f):
        header = pickle.load(f)  # XXX check that this is a good header... log it
        self.ridealong = pickle.load(f)
        crawler._seeds = pickle.load(f)
        self.q = asyncio.PriorityQueue()
        count = pickle.load(f)
        for _ in range(0, count):
            work = pickle.load(f)
            self.q.put_nowait(work)

    def dump_frontier(self):
        while True:
            try:
                work = self.q.get_nowait()
            except asyncio.queues.QueueEmpty:
                break
            priority, rand, surt = work
            ridealong = self.get_ridealong(surt)
            print(json.dumps({'priority': priority, 'rand': rand, 'url': ridealong['url'].url}))

    def summarize(self):
        '''
        Print a human-readable summary of what's in the queues
        '''
        print('{} items in the crawl queue'.format(self.q.qsize()))
        print('{} items in the ridealong dict'.format(len(self.ridealong)))

        if self.q.qsize() != len(self.ridealong):
            LOGGER.error('Different counts for queue size and ridealong size')
            q_keys = set()
            try:
                while True:
                    priority, rand, surt = self.q.get_nowait()
                    q_keys.add(surt)
            except asyncio.queues.QueueEmpty:
                pass
            ridealong_keys = set(self.ridealong.keys())
            extra_q = q_keys.difference(ridealong_keys)
            extra_r = ridealong_keys.difference(q_keys)
            if extra_q:
                print('Extra urls in queues and not ridealong')
                print(extra_q)
            if extra_r:
                print('Extra urls in ridealong and not queues')
                print(extra_r)
                for r in extra_r:
                    print('  ', r, self.ridealong[r])
            raise ValueError('cannot continue, I just destroyed the queue')

        priority_count = defaultdict(int)
        netlocs = defaultdict(int)
        for k, v in self.ridealong.items():
            priority_count[v['priority']] += 1
            url = v['url']
            netlocs[url.urlsplit.netloc] += 1

        print('{} different hosts in the queue'.format(len(netlocs)))
        print('Queue counts by priority:')
        for p in sorted(list(priority_count.keys())):
            if priority_count[p] > 0:
                print('  {}: {}'.format(p, priority_count[p]))
        print('Queue counts for top 10 netlocs')
        netloc_order = sorted(netlocs.items(), key=itemgetter(1), reverse=True)[0:10]
        for k, v in netloc_order:
            print('  {}: {}'.format(k, v))

    def update_priority(self, priority, rand):
        '''
        When a fail occurs, we get requeued with a bigger 'rand'.
        This means that as we crawl in a given priority, we accumulate
        more and more repeatedly failing pages as we get close to the
        end of the queue. This function increments the priority if
        rand is too large, kicking the can down the road.
        '''
        while rand > 1.2:
            priority += 1
            rand -= 1.0
        return priority, rand
<|endoftext|>