<commit_before>import re
import os
import sys
import subprocess
from argparse import ArgumentParser, RawTextHelpFormatter

if os.name == "nt":  # Windows
    make = "mingw32-make"
else:
    make = "make"

src_folder = "./test/expressions/"
build_folder = "./test/expressions/"
exceptions_list_location = "./test/expressions/stan_math_sigs_exceptions.expected"

eigen_types = ["matrix", "vector", "row_vector"]
arg_types = {
    "int": "int",
    "int[]": "std::vector<int>",
    "int[,]": "std::vector<std::vector<int>>",
    "real": "SCALAR",
    "real[]": "std::vector<SCALAR>",
    "real[,]": "std::vector<std::vector<SCALAR>>",
    "vector": "Eigen::Matrix<SCALAR, Eigen::Dynamic, 1>",
    "vector[]": "std::vector<Eigen::Matrix<SCALAR, Eigen::Dynamic, 1>>",
    "row_vector": "Eigen::Matrix<SCALAR, 1, Eigen::Dynamic>",
    "row_vector[]": "std::vector<Eigen::Matrix<SCALAR, 1, Eigen::Dynamic>>",
    "matrix": "Eigen::Matrix<SCALAR, Eigen::Dynamic, Eigen::Dynamic>",
    "(vector, vector, data real[], data int[]) => vector": "auto",
    "rng": "std::minstd_rand",
}

test_code_template = """
TEST(ExpressionTest{overload}, {function_name}{signature_number}) {{
{matrix_argument_declarations}
  auto res_mat = stan::math::{function_name}({matrix_argument_list});

{expression_argument_declarations}
  auto res_expr = stan::math::{function_name}({expression_argument_list});

  EXPECT_STAN_EQ(res_expr, res_mat);

{checks}
}}
"""

def get_ignored_signatures():
    """
    Loads list of ignored signatures from the file listing the exceptions.
    :return: set of ignored signatures
    """
    part_sig = ""
    ignored = set()
    for signature in open(exceptions_list_location):
        signature = part_sig + signature
        part_sig = ""
        if not signature.endswith(")\n"):
            part_sig = signature
            continue
        ignored.add(signature)
    return ignored

def parse_signature_file(sig_file):
    """
    Parses signatures from a file of signatures
    :param sig_file: file-like object to pares
    :return: list of signatures
    """
    res = []
    part_sig = ""
    for signature in sig_file:
        signature = part_sig + signature
        part_sig = ""
        if not signature.endswith(")\n"):
            part_sig = signature
            continue
        res.append(signature)
    return res

def get_signatures():
    """
    Retrieves function signatures from stanc3
    :return: list of signatures
    """
    if os.name == "nt":
        stanc3 = ".\\test\\expressions\\stanc.exe"
    else:
        stanc3 = "./test/expressions/stanc"
    p = subprocess.Popen((make, stanc3))
    if p.wait() != 0:
        sys.stderr.write("Error in making stanc3!")
        sys.exit(-1)

    p = subprocess.Popen(
        (stanc3 + " --dump-stan-math-signatures"),
        stdout=subprocess.PIPE,
        universal_newlines=True,
        shell=True,
    )

    res = parse_signature_file(p.stdout)

    if p.wait() != 0:
        sys.stderr.write("Error in getting signatures from stanc3!\n")
        sys.exit(-1)

    return res

def parse_signature(signature):
    """
    Parses one signature
    :param signature: stanc3 function signature
    :return: return type, fucntion name and list of function argument types
    """
    return_type, rest = signature.split(" ", 1)
    function_name, rest = rest.split("(", 1)
    args = re.findall(r"(?:[(][^()]+[)][^,()]+)|(?:[^,()]+(?:,*[]])?)", rest)
    args = [i.strip() for i in args if i.strip()]
    return return_type, function_name, args

# list of function arguments that need special scalar values.
# None means to use the default argument value.
special_arg_values = {
    "acosh": [1.4],
    "log1m_exp": [-0.6],
    "categorical_log": [None, 1],
    "categorical_rng": [1, None],
    "categorical_lpmf": [None, 1],
    "dirichlet_log" : [1, None],
    "dirichlet_lpdf" : [1, None],
    "hmm_hidden_state_prob": [None, 1, 1],
    "hmm_latent_rng": [None, 1, 1, None],
    "hmm_marginal": [None, 1, 1],
    "lkj_corr_lpdf": [1, None],
    "lkj_corr_log": [1, None],
    "log_diff_exp": [3, None],
    "log_inv_logit_diff": [1.2, 0.4],
    "multinomial_log" : [None, 1],
    "multinomial_lpmf" : [None, 1],
    "multinomial_rng" : [1, None, None],
    "wiener_log" : [0.8, None, 0.4, None],
    "wiener_lpdf" : [0.8, None, 0.4, None],
}

def make_arg_code(arg, scalar, var_name, var_number, function_name):
    """
    Makes code for declaration and initialization of an argument to function.

    Default argument range (between 0 and 1) works for most function, but some need
    values outside this range - these require special handling. Specific lambda is
    also hardcoded - if we use more functor arguments in future this may need to be
    extended.
    :param arg: stan lang type of the argument
    :param scalar: scalar type used in argument
    :param var_name: name of the variable to create
    :param var_number: number of variable in the function call
    :param function_name: name of the function that will be tested using this argument
    :return: code for declaration and initialization of an argument
    """
    arg_type = arg_types[arg].replace("SCALAR", scalar)
    if arg == "(vector, vector, data real[], data int[]) => vector":
        return (
            "  %s %s = [](const auto& a, const auto&, const auto&, const auto&){return a;}"
            % (arg_type, var_name)
        )
    elif (
        function_name in special_arg_values
        and special_arg_values[function_name][var_number] is not None
    ):
        return "  %s %s = stan::test::make_arg<%s>(%f)" % (
            arg_type,
            var_name,
            arg_type,
            special_arg_values[function_name][var_number],
        )
    else:
        return "  %s %s = stan::test::make_arg<%s>()" % (arg_type, var_name, arg_type,)

def save_tests_in_files(N_files, tests):
    """
    Saves tests in files
    :param N_files: number of files to distribute tests into
    :param tests: list of test sources
    """
    for i in range(N_files):
        start = i * len(tests) // N_files
        end = (i + 1) * len(tests) // N_files
        with open(src_folder + "tests%d_test.cpp" % i, "w") as out:
            out.write("#include <test/expressions/expression_test_helpers.hpp>\n\n")
            for test in tests[start:end]:
                out.write(test)

def handle_function_list(functions_input, signatures):
    """
    Handles list of functions, splitting elements between functions and signatures.
    :param functions_input: This can contain names of functions
    already supported by stanc3, full function signatures or file names of files containing
    any of the previous two.
    :param signatures:
    :return:
    """
    function_names = []
    function_signatures = []
    for f in functions_input:
        if "." in f or "/" in f or "\\" in f:
            functions_input.extend(parse_signature_file(open(f)))
        elif " " in f:
            function_signatures.append(f)
            signatures.append(f)
        else:
            function_names.append(f)
    return function_names, function_signatures

# lists of functions that do not support fwd or rev autodiff
no_rev_overload = ["hmm_hidden_state_prob"]
no_fwd_overload = ["hmm_hidden_state_prob"]

def main(functions=(), j=1):
    """
    Generates expression tests. Functions that do not support expressions yet are listed
    in stan_math/tests/expressions/stan_math_sigs_exceptions.expected

    For every signature prim, rev and fwd instantiations are tested (with all scalars
    of type double/var/fvar<double>). Tests check the following:
     - signatures can be compiled with expressions arguments
     - results when given expressions are same as when given plain matrices
       (including derivatives)
     - functions evaluate expressions at most once

    :param functions: functions to generate tests for. This can contain names of functions
    already supported by stanc3, full function signatures or file names of files containing
    any of the previous two. Default: all signatures supported by stanc3
    :param j: number of files to split tests in
    """
    ignored = get_ignored_signatures()

    test_n = {}
    tests = []
    signatures = get_signatures()
    functions, extra_signatures = handle_function_list(functions, signatures)
    remaining_functions = set(functions)
    for signature in signatures:
        return_type, function_name, function_args = parse_signature(signature)
        # skip ignored signatures
        if signature in ignored and not functions and signature not in extra_signatures:
            continue
        # skip default if we have list of function names/signatures to test
        if (
            (functions or extra_signatures)
            and function_name not in functions
            and signature not in extra_signatures
        ):
            continue
        # skip signatures without eigen inputs
        for arg2test in eigen_types:
            if arg2test in function_args:
                break
        else:
            continue

        if function_name in remaining_functions:
            remaining_functions.remove(function_name)
        func_test_n = test_n.get(function_name, 0)
        test_n[function_name] = func_test_n + 1

        if function_name.endswith("_rng"):
            function_args.append("rng")

        for overload, scalar in (
            ("Prim", "double"),
            ("Rev", "stan::math::var"),
            ("Fwd", "stan::math::fvar<double>"),
        ):
            if function_name.endswith("_rng") and overload != "Prim":
                continue
            if function_name in no_fwd_overload and overload == "Fwd":
                continue
            if function_name in no_rev_overload and overload == "Rev":
                continue

            mat_declarations = ""
            for n, arg in enumerate(function_args):
                mat_declarations += (
                    make_arg_code(arg, scalar, "arg_mat%d" % n, n, function_name)
                    + ";\n"
                )

            mat_arg_list = ", ".join("arg_mat%d" % n for n in range(len(function_args)))

            expression_declarations = ""
            for n, arg in enumerate(function_args):
                expression_declarations += (
                    make_arg_code(arg, scalar, "arg_expr%d" % n, n, function_name)
                    + ";\n"
                )
                if arg in eigen_types:
                    expression_declarations += "  int counter%d = 0;\n" % n
                    expression_declarations += (
                        "  stan::test::counterOp<%s> counter_op%d(&counter%d);\n"
                        % (scalar, n, n)
                    )

            expression_arg_list = ""
            for n, arg in enumerate(function_args[:-1]):
                if arg in eigen_types:
                    expression_arg_list += "arg_expr%d.unaryExpr(counter_op%d), " % (
                        n,
                        n,
                    )
                else:
                    expression_arg_list += "arg_expr%d, " % n
            if function_args[-1] in eigen_types:
                expression_arg_list += "arg_expr%d.unaryExpr(counter_op%d)" % (
                    len(function_args) - 1,
                    len(function_args) - 1,
                )
            else:
                expression_arg_list += "arg_expr%d" % (len(function_args) - 1)

            checks = ""
            for n, arg in enumerate(function_args):
                if arg in eigen_types:
                    # besides evaluating its input rank also accesses one of the elements,
                    # resulting in counter being incremented twice.
                    if function_name == "rank":
                        checks += "  EXPECT_LE(counter%d, 2);\n" % n
                    else:
                        checks += "  EXPECT_LE(counter%d, 1);\n" % n
            if overload == "Rev" and (
                return_type.startswith("real")
                or return_type.startswith("vector")
                or return_type.startswith("row_vector")
                or return_type.startswith("matrix")
            ):
                checks += "  (stan::test::recursive_sum(res_mat) + stan::test::recursive_sum(res_expr)).grad();\n"
                for n, arg in enumerate(function_args):
                    # functors don't have adjoints to check
                    if arg == "(vector, vector, data real[], data int[]) => vector":
                        continue
                    checks += "  EXPECT_STAN_ADJ_EQ(arg_expr%d,arg_mat%d);\n" % (n, n,)
            tests.append(
                test_code_template.format(
                    overload=overload,
                    function_name=function_name,
                    signature_number=func_test_n,
                    matrix_argument_declarations=mat_declarations,
                    matrix_argument_list=mat_arg_list,
                    expression_argument_declarations=expression_declarations,
                    expression_argument_list=expression_arg_list,
                    checks=checks,
                )
            )
    if remaining_functions:
        raise NameError("Functions not found: " + ", ".join(remaining_functions))
    save_tests_in_files(j, tests)

def processCLIArgs():
    """
    Define and process the command line interface to the generateExpressionTests
    .py script.
    """
    parser = ArgumentParser(
        description="Generate and run stan math expression tests.",
        formatter_class=RawTextHelpFormatter,
    )
    parser.add_argument(
        "-j",
        metavar="N",
        type=int,
        default=1,
        help="Number of cores for make to use. Also number of files tests are split in.",
    )
    parser.add_argument(
        "functions",
        nargs="+",
        type=str,
        default=[],
        help="Names of the functions to test. By default tests all functions.",
    )
    return parser.parse_args()

if __name__ == "__main__":
    cli_args = processCLIArgs()
    main(cli_args.functions, cli_args.j)
<commit_msg>bugfix test values<commit_after>import re
import os
import sys
import subprocess
from argparse import ArgumentParser, RawTextHelpFormatter

if os.name == "nt":  # Windows
    make = "mingw32-make"
else:
    make = "make"

src_folder = "./test/expressions/"
build_folder = "./test/expressions/"
exceptions_list_location = "./test/expressions/stan_math_sigs_exceptions.expected"

eigen_types = ["matrix", "vector", "row_vector"]
arg_types = {
    "int": "int",
    "int[]": "std::vector<int>",
    "int[,]": "std::vector<std::vector<int>>",
    "real": "SCALAR",
    "real[]": "std::vector<SCALAR>",
    "real[,]": "std::vector<std::vector<SCALAR>>",
    "vector": "Eigen::Matrix<SCALAR, Eigen::Dynamic, 1>",
    "vector[]": "std::vector<Eigen::Matrix<SCALAR, Eigen::Dynamic, 1>>",
    "row_vector": "Eigen::Matrix<SCALAR, 1, Eigen::Dynamic>",
    "row_vector[]": "std::vector<Eigen::Matrix<SCALAR, 1, Eigen::Dynamic>>",
    "matrix": "Eigen::Matrix<SCALAR, Eigen::Dynamic, Eigen::Dynamic>",
    "(vector, vector, data real[], data int[]) => vector": "auto",
    "rng": "std::minstd_rand",
}

test_code_template = """
TEST(ExpressionTest{overload}, {function_name}{signature_number}) {{
{matrix_argument_declarations}
  auto res_mat = stan::math::{function_name}({matrix_argument_list});

{expression_argument_declarations}
  auto res_expr = stan::math::{function_name}({expression_argument_list});

  EXPECT_STAN_EQ(res_expr, res_mat);

{checks}
}}
"""

def get_ignored_signatures():
    """
    Loads list of ignored signatures from the file listing the exceptions.
    :return: set of ignored signatures
    """
    part_sig = ""
    ignored = set()
    for signature in open(exceptions_list_location):
        signature = part_sig + signature
        part_sig = ""
        if not signature.endswith(")\n"):
            part_sig = signature
            continue
        ignored.add(signature)
    return ignored

def parse_signature_file(sig_file):
    """
    Parses signatures from a file of signatures
    :param sig_file: file-like object to pares
    :return: list of signatures
    """
    res = []
    part_sig = ""
    for signature in sig_file:
        signature = part_sig + signature
        part_sig = ""
        if not signature.endswith(")\n"):
            part_sig = signature
            continue
        res.append(signature)
    return res

def get_signatures():
    """
    Retrieves function signatures from stanc3
    :return: list of signatures
    """
    if os.name == "nt":
        stanc3 = ".\\test\\expressions\\stanc.exe"
    else:
        stanc3 = "./test/expressions/stanc"
    p = subprocess.Popen((make, stanc3))
    if p.wait() != 0:
        sys.stderr.write("Error in making stanc3!")
        sys.exit(-1)

    p = subprocess.Popen(
        (stanc3 + " --dump-stan-math-signatures"),
        stdout=subprocess.PIPE,
        universal_newlines=True,
        shell=True,
    )

    res = parse_signature_file(p.stdout)

    if p.wait() != 0:
        sys.stderr.write("Error in getting signatures from stanc3!\n")
        sys.exit(-1)

    return res

def parse_signature(signature):
    """
    Parses one signature
    :param signature: stanc3 function signature
    :return: return type, fucntion name and list of function argument types
    """
    return_type, rest = signature.split(" ", 1)
    function_name, rest = rest.split("(", 1)
    args = re.findall(r"(?:[(][^()]+[)][^,()]+)|(?:[^,()]+(?:,*[]])?)", rest)
    args = [i.strip() for i in args if i.strip()]
    return return_type, function_name, args

# list of function arguments that need special scalar values.
# None means to use the default argument value.
special_arg_values = {
    "acosh": [1.4],
    "log1m_exp": [-0.6],
    "categorical_log": [None, 1],
    "categorical_rng": [1, None],
    "categorical_lpmf": [None, 1],
    "dirichlet_log" : [1, None],
    "dirichlet_lpdf" : [1, None],
    "hmm_hidden_state_prob": [None, 1, 1],
    "hmm_latent_rng": [None, 1, 1, None],
    "hmm_marginal": [None, 1, 1],
    "lkj_corr_lpdf": [1, None],
    "lkj_corr_log": [1, None],
    "log_diff_exp": [3, None],
    "log_inv_logit_diff": [1.2, 0.4],
    "multinomial_log" : [None, 1],
    "multinomial_lpmf" : [None, 1],
    "multinomial_rng" : [1, None, None],
    "wiener_log" : [0.8, None, 0.4, None, None],
    "wiener_lpdf" : [0.8, None, 0.4, None, None],
}

def make_arg_code(arg, scalar, var_name, var_number, function_name):
    """
    Makes code for declaration and initialization of an argument to function.

    Default argument range (between 0 and 1) works for most function, but some need
    values outside this range - these require special handling. Specific lambda is
    also hardcoded - if we use more functor arguments in future this may need to be
    extended.
    :param arg: stan lang type of the argument
    :param scalar: scalar type used in argument
    :param var_name: name of the variable to create
    :param var_number: number of variable in the function call
    :param function_name: name of the function that will be tested using this argument
    :return: code for declaration and initialization of an argument
    """
    arg_type = arg_types[arg].replace("SCALAR", scalar)
    if arg == "(vector, vector, data real[], data int[]) => vector":
        return (
            "  %s %s = [](const auto& a, const auto&, const auto&, const auto&){return a;}"
            % (arg_type, var_name)
        )
    elif (
        function_name in special_arg_values
        and special_arg_values[function_name][var_number] is not None
    ):
        return "  %s %s = stan::test::make_arg<%s>(%f)" % (
            arg_type,
            var_name,
            arg_type,
            special_arg_values[function_name][var_number],
        )
    else:
        return "  %s %s = stan::test::make_arg<%s>()" % (arg_type, var_name, arg_type,)

def save_tests_in_files(N_files, tests):
    """
    Saves tests in files
    :param N_files: number of files to distribute tests into
    :param tests: list of test sources
    """
    for i in range(N_files):
        start = i * len(tests) // N_files
        end = (i + 1) * len(tests) // N_files
        with open(src_folder + "tests%d_test.cpp" % i, "w") as out:
            out.write("#include <test/expressions/expression_test_helpers.hpp>\n\n")
            for test in tests[start:end]:
                out.write(test)

def handle_function_list(functions_input, signatures):
    """
    Handles list of functions, splitting elements between functions and signatures.
    :param functions_input: This can contain names of functions
    already supported by stanc3, full function signatures or file names of files containing
    any of the previous two.
    :param signatures:
    :return:
    """
    function_names = []
    function_signatures = []
    for f in functions_input:
        if "." in f or "/" in f or "\\" in f:
            functions_input.extend(parse_signature_file(open(f)))
        elif " " in f:
            function_signatures.append(f)
            signatures.append(f)
        else:
            function_names.append(f)
    return function_names, function_signatures

# lists of functions that do not support fwd or rev autodiff
no_rev_overload = ["hmm_hidden_state_prob"]
no_fwd_overload = ["hmm_hidden_state_prob"]

def main(functions=(), j=1):
    """
    Generates expression tests. Functions that do not support expressions yet are listed
    in stan_math/tests/expressions/stan_math_sigs_exceptions.expected

    For every signature prim, rev and fwd instantiations are tested (with all scalars
    of type double/var/fvar<double>). Tests check the following:
     - signatures can be compiled with expressions arguments
     - results when given expressions are same as when given plain matrices
       (including derivatives)
     - functions evaluate expressions at most once

    :param functions: functions to generate tests for. This can contain names of functions
    already supported by stanc3, full function signatures or file names of files containing
    any of the previous two. Default: all signatures supported by stanc3
    :param j: number of files to split tests in
    """
    ignored = get_ignored_signatures()

    test_n = {}
    tests = []
    signatures = get_signatures()
    functions, extra_signatures = handle_function_list(functions, signatures)
    remaining_functions = set(functions)
    for signature in signatures:
        return_type, function_name, function_args = parse_signature(signature)
        # skip ignored signatures
        if signature in ignored and not functions and signature not in extra_signatures:
            continue
        # skip default if we have list of function names/signatures to test
        if (
            (functions or extra_signatures)
            and function_name not in functions
            and signature not in extra_signatures
        ):
            continue
        # skip signatures without eigen inputs
        for arg2test in eigen_types:
            if arg2test in function_args:
                break
        else:
            continue

        if function_name in remaining_functions:
            remaining_functions.remove(function_name)
        func_test_n = test_n.get(function_name, 0)
        test_n[function_name] = func_test_n + 1

        if function_name.endswith("_rng"):
            function_args.append("rng")

        for overload, scalar in (
            ("Prim", "double"),
            ("Rev", "stan::math::var"),
            ("Fwd", "stan::math::fvar<double>"),
        ):
            if function_name.endswith("_rng") and overload != "Prim":
                continue
            if function_name in no_fwd_overload and overload == "Fwd":
                continue
            if function_name in no_rev_overload and overload == "Rev":
                continue

            mat_declarations = ""
            for n, arg in enumerate(function_args):
                mat_declarations += (
                    make_arg_code(arg, scalar, "arg_mat%d" % n, n, function_name)
                    + ";\n"
                )

            mat_arg_list = ", ".join("arg_mat%d" % n for n in range(len(function_args)))

            expression_declarations = ""
            for n, arg in enumerate(function_args):
                expression_declarations += (
                    make_arg_code(arg, scalar, "arg_expr%d" % n, n, function_name)
                    + ";\n"
                )
                if arg in eigen_types:
                    expression_declarations += "  int counter%d = 0;\n" % n
                    expression_declarations += (
                        "  stan::test::counterOp<%s> counter_op%d(&counter%d);\n"
                        % (scalar, n, n)
                    )

            expression_arg_list = ""
            for n, arg in enumerate(function_args[:-1]):
                if arg in eigen_types:
                    expression_arg_list += "arg_expr%d.unaryExpr(counter_op%d), " % (
                        n,
                        n,
                    )
                else:
                    expression_arg_list += "arg_expr%d, " % n
            if function_args[-1] in eigen_types:
                expression_arg_list += "arg_expr%d.unaryExpr(counter_op%d)" % (
                    len(function_args) - 1,
                    len(function_args) - 1,
                )
            else:
                expression_arg_list += "arg_expr%d" % (len(function_args) - 1)

            checks = ""
            for n, arg in enumerate(function_args):
                if arg in eigen_types:
                    # besides evaluating its input rank also accesses one of the elements,
                    # resulting in counter being incremented twice.
                    if function_name == "rank":
                        checks += "  EXPECT_LE(counter%d, 2);\n" % n
                    else:
                        checks += "  EXPECT_LE(counter%d, 1);\n" % n
            if overload == "Rev" and (
                return_type.startswith("real")
                or return_type.startswith("vector")
                or return_type.startswith("row_vector")
                or return_type.startswith("matrix")
            ):
                checks += "  (stan::test::recursive_sum(res_mat) + stan::test::recursive_sum(res_expr)).grad();\n"
                for n, arg in enumerate(function_args):
                    # functors don't have adjoints to check
                    if arg == "(vector, vector, data real[], data int[]) => vector":
                        continue
                    checks += "  EXPECT_STAN_ADJ_EQ(arg_expr%d,arg_mat%d);\n" % (n, n,)
            tests.append(
                test_code_template.format(
                    overload=overload,
                    function_name=function_name,
                    signature_number=func_test_n,
                    matrix_argument_declarations=mat_declarations,
                    matrix_argument_list=mat_arg_list,
                    expression_argument_declarations=expression_declarations,
                    expression_argument_list=expression_arg_list,
                    checks=checks,
                )
            )
    if remaining_functions:
        raise NameError("Functions not found: " + ", ".join(remaining_functions))
    save_tests_in_files(j, tests)

def processCLIArgs():
    """
    Define and process the command line interface to the generateExpressionTests
    .py script.
    """
    parser = ArgumentParser(
        description="Generate and run stan math expression tests.",
        formatter_class=RawTextHelpFormatter,
    )
    parser.add_argument(
        "-j",
        metavar="N",
        type=int,
        default=1,
        help="Number of cores for make to use. Also number of files tests are split in.",
    )
    parser.add_argument(
        "functions",
        nargs="+",
        type=str,
        default=[],
        help="Names of the functions to test. By default tests all functions.",
    )
    return parser.parse_args()

if __name__ == "__main__":
    cli_args = processCLIArgs()
    main(cli_args.functions, cli_args.j)
<|endoftext|>


<commit_before>import numpy as np

class NormalizePositive(object):

    def fit(self, X, y=None):
        # count features that are greater than zero in axis 0:
        binary = (X > 0)
        count0 = binary.sum(axis=0)

        # to avoid division by zero, set zero counts to one:
        count0 += (count0 == 0)

        self.mean = X.sum(axis=0)/count0

        # Compute variance by average squared difference to the mean, but only
        # consider differences where binary is True (i.e., where there was a
        # true rating):
        diff = (X - self.mean) * binary
        diff **= 2
        self.std = np.sqrt(0.1 + diff.sum(axis=0)/count0)
        return self

    def fit_transform(self, X):
        return self.fit(X).transform(X)

    def transform(self, X):
        binary = (X > 0)
        X = X - self.mean
        X /= self.std
        X *= binary
        return X

    def inverse_transform(self, X, copy=True):
        if copy:
            X = X.copy()
        X *= self.std
        X += self.mean
        return X

def predict(train):
    norm = NormalizePositive()
    train = norm.fit_transform(train)
    return norm.inverse_transform(train * 0.)

def main(transpose_inputs=False):
    from load_ml100k import get_train_test
    from sklearn import metrics
    train,test = get_train_test(random_state=12)
    if transpose_inputs:
        train = train.T
        test = test.T
    predicted = predict(train)
    r2 = metrics.r2_score(test[test > 0], predicted[test > 0])
    print('R2 score ({} normalization): {:.1%}'.format(
        ('movie' if transpose_inputs else 'user'),
        r2))
if __name__ == '__main__':
    main()
    main(transpose_inputs=True)
<commit_msg>ENH More readable code<commit_after>import numpy as np

class NormalizePositive(object):

    def fit(self, features, y=None):
        # count features that are greater than zero in axis 0:
        binary = (features > 0)
        count0 = binary.sum(axis=0)

        # to avoid division by zero, set zero counts to one:
        count0[count0 == 0] = 1.

        self.mean = features.sum(axis=0)/count0

        # Compute variance by average squared difference to the mean, but only
        # consider differences where binary is True (i.e., where there was a
        # true rating):
        diff = (features - self.mean) * binary
        diff **= 2
        # regularize the estimate of std by adding 0.1
        self.std = np.sqrt(0.1 + diff.sum(axis=0)/count0)
        return self

    def transform(self, features):
        binary = (features > 0)
        features = features - self.mean
        features /= self.std
        features *= binary
        return features

    def inverse_transform(self, features, copy=True):
        if copy:
            features = features.copy()
        features *= self.std
        features += self.mean
        return features

    def fit_transform(self, features):
        return self.fit(features).transform(features)

def predict(train):
    norm = NormalizePositive()
    train = norm.fit_transform(train)
    return norm.inverse_transform(train * 0.)

def main(transpose_inputs=False):
    from load_ml100k import get_train_test
    from sklearn import metrics
    train,test = get_train_test(random_state=12)
    if transpose_inputs:
        train = train.T
        test = test.T
    predicted = predict(train)
    r2 = metrics.r2_score(test[test > 0], predicted[test > 0])
    print('R2 score ({} normalization): {:.1%}'.format(
        ('movie' if transpose_inputs else 'user'),
        r2))
if __name__ == '__main__':
    main()
    main(transpose_inputs=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
############################################################################################
#
#    Zoook. OpenERP e-sale, e-commerce Open Source Management Solution
#    Copyright (C) 2011 Zikzakmedia S.L. (<http://www.zikzakmedia.com>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################################

from django import template
from django.template import Library, Node
from django_zoook.settings import *

import os
import cStringIO # *much* faster than StringIO
import urllib
from PIL import Image

register = template.Library()

class ThumbnailNode(Node):
    def __init__(self, source_var, size, default):
        self.source_var = source_var
        self.size = size
        self.default = '/static/images/%s' % default

    def render(self, context):
        try:
            fileimage = template.Variable(self.source_var).resolve(context)
        except:
            fileimage = str(self.source_var)[1:-1]

        size = str(self.size)

        try:
            fname = fileimage.split('/')[-1:]
            fname = fname[0].split('.')
            size = size.split('x')
            base_image = '%s/%s/%s.%s' % (MEDIA_ROOT, 'catalog', fname[0], fname[1])
            thumb_image = '%s/%s/%sx%s/%s.%s' % (MEDIA_ROOT, 'catalog', size[0], size[1], fname[0], fname[1])

            #check thumb exist
            if not os.path.exists('%s/%s/%sx%s/%s.%s' % (MEDIA_ROOT, 'catalog', size[0], size[1], fname[0], fname[1])):
                #check if dir exist
                directory = "%s/%s/%sx%s" % (MEDIA_ROOT, 'catalog',size[0], size[1])
                if not os.path.exists(directory):
                    os.makedirs(directory)
                #check base image exist
                if not os.path.exists(base_image):
                    file = urllib.urlopen(fileimage)
                    im = cStringIO.StringIO(file.read()) # constructs a StringIO holding the image
                    img = Image.open(im)
                    # print img.format, img.size, img.mode
                    img.save(base_image)
                image = Image.open(base_image)
                image.thumbnail([int(size[0]), int(size[1])], Image.ANTIALIAS)
                image.save(thumb_image, image.format)
            filename = '%s%s/%sx%s/%s.%s' % (MEDIA_URL, 'catalog', size[0], size[1], fname[0], fname[1])
        except:
            filename = self.default

        return filename

def thumbnail(parser, token):
    """
    Creates a thumbnail image file.

    Basic tag Syntax::
        {% thumbnail [source] [size] [base image]%}

    *source* must be a ``File`` object: 'http://domain/file.png'

    *size* the size in the format ``[width]x[height]`` For example,
      ``{% thumbnail 'http://domain/file.png' 100x50 %}``

    *img_default* image name default. For example,
      ``{% thumbnail 'http://domain/file.png' 100x50 product_thumb.png %}``

    Demo:
      {% thumbnail 'http://domain/file.png' 100x50 product_thumb.png %}
      {% thumbnail value.base_image.filename 150x150 product_thumb.png %}

    Thumbnail image is created in MEDIA_ROOT/catalog directory. Thumbnail create directory size, for example, MEDIA_ROOT/50x50
    """

    parts = token.split_contents()

    if len(parts) < 3:
        raise template.TemplateSyntaxError("'thumbnail' tag must be of the form:  {% thumbnail [source] [size] [base image]%}")

    return ThumbnailNode(parts[1], parts[2], parts[3])

register.tag(thumbnail)
<commit_msg>Fix prevent default icon when the thumbnail have more than one '.' in the filename<commit_after># -*- coding: utf-8 -*-
############################################################################################
#
#    Zoook. OpenERP e-sale, e-commerce Open Source Management Solution
#    Copyright (C) 2011 Zikzakmedia S.L. (<http://www.zikzakmedia.com>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################################

from django import template
from django.template import Library, Node
from django_zoook.settings import *

import os
import cStringIO # *much* faster than StringIO
import urllib
from PIL import Image

register = template.Library()

class ThumbnailNode(Node):
    def __init__(self, source_var, size, default):
        self.source_var = source_var
        self.size = size
        self.default = '/static/images/%s' % default

    def render(self, context):
        try:
            fileimage = template.Variable(self.source_var).resolve(context)
        except:
            fileimage = str(self.source_var)[1:-1]

        size = str(self.size)

        try:
            fname = fileimage.split('/')[-1:]
            fname = fname[0].split('.')
            if len(fname)>2:    # The name of the file have more than one '.'
                for i in range(1, len(fname)-1):
                    fname[0] += '.' + fname[i]
                fname[1] = fname[-1]
            size = size.split('x')
            base_image = '%s/%s/%s.%s' % (MEDIA_ROOT, 'catalog', fname[0], fname[1])
            thumb_image = '%s/%s/%sx%s/%s.%s' % (MEDIA_ROOT, 'catalog', size[0], size[1], fname[0], fname[1])

            #check thumb exist
            if not os.path.exists('%s/%s/%sx%s/%s.%s' % (MEDIA_ROOT, 'catalog', size[0], size[1], fname[0], fname[1])):
                #check if dir exist
                directory = "%s/%s/%sx%s" % (MEDIA_ROOT, 'catalog',size[0], size[1])
                if not os.path.exists(directory):
                    os.makedirs(directory)
                #check base image exist
                if not os.path.exists(base_image):
                    file = urllib.urlopen(fileimage)
                    im = cStringIO.StringIO(file.read()) # constructs a StringIO holding the image
                    img = Image.open(im)
                    # print img.format, img.size, img.mode
                    img.save(base_image)
                image = Image.open(base_image)
                image.thumbnail([int(size[0]), int(size[1])], Image.ANTIALIAS)
                image.save(thumb_image, image.format)
            filename = '%s%s/%sx%s/%s.%s' % (MEDIA_URL, 'catalog', size[0], size[1], fname[0], fname[1])
        except:
            filename = self.default

        return filename

def thumbnail(parser, token):
    """
    Creates a thumbnail image file.

    Basic tag Syntax::
        {% thumbnail [source] [size] [base image]%}

    *source* must be a ``File`` object: 'http://domain/file.png'

    *size* the size in the format ``[width]x[height]`` For example,
      ``{% thumbnail 'http://domain/file.png' 100x50 %}``

    *img_default* image name default. For example,
      ``{% thumbnail 'http://domain/file.png' 100x50 product_thumb.png %}``

    Demo:
      {% thumbnail 'http://domain/file.png' 100x50 product_thumb.png %}
      {% thumbnail value.base_image.filename 150x150 product_thumb.png %}

    Thumbnail image is created in MEDIA_ROOT/catalog directory. Thumbnail create directory size, for example, MEDIA_ROOT/50x50
    """

    parts = token.split_contents()

    if len(parts) < 3:
        raise template.TemplateSyntaxError("'thumbnail' tag must be of the form:  {% thumbnail [source] [size] [base image]%}")

    return ThumbnailNode(parts[1], parts[2], parts[3])

register.tag(thumbnail)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from collections import namedtuple as _namedtuple, OrderedDict as _OrderedDict

MenuResult = _namedtuple('MenuResult', 'selected value returncode')
class Menu(object):
    def __init__(self, command):
        "Create a python wrapper for command"
        self.command = command

    def __call__(self, entries):
        """Send entries to menu, return selected entry

        entries is an iterable where each element is a string that corresponds
        to an entry in the menu.
        """
        return self._run(self.command, entries)

    @classmethod
    def _run(cls, cmd, entries, entry_sep='\n'):
        res, returncode = cls._launch_menu_proc(cmd, entries)
        try:
            val = entries.get(res)
        except AttributeError:
            val = None
        return MenuResult(res, val, returncode)

    def sort(self, entries, key=None, reverse=False):
        """Sort and send entries to menu, return selected entry

        entries is an iterable where each element is a string that corresponds
        to an entry in the menu.
        """
        try:
            data = sorted(entries.items(), key=key, reverse=reverse)
            data = _OrderedDict(data)
        except AttributeError:
            data = sorted(entries, key=key, reverse=reverse)
        return self(data)

    @staticmethod
    def _launch_menu_proc(cmd, data, entry_sep='\n'):
        entries = entry_sep.join(data)
        from subprocess import Popen as _Popen, PIPE as _PIPE
        p = _Popen(cmd, stdout=_PIPE, stdin=_PIPE)
        stdout, stderr = p.communicate(entries.encode())
        try:
            p.terminate()
        except OSError:
            pass                # python2 compatibility
        return stdout.decode().rstrip(), p.returncode

    def __repr__(self):
        clsname = self.__class__.__name__
        toret = [clsname, '(command=', repr(self.command), ')']
        return ''.join(toret)

# rofi = Menu(command = ('rofi', '-fullscreen', '-dmenu', '-i'))
# dmenu = Menu(command = ('dmenu', '-l', '10', '-i'))
# fzf = Menu(command = ('fzf',))
<commit_msg>Added logging to Menu._run<commit_after># -*- coding: utf-8 -*-
from collections import namedtuple as _namedtuple, OrderedDict as _OrderedDict
import logging as _logging
_logr = _logging.getLogger(__name__)
_logr.addHandler(_logging.NullHandler())

MenuResult = _namedtuple('MenuResult', 'selected value returncode')
class Menu(object):
    def __init__(self, command):
        "Create a python wrapper for command"
        self.command = command

    def __call__(self, entries):
        """Send entries to menu, return selected entry

        entries is an iterable where each element is a string that corresponds
        to an entry in the menu.
        """
        return self._run(self.command, entries)

    @classmethod
    def _run(cls, cmd, entries, entry_sep='\n'):
        res, returncode = cls._launch_menu_proc(cmd, entries)
        try:
            val = entries.get(res)
        except AttributeError:
            val = None
        totres = MenuResult(res, val, returncode)
        _logr.debug('{} returned: {}'.format(cmd[0], totres))
        return totres

    def sort(self, entries, key=None, reverse=False):
        """Sort and send entries to menu, return selected entry

        entries is an iterable where each element is a string that corresponds
        to an entry in the menu.
        """
        try:
            data = sorted(entries.items(), key=key, reverse=reverse)
            data = _OrderedDict(data)
        except AttributeError:
            data = sorted(entries, key=key, reverse=reverse)
        return self(data)

    @staticmethod
    def _launch_menu_proc(cmd, data, entry_sep='\n'):
        entries = entry_sep.join(data)
        from subprocess import Popen as _Popen, PIPE as _PIPE
        p = _Popen(cmd, stdout=_PIPE, stdin=_PIPE)
        stdout, stderr = p.communicate(entries.encode())
        try:
            p.terminate()
        except OSError:
            pass                # python2 compatibility
        return stdout.decode().rstrip(), p.returncode

    def __repr__(self):
        clsname = self.__class__.__name__
        toret = [clsname, '(command=', repr(self.command), ')']
        return ''.join(toret)

# rofi = Menu(command = ('rofi', '-fullscreen', '-dmenu', '-i'))
# dmenu = Menu(command = ('dmenu', '-l', '10', '-i'))
# fzf = Menu(command = ('fzf',))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2020-2022 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the quorum detection of avalanche."""

from time import time

from test_framework.avatools import (
    AvaP2PInterface,
    create_coinbase_stakes,
    get_ava_p2p_interface,
)
from test_framework.key import ECKey, ECPubKey
from test_framework.messages import (
    NODE_AVALANCHE,
    NODE_NETWORK,
    AvalancheVote,
    AvalancheVoteError,
    msg_avaproofs,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal
from test_framework.wallet_util import bytes_to_wif

class AvalancheQuorumTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 1
        self.min_avaproofs_node_count = 8
        self.extra_args = [
            [
                '-enableavalanche=1',
                '-avacooldown=0',
                '-avatimeout=0',
                '-avaminquorumstake=100000000',
                '-avaminquorumconnectedstakeratio=0.8',
                f'-avaminavaproofsnodecount={self.min_avaproofs_node_count}',
            ]
        ]

    def mock_forward(self, delta):
        self.mock_time += delta
        self.nodes[0].setmocktime(self.mock_time)

    def run_test(self):
        self.mock_time = int(time())
        self.mock_forward(0)

        # Create a local node to poll from and a helper to send polls from it
        # and assert on the response
        node = self.nodes[0]
        poll_node = get_ava_p2p_interface(node)
        poll_node_pubkey = ECPubKey()
        poll_node_pubkey.set(bytes.fromhex(node.getavalanchekey()))

        def poll_and_assert_response(expected):
            # Send poll for best block
            block = int(node.getbestblockhash(), 16)
            poll_node.send_poll([block])

            # Get response and check that the vote is what we expect
            response = poll_node.wait_for_avaresponse()
            r = response.response
            assert poll_node_pubkey.verify_schnorr(response.sig, r.get_hash())
            assert_equal(len(r.votes), 1)

            actual = repr(r.votes[0])
            expected = repr(AvalancheVote(expected, block))
            assert_equal(actual, expected)

        # Create peers to poll
        coinbase_key = node.get_deterministic_priv_key().key
        blocks = node.generate(self.min_avaproofs_node_count)
        peers = []

        for i in range(0, self.min_avaproofs_node_count):
            keyHex = f"12b004fff7f4b69ef8650e767f18f11ede158148b425660723b9f9a66e61f7{i:02}"
            k = ECKey()
            k.set(bytes.fromhex(keyHex), True)
            stakes = create_coinbase_stakes(
                node, [blocks[i]], coinbase_key)
            proof = node.buildavalancheproof(1, 1, bytes_to_wif(k.get_bytes()),
                                             stakes)
            peers.append({'key': k, 'proof': proof, 'stake': stakes})

        def addavalanchenode(peer):
            pubkey = peer['key'].get_pubkey().get_bytes().hex()
            assert node.addavalanchenode(
                peer['node'].nodeid, pubkey, peer['proof']) is True

        p2p_idx = 0

        def get_ava_outbound(n, peer):
            nonlocal p2p_idx

            avapeer = AvaP2PInterface()
            n.add_outbound_p2p_connection(
                avapeer,
                p2p_idx=p2p_idx,
                connection_type="avalanche",
                services=NODE_NETWORK | NODE_AVALANCHE,
            )
            p2p_idx += 1
            avapeer.nodeid = node.getpeerinfo()[-1]['id']

            peer['node'] = avapeer
            addavalanchenode(peer)

            avapeer.wait_until(
                lambda: avapeer.last_message.get("getavaproofs"))
            avapeer.send_and_ping(msg_avaproofs())
            avapeer.wait_until(
                lambda: avapeer.last_message.get("avaproofsreq"))

            return avapeer

        # Start polling. The response should be UNKNOWN because there's no
        # score
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Create one peer with half the score and add one node
        get_ava_outbound(node, peers[0])
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Create a second peer with the other half and add one node.
        # This is not enough because we are lacking avaproofs messages
        get_ava_outbound(node, peers[1])
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Add more peers for triggering the avaproofs messaging
        for i in range(2, self.min_avaproofs_node_count - 1):
            get_ava_outbound(node, peers[i])
            poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        get_ava_outbound(node, peers[self.min_avaproofs_node_count - 1])
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

        # Disconnect peer 1's node which drops us below the threshold, but we've
        # latched that the quorum is established
        self.mock_forward(1)
        peers[1]['node'].peer_disconnect()
        peers[1]['node'].wait_for_disconnect()
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

        # Reconnect node and re-establish quorum
        get_ava_outbound(node, peers[1])
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

if __name__ == '__main__':
    AvalancheQuorumTest().main()
<commit_msg>[avalanche] Remove unused mock time from the quorum functional test<commit_after>#!/usr/bin/env python3
# Copyright (c) 2020-2022 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the quorum detection of avalanche."""

from test_framework.avatools import (
    AvaP2PInterface,
    create_coinbase_stakes,
    get_ava_p2p_interface,
)
from test_framework.key import ECKey, ECPubKey
from test_framework.messages import (
    NODE_AVALANCHE,
    NODE_NETWORK,
    AvalancheVote,
    AvalancheVoteError,
    msg_avaproofs,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal
from test_framework.wallet_util import bytes_to_wif

class AvalancheQuorumTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 1
        self.min_avaproofs_node_count = 8
        self.extra_args = [
            [
                '-enableavalanche=1',
                '-avacooldown=0',
                '-avatimeout=0',
                '-avaminquorumstake=100000000',
                '-avaminquorumconnectedstakeratio=0.8',
                f'-avaminavaproofsnodecount={self.min_avaproofs_node_count}',
            ]
        ]

    def run_test(self):
        # Create a local node to poll from and a helper to send polls from it
        # and assert on the response
        node = self.nodes[0]
        poll_node = get_ava_p2p_interface(node)
        poll_node_pubkey = ECPubKey()
        poll_node_pubkey.set(bytes.fromhex(node.getavalanchekey()))

        def poll_and_assert_response(expected):
            # Send poll for best block
            block = int(node.getbestblockhash(), 16)
            poll_node.send_poll([block])

            # Get response and check that the vote is what we expect
            response = poll_node.wait_for_avaresponse()
            r = response.response
            assert poll_node_pubkey.verify_schnorr(response.sig, r.get_hash())
            assert_equal(len(r.votes), 1)

            actual = repr(r.votes[0])
            expected = repr(AvalancheVote(expected, block))
            assert_equal(actual, expected)

        # Create peers to poll
        coinbase_key = node.get_deterministic_priv_key().key
        blocks = node.generate(self.min_avaproofs_node_count)
        peers = []

        for i in range(0, self.min_avaproofs_node_count):
            keyHex = f"12b004fff7f4b69ef8650e767f18f11ede158148b425660723b9f9a66e61f7{i:02}"
            k = ECKey()
            k.set(bytes.fromhex(keyHex), True)
            stakes = create_coinbase_stakes(
                node, [blocks[i]], coinbase_key)
            proof = node.buildavalancheproof(1, 1, bytes_to_wif(k.get_bytes()),
                                             stakes)
            peers.append({'key': k, 'proof': proof, 'stake': stakes})

        def addavalanchenode(peer):
            pubkey = peer['key'].get_pubkey().get_bytes().hex()
            assert node.addavalanchenode(
                peer['node'].nodeid, pubkey, peer['proof']) is True

        p2p_idx = 0

        def get_ava_outbound(n, peer):
            nonlocal p2p_idx

            avapeer = AvaP2PInterface()
            n.add_outbound_p2p_connection(
                avapeer,
                p2p_idx=p2p_idx,
                connection_type="avalanche",
                services=NODE_NETWORK | NODE_AVALANCHE,
            )
            p2p_idx += 1
            avapeer.nodeid = node.getpeerinfo()[-1]['id']

            peer['node'] = avapeer
            addavalanchenode(peer)

            avapeer.wait_until(
                lambda: avapeer.last_message.get("getavaproofs"))
            avapeer.send_and_ping(msg_avaproofs())
            avapeer.wait_until(
                lambda: avapeer.last_message.get("avaproofsreq"))

            return avapeer

        # Start polling. The response should be UNKNOWN because there's no
        # score
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Create one peer with half the score and add one node
        get_ava_outbound(node, peers[0])
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Create a second peer with the other half and add one node.
        # This is not enough because we are lacking avaproofs messages
        get_ava_outbound(node, peers[1])
        poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        # Add more peers for triggering the avaproofs messaging
        for i in range(2, self.min_avaproofs_node_count - 1):
            get_ava_outbound(node, peers[i])
            poll_and_assert_response(AvalancheVoteError.UNKNOWN)

        get_ava_outbound(node, peers[self.min_avaproofs_node_count - 1])
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

        # Disconnect peer 1's node which drops us below the threshold, but we've
        # latched that the quorum is established
        peers[1]['node'].peer_disconnect()
        peers[1]['node'].wait_for_disconnect()
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

        # Reconnect node and re-establish quorum
        get_ava_outbound(node, peers[1])
        poll_and_assert_response(AvalancheVoteError.ACCEPTED)

if __name__ == '__main__':
    AvalancheQuorumTest().main()
<|endoftext|>


<commit_before>from __future__ import absolute_import

from docker_registry_client._BaseClient import BaseClient
from docker_registry_client.Repository import Repository

class DockerRegistryClient(object):
    def __init__(self, host, verify_ssl=None):
        self._base_client = BaseClient(host, verify_ssl=verify_ssl)
        self._repositories = {}
        self._repositories_by_namespace = {}
        self.refresh()

    def namespaces(self):
        return list(self._repositories_by_namespace.keys())

    def repository(self, repository, namespace=None):
        if '/' in repository:
            if namespace is not None:
                raise RuntimeError('cannot specify namespace twice')
            namespace, repository = repository.split('/', 1)

        return Repository(self._base_client, repository, namespace=namespace)

    def repositories(self, namespace=None):
        if namespace:
            return self._repositories_by_namespace[namespace]

        return self._repositories

    def refresh(self):
        if self._base_client.version == 1:
            self._refresh_v1()
        else:
            assert self._base_client.version == 2
            self._refresh_v2()

    def _refresh_v1(self):
        _repositories = self._base_client.search()['results']
        for repository in _repositories:
            name = repository['name']
            ns, repo = name.split('/', 1)

            r = Repository(self._base_client, repo, namespace=ns)
            self._repositories_by_namespace.setdefault(ns, {})
            self._repositories_by_namespace[ns][name] = r
            self._repositories[name] = r

    def _refresh_v2(self):
        repositories = self._base_client.catalog()['repositories']
        for name in repositories:
            try:
                ns, repo = name.split('/', 1)
            except ValueError:
                ns = None
                repo = name

            r = Repository(self._base_client, repo, namespace=ns)

            if ns is None:
                ns = 'library'

            self._repositories_by_namespace.setdefault(ns, {})
            self._repositories_by_namespace[ns][name] = r
            self._repositories[name] = r
<commit_msg>Use search/catalog only when required<commit_after>from __future__ import absolute_import

from docker_registry_client._BaseClient import BaseClient
from docker_registry_client.Repository import Repository

class DockerRegistryClient(object):
    def __init__(self, host, verify_ssl=None):
        self._base_client = BaseClient(host, verify_ssl=verify_ssl)
        self._repositories = {}
        self._repositories_by_namespace = {}

    def namespaces(self):
        if not self._repositories:
            self.refresh()

        return list(self._repositories_by_namespace.keys())

    def repository(self, repository, namespace=None):
        if '/' in repository:
            if namespace is not None:
                raise RuntimeError('cannot specify namespace twice')
            namespace, repository = repository.split('/', 1)

        return Repository(self._base_client, repository, namespace=namespace)

    def repositories(self, namespace=None):
        if not self._repositories:
            self.refresh()

        if namespace:
            return self._repositories_by_namespace[namespace]

        return self._repositories

    def refresh(self):
        if self._base_client.version == 1:
            self._refresh_v1()
        else:
            assert self._base_client.version == 2
            self._refresh_v2()

    def _refresh_v1(self):
        _repositories = self._base_client.search()['results']
        for repository in _repositories:
            name = repository['name']
            ns, repo = name.split('/', 1)

            r = Repository(self._base_client, repo, namespace=ns)
            self._repositories_by_namespace.setdefault(ns, {})
            self._repositories_by_namespace[ns][name] = r
            self._repositories[name] = r

    def _refresh_v2(self):
        repositories = self._base_client.catalog()['repositories']
        for name in repositories:
            try:
                ns, repo = name.split('/', 1)
            except ValueError:
                ns = None
                repo = name

            r = Repository(self._base_client, repo, namespace=ns)

            if ns is None:
                ns = 'library'

            self._repositories_by_namespace.setdefault(ns, {})
            self._repositories_by_namespace[ns][name] = r
            self._repositories[name] = r
<|endoftext|>


<commit_before>from discord.ext import commands
from Cogs import Message
from Cogs import DL
from Cogs import PickList
from urllib.parse import quote
import re

def setup(bot):
    # Add the bot
    bot.add_cog(NvidiaArk(bot))

class NvidiaArk(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(no_pm=True, aliases=("nark", "nvark"))
    async def nvidiaark(self, ctx, *, text: str = None):
        """Searches TechPowerUP's site for GPU info."""

        args = {
            "title": "TechPowerUP Search",
            "description": "Usage: `{}nvark [gpu model]`".format(ctx.prefix),
            "footer": "Powered by https://www.techpowerup.com/",
            "color": ctx.author,
        }

        if not text:
            return await Message.EmbedText(**args).send(ctx)

        # Strip single quotes
        text = text.replace("'", "")
        text = re.sub(r"(?i)(nvidia)", "", text)

        # Ensure we have a space between models and "ti"
        text = " ".join([x[:-2]+" "+x[-2:] if len(x) > 4 and x[-2:].lower() == "ti" and x[:-2].isdigit() else x for x in text.split()])

        if not text:
            return await Message.EmbedText(**args).send(ctx)

        args["description"] = "Gathering info..."
        message = await Message.EmbedText(**args).send(ctx)

        response = await self.search(text)

        if response == -1:
            args[
                "description"
            ] = "I see what you tried... good one! But we only support NVidia queries."
            return await Message.EmbedText(**args).edit(ctx, message)

        if not response:
            args["description"] = "No results returned for `{}`.".format(
                text.replace("`", "").replace("\\", "")
            )
            return await Message.EmbedText(**args).edit(ctx, message)

        elif len(response) == 1:
            # Set it to the first item
            response = await self.get_match_data(response[0])

        elif len(response) > 1:
            # Allow the user to choose which one they meant.
            index, message = await PickList.Picker(
                message=message,
                title="Multiple Matches Returned For `{}`:".format(
                    text.replace("`", "").replace("\\", "")
                ),
                list=[x.get("name", "UNDETERMINABLE")
                      for x in self.prettify(response)],
                ctx=ctx,
            ).pick()

            if index < 0:
                args["description"] = "Search cancelled."
                return await Message.EmbedText(**args).edit(ctx, message)

            # Got something
            response = await self.get_match_data(response[index])

        # At this point - we should have a single response
        # Let's display the data.
        await PickList.PagePicker(
            title=response.get("name", "NVidia Search"),
            list=response["fields"],
            url=response.get("url"),
            thumbnail=response.get("image", None),
            footer="Powered by https://www.techpowerup.com/",
            color=ctx.author,
            ctx=ctx,
            max=18,
            message=message,
        ).pick()

    async def search(self, gpu_name):
        BASE_URL = "https://www.techpowerup.com"
        URL = BASE_URL + "/gpu-specs/?ajaxsrch={}".format(quote(gpu_name))

        contents = await DL.async_text(URL)

        if "nothing found" in contents.lower():
            return []

        if not "vendor-nvidia" in contents.lower():
            return -1

        nvidia = True
        lines = contents.split("\n")
        data = []
        url = ""
        temp_name = ""

        for line_index in range(len(lines)):
            line = lines[line_index].strip()

            if '<td class="vendor-' in line.lower():
                if not "vendor-nvidia" in line.lower():
                    nvidia = False
                elif "vendor-nvidia" in line.lower():
                    nvidia = True

            if not nvidia:
                continue

            if '")[1].split("<")[0]
                except IndexError:
                    continue

                if (
                    name.lower().strip() in gpu_name.lower().strip()
                    or gpu_name.lower().strip() in name.lower().strip()
                ):
                    if temp_name:
                        data.append({"name": temp_name, "url": url})
                        temp_name = ""

                    url = BASE_URL + \
                        lines[line_index].split('href="')[1].split('">')[0]
                    temp_name = name

                if "nvidia-" in line.lower():
                    temp_name += " ({0})".format(name)

            if (
                temp_name
                and "" in line.lower()
                and ("GB" in line or "MB" in line or "KB" in line)
            ):
                temp_name += " -> {0} {1} {2}".format(
                    "{", line.split(">")[1].split("<")[0], "}"
                )

        return data

    async def get_match_data(self, match):
        BASE_URL = "https://www.techpowerup.com"
        contents = await DL.async_text(match["url"])
        lines = contents.split("\n")
        blacklist = {
            "Relative Performance",
            "Theoretical Performance",
            "Retail boards based on this design",
        }
        data = {"name": "", "url": match["url"], "image": None, "fields": []}
        capturing = False

        for line_index in range(len(lines)):
            line = lines[line_index]

            if '
' in line.lower():
                try:
                    clean = line.split(">")[1].split("<")[0].strip()
                    data["name"] = clean
                    continue
                except IndexError:
                    continue

            if '<img class="gpudb-large-image' in line.lower() and not data["image"]:
                data["image"] = line.split('src="')[1].split('"')[0]

            if any(x.lower() in line.lower() for x in blacklist):
                capturing = False
                continue

            if "
" in line.lower() and "
" != line.lower().strip():
                if "
" != line.lower().strip():
                    capturing = True
                    continue
                else:
                    capturing = False
                    continue

            if '
' in line.lower() and capturing:
                title = ""
                value = ""
                value_url = ""

                try:
                    if "
" in lines[line_index + 1].strip():
                        title = lines[line_index +
                                      1].split(">")[1].split("<")[0]
                        value = (
                            lines[line_index + 3].strip()
                            if "
" in lines[line_index + 2].strip()
                            or "
" == lines[line_index + 2].strip()
                            else lines[line_index + 2].split("dd>")[1].split("</dd")[0]
                        )

                        if '<a href="' in value.lower():
                            value_url = value.split(
                                'href="')[1].split('"')[0].strip()
                            value = (
                                value.split(
                                    ''.format(value_url))[1]
                                .split("")[0]
                                .strip()
                            )

                        value = value.replace("</", "").replace("
", " ")

                        if "<span" in value.lower():
                            value = value.split("<")[0].strip()

                    elif '
' in lines[line_index + 1].strip():
                        title = lines[line_index +
                                      2].split(">")[1].split("<")[0]
                        value = (
                            lines[line_index + 4].strip()
                            if "
" in lines[line_index + 3].strip()
                            or "
" == lines[line_index + 3].strip()
                            else lines[line_index + 3].split("dd>")[1].split("</dd")[0]
                        )

                        if '<a href="' in value.lower():
                            value_url = value.split(
                                'href="')[1].split('"')[0].strip()
                            value = (
                                value.split(
                                    ''.format(value_url))[1]
                                .split("")[0]
                                .strip()
                            )

                        value = value.replace("</", "").replace("
", " ")

                        if "<span" in value.lower():
                            value = value.split("<")[0].strip()
                except IndexError:
                    continue

                if "current price" in title.lower():
                    continue

                if title and value:
                    formatted = (
                        "[{0}]({1})".format(value, BASE_URL + value_url)
                        if value_url
                        else value
                    )

                    if any(
                        x.get("name") == title or x.get("value") == formatted
                        for x in data["fields"]
                    ):
                        continue

                    data["fields"].append(
                        {"name": title, "value": formatted, "inline": True}
                    )

        return data

    def prettify(self, items):
        # Extracts the longest item from a filtered list.
        #
        # FILTERING:
        # For each item, we do the following:
        #
        #     GeForce 940M (GM107) -> { 2GB, DDR3, 64 bit }   <-- split by ' (' here
        #
        #           => [ 'GeForce 940M', 'GM107) -> { 2GB, DDR3, 64 bit }
        #       [0] -> [ 'GeForce 940M']
        #       ^ index after splitting on ' ('
        #
        # We are essentially mapping each item by
        # capturing what comes before `(GM107)`
        longest = len(max([x.get("name", "").split(" (")[0]
                      for x in items], key=len))

        for index in range(len(items)):
            try:
                first, second = items[index].get("name", "").split(" (")
            except ValueError:
                continue

            items[index]["name"] = "{0}{1}{2}".format(
                first, " " * (longest - len(first) + 2) + "(", second
            )

        return self._prettify_ram(items)

    def _prettify_ram(self, items):
        # Extracts the longest item from a filtered list.
        #
        # FILTERING:
        # For each item, we do the following:
        #
        #     GeForce 940M (GM107) -> { 2GB, DDR3, 64 bit }   <-- split by '{' here
        #
        #           => [ 'GeForce 940M (GM107) -> ', '2GB, DDR3, 64 bit }' ]
        #       [1] -> [ '2GB, DDR3, 64 bit }'] <-- split by '}' here
        #       [0] -> [ '2GB, DDR3, 64 bit' ]
        #       ^ index after splitting on '}'
        #
        # We are essentially mapping each item by
        # capturing what's inside of `{` and `}`
        longest = len(max([x.get("name", "").split(
            "{ ")[1].split(" }")[0] for x in items], key=len))

        for index in range(len(items)):
            try:
                first, item = [
                    x.replace(" }", "") for x in items[index].get("name", "").split("{ ")]
            except ValueError:
                continue

            from math import floor, ceil

            total = longest + 4
            N = len(" " * (ceil((total - len(item)) / 2)))
            J = len(" " * (floor((total - len(item)) / 2)))

            items[index]["name"] = "{0}{1}{2}{3}{4}{5}".format(
                first, "{", " " * (N + 1), item, " " * (J + 1), "}"
            )

        return items
<commit_msg>Only allow up to 10 responses<commit_after>from discord.ext import commands
from Cogs import Message
from Cogs import DL
from Cogs import PickList
from urllib.parse import quote
import re

def setup(bot):
    # Add the bot
    bot.add_cog(NvidiaArk(bot))

class NvidiaArk(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(no_pm=True, aliases=("nark", "nvark"))
    async def nvidiaark(self, ctx, *, text: str = None):
        """Searches TechPowerUP's site for GPU info."""

        args = {
            "title": "TechPowerUP Search",
            "description": "Usage: `{}nvark [gpu model]`".format(ctx.prefix),
            "footer": "Powered by https://www.techpowerup.com/",
            "color": ctx.author,
        }

        if not text:
            return await Message.EmbedText(**args).send(ctx)

        # Strip single quotes
        text = text.replace("'", "")
        text = re.sub(r"(?i)(nvidia)", "", text)

        # Ensure we have a space between models and "ti"
        text = " ".join([x[:-2]+" "+x[-2:] if len(x) > 4 and x[-2:].lower() == "ti" and x[:-2].isdigit() else x for x in text.split()])

        if not text:
            return await Message.EmbedText(**args).send(ctx)

        args["description"] = "Gathering info..."
        message = await Message.EmbedText(**args).send(ctx)

        response = await self.search(text)

        if response == -1:
            args[
                "description"
            ] = "I see what you tried... good one! But we only support NVidia queries."
            return await Message.EmbedText(**args).edit(ctx, message)

        if not response:
            args["description"] = "No results returned for `{}`.".format(
                text.replace("`", "").replace("\\", "")
            )
            return await Message.EmbedText(**args).edit(ctx, message)

        elif len(response) == 1:
            # Set it to the first item
            response = await self.get_match_data(response[0])

        elif len(response) > 1:
            # Allow the user to choose which one they meant.
            index, message = await PickList.Picker(
                message=message,
                title="Multiple Matches Returned For `{}`:".format(
                    text.replace("`", "").replace("\\", "")
                ),
                list=[x.get("name", "UNDETERMINABLE")
                      for x in self.prettify(response[:10])],
                ctx=ctx,
            ).pick()

            if index < 0:
                args["description"] = "Search cancelled."
                return await Message.EmbedText(**args).edit(ctx, message)

            # Got something
            response = await self.get_match_data(response[index])

        # At this point - we should have a single response
        # Let's display the data.
        await PickList.PagePicker(
            title=response.get("name", "NVidia Search"),
            list=response["fields"],
            url=response.get("url"),
            thumbnail=response.get("image", None),
            footer="Powered by https://www.techpowerup.com/",
            color=ctx.author,
            ctx=ctx,
            max=18,
            message=message,
        ).pick()

    async def search(self, gpu_name):
        BASE_URL = "https://www.techpowerup.com"
        URL = BASE_URL + "/gpu-specs/?ajaxsrch={}".format(quote(gpu_name))

        contents = await DL.async_text(URL)

        if "nothing found" in contents.lower():
            return []

        if not "vendor-nvidia" in contents.lower():
            return -1

        nvidia = True
        lines = contents.split("\n")
        data = []
        url = ""
        temp_name = ""

        for line_index in range(len(lines)):
            line = lines[line_index].strip()

            if '<td class="vendor-' in line.lower():
                if not "vendor-nvidia" in line.lower():
                    nvidia = False
                elif "vendor-nvidia" in line.lower():
                    nvidia = True

            if not nvidia:
                continue

            if '")[1].split("<")[0]
                except IndexError:
                    continue

                if (
                    name.lower().strip() in gpu_name.lower().strip()
                    or gpu_name.lower().strip() in name.lower().strip()
                ):
                    if temp_name:
                        data.append({"name": temp_name, "url": url})
                        temp_name = ""

                    url = BASE_URL + \
                        lines[line_index].split('href="')[1].split('">')[0]
                    temp_name = name

                if "nvidia-" in line.lower():
                    temp_name += " ({0})".format(name)

            if (
                temp_name
                and "" in line.lower()
                and ("GB" in line or "MB" in line or "KB" in line)
            ):
                temp_name += " -> {0} {1} {2}".format(
                    "{", line.split(">")[1].split("<")[0], "}"
                )

        return data

    async def get_match_data(self, match):
        BASE_URL = "https://www.techpowerup.com"
        contents = await DL.async_text(match["url"])
        lines = contents.split("\n")
        blacklist = {
            "Relative Performance",
            "Theoretical Performance",
            "Retail boards based on this design",
        }
        data = {"name": "", "url": match["url"], "image": None, "fields": []}
        capturing = False

        for line_index in range(len(lines)):
            line = lines[line_index]

            if '
' in line.lower():
                try:
                    clean = line.split(">")[1].split("<")[0].strip()
                    data["name"] = clean
                    continue
                except IndexError:
                    continue

            if '<img class="gpudb-large-image' in line.lower() and not data["image"]:
                data["image"] = line.split('src="')[1].split('"')[0]

            if any(x.lower() in line.lower() for x in blacklist):
                capturing = False
                continue

            if "
" in line.lower() and "
" != line.lower().strip():
                if "
" != line.lower().strip():
                    capturing = True
                    continue
                else:
                    capturing = False
                    continue

            if '
' in line.lower() and capturing:
                title = ""
                value = ""
                value_url = ""

                try:
                    if "
" in lines[line_index + 1].strip():
                        title = lines[line_index +
                                      1].split(">")[1].split("<")[0]
                        value = (
                            lines[line_index + 3].strip()
                            if "
" in lines[line_index + 2].strip()
                            or "
" == lines[line_index + 2].strip()
                            else lines[line_index + 2].split("dd>")[1].split("</dd")[0]
                        )

                        if '<a href="' in value.lower():
                            value_url = value.split(
                                'href="')[1].split('"')[0].strip()
                            value = (
                                value.split(
                                    ''.format(value_url))[1]
                                .split("")[0]
                                .strip()
                            )

                        value = value.replace("</", "").replace("
", " ")

                        if "<span" in value.lower():
                            value = value.split("<")[0].strip()

                    elif '
' in lines[line_index + 1].strip():
                        title = lines[line_index +
                                      2].split(">")[1].split("<")[0]
                        value = (
                            lines[line_index + 4].strip()
                            if "
" in lines[line_index + 3].strip()
                            or "
" == lines[line_index + 3].strip()
                            else lines[line_index + 3].split("dd>")[1].split("</dd")[0]
                        )

                        if '<a href="' in value.lower():
                            value_url = value.split(
                                'href="')[1].split('"')[0].strip()
                            value = (
                                value.split(
                                    ''.format(value_url))[1]
                                .split("")[0]
                                .strip()
                            )

                        value = value.replace("</", "").replace("
", " ")

                        if "<span" in value.lower():
                            value = value.split("<")[0].strip()
                except IndexError:
                    continue

                if "current price" in title.lower():
                    continue

                if title and value:
                    formatted = (
                        "[{0}]({1})".format(value, BASE_URL + value_url)
                        if value_url
                        else value
                    )

                    if any(
                        x.get("name") == title or x.get("value") == formatted
                        for x in data["fields"]
                    ):
                        continue

                    data["fields"].append(
                        {"name": title, "value": formatted, "inline": True}
                    )

        return data

    def prettify(self, items):
        # Extracts the longest item from a filtered list.
        #
        # FILTERING:
        # For each item, we do the following:
        #
        #     GeForce 940M (GM107) -> { 2GB, DDR3, 64 bit }   <-- split by ' (' here
        #
        #           => [ 'GeForce 940M', 'GM107) -> { 2GB, DDR3, 64 bit }
        #       [0] -> [ 'GeForce 940M']
        #       ^ index after splitting on ' ('
        #
        # We are essentially mapping each item by
        # capturing what comes before `(GM107)`
        longest = len(max([x.get("name", "").split(" (")[0]
                      for x in items], key=len))

        for index in range(len(items)):
            try:
                first, second = items[index].get("name", "").split(" (")
            except ValueError:
                continue

            items[index]["name"] = "{0}{1}{2}".format(
                first, " " * (longest - len(first) + 2) + "(", second
            )

        return self._prettify_ram(items)

    def _prettify_ram(self, items):
        # Extracts the longest item from a filtered list.
        #
        # FILTERING:
        # For each item, we do the following:
        #
        #     GeForce 940M (GM107) -> { 2GB, DDR3, 64 bit }   <-- split by '{' here
        #
        #           => [ 'GeForce 940M (GM107) -> ', '2GB, DDR3, 64 bit }' ]
        #       [1] -> [ '2GB, DDR3, 64 bit }'] <-- split by '}' here
        #       [0] -> [ '2GB, DDR3, 64 bit' ]
        #       ^ index after splitting on '}'
        #
        # We are essentially mapping each item by
        # capturing what's inside of `{` and `}`
        longest = len(max([x.get("name", "").split(
            "{ ")[1].split(" }")[0] for x in items], key=len))

        for index in range(len(items)):
            try:
                first, item = [
                    x.replace(" }", "") for x in items[index].get("name", "").split("{ ")]
            except ValueError:
                continue

            from math import floor, ceil

            total = longest + 4
            N = len(" " * (ceil((total - len(item)) / 2)))
            J = len(" " * (floor((total - len(item)) / 2)))

            items[index]["name"] = "{0}{1}{2}{3}{4}{5}".format(
                first, "{", " " * (N + 1), item, " " * (J + 1), "}"
            )

        return items
<|endoftext|>