<commit_before>#!/usr/bin/env python

# Copyright 2018-2020 Arm Limited.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import logging
import multiprocessing
import os
import subprocess
import sys
import shutil

import copy_with_deps
import kernel_config_parser

logger = logging.getLogger(__name__)

kernel_search_paths = [
    "{kdir}/arch/{arch}/include",
    "{kdir}/arch/{arch}/include/generated",
    "{kdir}/arch/{arch}/include/generated", "uapi",
    "{kdir}/arch/{arch}/include/uapi",
    "{kdir}/include",
    "{kdir}/include/generated/uapi",
    "{kdir}/include/uapi"
]

def parse_kbuild_options(kbuild_options):
    """Return a dictionary of kbuild_options {option:value}"""
    options = dict()

    for option in kbuild_options:
        parts = option.split("=", 1)

        if len(parts) != 2:
            logger.error("Invalid Kbuild option: '%s'", option)
        else:
            options[parts[0]] = parts[1]

    return options

def check_kbuild_option_conflicts(kdir, option, value):
    """
    Checks whether kbuild_option is already defined in the kernel config
    causing conflicts by overriding them in EXTRA_CFLAGS
    :return: Zero(0) if there is no conflict, one(1) otherwise
    """
    message = "Overridden '{0}' option in EXTRA_CFLAGS. " \
              "Bob was asked to set the kernel option '{0}={1}', " \
              "which is already present in the kernel's config. " \
              "Please disable this option in the kernel " \
              "or disable out-of-tree kernel module builds in Bob to continue."

    # There are few cases when Bob is allowed to override a kernel option:
    #
    # 1. If the kernel has an option present, it is set to `y` or `m` and Bob is overriding it
    #    with the same value
    # 2. If the kernel has an option present, it is set to `n` and Bob is overriding it
    #    either with `y` or `m` or `n`
    # 3. If the kernel has an option present, it is set to any value and Bob is overriding it
    #    with the same value (imply first case)

    k_option_val = kernel_config_parser.get_value(kdir, option)

    if k_option_val:
        if not (k_option_val == value) and not (k_option_val == 'n' and value in ['y', 'm']):
            logger.error(message.format(option, value))
            return 1

    return 0

def kbuild_to_cflag(option, value):
    if value in ['m', 'y']:
        cflag = str.format("-D{}=1", option)
    elif value == 'n':
        cflag = str.format("-U{}", option)
    elif value.isdigit():
        cflag = str.format("-D{}={}", option, value)
    else:
        # passing values as string needs special treatment due to shell interpretation
        # -DCONFIG_A='"string value"' conforms to #define CONFIG_A "string value"
        cflag = str.format("-D{}='\"{}\"'", option, value)

    return cflag

def build_module(output_dir, module_ko, kdir, module_dir, make_command, make_args, extra_cflags):
    """
    Invoke an out of tree kernel build.
    """
    # Invoke the kernel build system
    cmd = [make_command, "-C", kdir, "M=" + module_dir, "EXTRA_CFLAGS=" + extra_cflags]
    cmd.extend(make_args)

    # Sanitize the environment - we should only use build options passed in via
    # the command line.
    env = dict(os.environ)
    for var in ["ARCH", "CROSS_COMPILE", "CC", "HOSTCC",
                "CLANG_TRIPLE", "KBUILD_EXTRA_SYMBOLS", "LD"]:
        env.pop(var, None)

    try:
        subprocess.check_call(cmd, env=env)
    except subprocess.CalledProcessError as e:
        logger.error("Command failed: %s", str(e.cmd))
        sys.exit(e.returncode)

    # Copy the output of the kernel build to the directory that Bob expects
    built_files = [module_ko, "Module.symvers"]
    for built_file in built_files:
        try:
            # Don't copy if already existing in desired location
            if module_dir != os.path.abspath(output_dir):
                abs_built_file = os.path.join(module_dir, built_file)
                shutil.copy(abs_built_file, output_dir)
        except (OSError, IOError) as e:
            msg = "Copy file from input path: {}\n" \
                  "to output path: {}\n" \
                  "finished with error: {}"
            logger.error(msg.format(abs_built_file, output_dir, e))
            sys.exit(1)

def get_tool_abspath(tool):
    """Get absolute path to tool if argument contains a path otherwise assume it's a $PATH tool
    :param tool: path to tool or $PATH prefix
    :return: Absolute path or tool
    """
    if tool and os.path.dirname(tool):
        return os.path.abspath(tool)
    return tool

def parse_source_list(sources):
    module_sources = []
    extra_symbols = []
    for source in sources:
        if os.path.basename(source) == "Module.symvers":
            extra_symbols.append(source)
        elif os.path.splitext(source)[1] == ".ko":
            # Ignore .ko files - we will detect their symbols via their
            # corresponding Module.symvers file.
            pass
        else:
            module_sources.append(source)

    return module_sources, extra_symbols

def parse_output_list(parser, outputs):
    """When this script is called from a `genrule` module, the .ko _and_
    Module.symvers file may be listed as outputs. Filter out the symvers file.
    """
    module_output = None
    module_symvers = None
    for output in outputs:
        if os.path.basename(output) == "Module.symvers":
            if module_symvers:
                parser.error("Module.symvers specified multiple times: {} and {}".format(
                             module_symvers, output))
            module_symvers = output
        elif os.path.splitext(output)[1] == ".ko":
            if module_output:
                parser.error(".ko output specified multiple times: {} and {}".format(
                             module_output, output))
            module_output = output
        else:
            parser.error("Unknown output file type: {}".format(os.path.basename(output)))

    if not module_output:
        parser.error("No .ko output file specified")

    return module_output

def parse_args():
    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.WARNING)

    cli_description = "Encapsulate an out-of-tree kernel module build, " \
                      "where the build does not modify the source directory"
    parser = argparse.ArgumentParser(description=cli_description)
    parser.add_argument("--output", "-o", required=True, nargs="+",
                        help="Kernel module to build (including output path)")
    parser.add_argument("--sources", "-s", metavar="FILE", nargs="+", required=True,
                        help="Kernel module source files")
    parser.add_argument("--depfile", "-d", metavar="DEPFILE", required=True,
                        help="Dependency file to generate")
    parser.add_argument("--common-root", "-r", required=True,
                        help="Common root directory that can be stripped from source paths")
    parser.add_argument("--module-dir", "-m",
                        help="Module output directory in kernel build")
    parser.add_argument("--jobs", "-j", metavar="N", default=None, type=int,
                        help="Allow N jobs at once")
    parser.add_argument("--make-command", "-M", default="make",
                        help="Path to `make` command")

    group = parser.add_argument_group("Kernel options")
    group.add_argument("--kernel", "-k", metavar="KDIR", required=True,
                       help="Kernel directory")
    group.add_argument("--cc", default=None,
                       help="Target C compiler")
    group.add_argument("--hostcc", default=None,
                       help="Host C compiler")
    group.add_argument("--cross-compile", default=None,
                       help="Kernel CROSS_COMPILE")
    group.add_argument("--clang-triple", default=None,
                       help="Kernel CLANG_TRIPLE")
    group.add_argument("--ld", default=None,
                       help="Kernel LD")
    group.add_argument("--kbuild-options", nargs="+", default=[],
                       help="Kernel config options to enable, that get added to EXTRA_CFLAGS too")
    group.add_argument("--extra-cflags", default="",
                       help="Options to add to EXTRA_CFLAGS as a string")
    group.add_argument("make_args", nargs=argparse.REMAINDER, default=[],
                       help="Make variables to be set")

    group = parser.add_argument_group("Dependency checking")
    group.add_argument("--include-dir", "-I", metavar="INCLUDE_DIR", action="append", default=[],
                       help="Include file search path")

    args = parser.parse_args()

    args.module_sources, args.extra_symbols = parse_source_list(args.sources)
    args.output = parse_output_list(parser, args.output)

    return args

def main():
    args = parse_args()

    # The build is run in $KDIR, rather than the usual build workdir, so
    # parameters need to be absolute so they are accessible with a different CWD.
    output_dir = os.path.dirname(args.output)
    abs_output_dir = os.path.abspath(output_dir)
    abs_kdir = os.path.abspath(args.kernel)
    search_path = [os.path.abspath(d) for d in args.include_dir]

    cross_compile = get_tool_abspath(args.cross_compile)
    target_cc = get_tool_abspath(args.cc)
    host_cc = get_tool_abspath(args.hostcc)
    make_command = get_tool_abspath(args.make_command)

    # Check the kernel ARCH
    arch = kernel_config_parser.get_arch(abs_kdir)
    if not arch:
        sys.exit(1)

    kbuild_cflags = []
    kbuild_conflicts = 0
    # Parse kbuild_options and make them cflags style
    for option, value in parse_kbuild_options(args.kbuild_options).items():
        kbuild_conflicts += check_kbuild_option_conflicts(abs_kdir, option, value)
        kbuild_cflags.append(kbuild_to_cflag(option, value))

    if kbuild_conflicts > 0:
        sys.exit(1)

    # Prepend EXTRA_CFLAGS with modified include paths
    includes = ["-I" + s for s in search_path]
    extra_cflags = " ".join(includes) + " " + args.extra_cflags + " " + \
                   " ".join(kbuild_cflags)

    deps = []

    # Add commonly needed search paths for copy_with_deps
    search_path.extend([str.format(d, kdir=abs_kdir, arch=arch) for d in kernel_search_paths])
    kconfig = os.path.join(abs_kdir, "linux", "kconfig.h")
    root = os.path.abspath(args.common_root)
    for src in args.module_sources:
        src_rel = os.path.relpath(os.path.abspath(src), root)
        if src_rel.startswith("../"):
            msg = "Source path: %s doesn't share common root directory: %s"
            logger.error(msg, src, args.common_root)
            sys.exit(1)

        dest = os.path.join(output_dir, src_rel)
        deps.extend(copy_with_deps.copy_with_deps(src, dest, search_path, [kconfig]))

    deps = sorted(set(deps))

    # Add a dependency on copy_with_deps.py, which won't have been set by Bob
    deps.append(os.path.join(os.path.dirname(sys.argv[0]), "copy_with_deps.py"))

    # Add a dependency on the test kernel Makefile. We do not attempt to add
    # dependencies on every part of the kernel's build system - this is just
    # enough to ensure that incremental builds of the Bob tests work OK.
    deps.append(os.path.join(abs_kdir, "Makefile"))

    copy_with_deps.write_depfile(args.depfile, args.output, deps)

    make_args = args.make_args
    make_args.extend(args.kbuild_options)
    make_args.append("ARCH=" + arch)

    # CROSS_COMPILE is still required with CC=clang
    if cross_compile:
        make_args.append("CROSS_COMPILE=" + cross_compile)
    if target_cc:
        make_args.append("CC=" + target_cc)
    if host_cc:
        make_args.append("HOSTCC=" + host_cc)
    if args.clang_triple:
        make_args.append("CLANG_TRIPLE=" + args.clang_triple)
    if args.ld:
        make_args.append("LD=" + args.ld)
    elif kernel_config_parser.option_enabled(abs_kdir, "CONFIG_LD_IS_LLD"):
        # Auto-set LD to `ld.lld` if LTO has been enabled
        make_args.append("LD=ld.lld")
    if args.extra_symbols:
        extra_symbols = [os.path.abspath(d) for d in args.extra_symbols]
        make_args.append("KBUILD_EXTRA_SYMBOLS=" + " ".join(extra_symbols))

    if args.jobs:
        make_args.append("-j" + str(args.jobs))
    else:
        # If the following env var is set, we are running in a build
        # farm where we should avoid increasing thread
        # count. Therefore leave make to run with a single core.
        # If not, build kernel modules with the number of CPUs we have.
        if os.getenv("MPDTI_BUILD_PARALLELISM") is None:
            make_args.append("-j" + str(multiprocessing.cpu_count()))

    module_ko = os.path.basename(args.output)
    abs_module_dir = os.path.abspath(args.module_dir)
    build_module(output_dir, module_ko, abs_kdir, abs_module_dir,
                 make_command, make_args, extra_cflags)

if __name__ == "__main__":
    main()
<commit_msg>Fix kconfig header lookup<commit_after>#!/usr/bin/env python

# Copyright 2018-2021 Arm Limited.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import logging
import multiprocessing
import os
import subprocess
import sys
import shutil

import copy_with_deps
import kernel_config_parser

logger = logging.getLogger(__name__)

kernel_search_paths = [
    "{kdir}/arch/{arch}/include",
    "{kdir}/arch/{arch}/include/generated",
    "{kdir}/arch/{arch}/include/generated", "uapi",
    "{kdir}/arch/{arch}/include/uapi",
    "{kdir}/include",
    "{kdir}/include/generated/uapi",
    "{kdir}/include/uapi"
]

def parse_kbuild_options(kbuild_options):
    """Return a dictionary of kbuild_options {option:value}"""
    options = dict()

    for option in kbuild_options:
        parts = option.split("=", 1)

        if len(parts) != 2:
            logger.error("Invalid Kbuild option: '%s'", option)
        else:
            options[parts[0]] = parts[1]

    return options

def check_kbuild_option_conflicts(kdir, option, value):
    """
    Checks whether kbuild_option is already defined in the kernel config
    causing conflicts by overriding them in EXTRA_CFLAGS
    :return: Zero(0) if there is no conflict, one(1) otherwise
    """
    message = "Overridden '{0}' option in EXTRA_CFLAGS. " \
              "Bob was asked to set the kernel option '{0}={1}', " \
              "which is already present in the kernel's config. " \
              "Please disable this option in the kernel " \
              "or disable out-of-tree kernel module builds in Bob to continue."

    # There are few cases when Bob is allowed to override a kernel option:
    #
    # 1. If the kernel has an option present, it is set to `y` or `m` and Bob is overriding it
    #    with the same value
    # 2. If the kernel has an option present, it is set to `n` and Bob is overriding it
    #    either with `y` or `m` or `n`
    # 3. If the kernel has an option present, it is set to any value and Bob is overriding it
    #    with the same value (imply first case)

    k_option_val = kernel_config_parser.get_value(kdir, option)

    if k_option_val:
        if not (k_option_val == value) and not (k_option_val == 'n' and value in ['y', 'm']):
            logger.error(message.format(option, value))
            return 1

    return 0

def kbuild_to_cflag(option, value):
    if value in ['m', 'y']:
        cflag = str.format("-D{}=1", option)
    elif value == 'n':
        cflag = str.format("-U{}", option)
    elif value.isdigit():
        cflag = str.format("-D{}={}", option, value)
    else:
        # passing values as string needs special treatment due to shell interpretation
        # -DCONFIG_A='"string value"' conforms to #define CONFIG_A "string value"
        cflag = str.format("-D{}='\"{}\"'", option, value)

    return cflag

def build_module(output_dir, module_ko, kdir, module_dir, make_command, make_args, extra_cflags):
    """
    Invoke an out of tree kernel build.
    """
    # Invoke the kernel build system
    cmd = [make_command, "-C", kdir, "M=" + module_dir, "EXTRA_CFLAGS=" + extra_cflags]
    cmd.extend(make_args)

    # Sanitize the environment - we should only use build options passed in via
    # the command line.
    env = dict(os.environ)
    for var in ["ARCH", "CROSS_COMPILE", "CC", "HOSTCC",
                "CLANG_TRIPLE", "KBUILD_EXTRA_SYMBOLS", "LD"]:
        env.pop(var, None)

    try:
        subprocess.check_call(cmd, env=env)
    except subprocess.CalledProcessError as e:
        logger.error("Command failed: %s", str(e.cmd))
        sys.exit(e.returncode)

    # Copy the output of the kernel build to the directory that Bob expects
    built_files = [module_ko, "Module.symvers"]
    for built_file in built_files:
        try:
            # Don't copy if already existing in desired location
            if module_dir != os.path.abspath(output_dir):
                abs_built_file = os.path.join(module_dir, built_file)
                shutil.copy(abs_built_file, output_dir)
        except (OSError, IOError) as e:
            msg = "Copy file from input path: {}\n" \
                  "to output path: {}\n" \
                  "finished with error: {}"
            logger.error(msg.format(abs_built_file, output_dir, e))
            sys.exit(1)

def get_tool_abspath(tool):
    """Get absolute path to tool if argument contains a path otherwise assume it's a $PATH tool
    :param tool: path to tool or $PATH prefix
    :return: Absolute path or tool
    """
    if tool and os.path.dirname(tool):
        return os.path.abspath(tool)
    return tool

def parse_source_list(sources):
    module_sources = []
    extra_symbols = []
    for source in sources:
        if os.path.basename(source) == "Module.symvers":
            extra_symbols.append(source)
        elif os.path.splitext(source)[1] == ".ko":
            # Ignore .ko files - we will detect their symbols via their
            # corresponding Module.symvers file.
            pass
        else:
            module_sources.append(source)

    return module_sources, extra_symbols

def parse_output_list(parser, outputs):
    """When this script is called from a `genrule` module, the .ko _and_
    Module.symvers file may be listed as outputs. Filter out the symvers file.
    """
    module_output = None
    module_symvers = None
    for output in outputs:
        if os.path.basename(output) == "Module.symvers":
            if module_symvers:
                parser.error("Module.symvers specified multiple times: {} and {}".format(
                             module_symvers, output))
            module_symvers = output
        elif os.path.splitext(output)[1] == ".ko":
            if module_output:
                parser.error(".ko output specified multiple times: {} and {}".format(
                             module_output, output))
            module_output = output
        else:
            parser.error("Unknown output file type: {}".format(os.path.basename(output)))

    if not module_output:
        parser.error("No .ko output file specified")

    return module_output

def parse_args():
    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.WARNING)

    cli_description = "Encapsulate an out-of-tree kernel module build, " \
                      "where the build does not modify the source directory"
    parser = argparse.ArgumentParser(description=cli_description)
    parser.add_argument("--output", "-o", required=True, nargs="+",
                        help="Kernel module to build (including output path)")
    parser.add_argument("--sources", "-s", metavar="FILE", nargs="+", required=True,
                        help="Kernel module source files")
    parser.add_argument("--depfile", "-d", metavar="DEPFILE", required=True,
                        help="Dependency file to generate")
    parser.add_argument("--common-root", "-r", required=True,
                        help="Common root directory that can be stripped from source paths")
    parser.add_argument("--module-dir", "-m",
                        help="Module output directory in kernel build")
    parser.add_argument("--jobs", "-j", metavar="N", default=None, type=int,
                        help="Allow N jobs at once")
    parser.add_argument("--make-command", "-M", default="make",
                        help="Path to `make` command")

    group = parser.add_argument_group("Kernel options")
    group.add_argument("--kernel", "-k", metavar="KDIR", required=True,
                       help="Kernel directory")
    group.add_argument("--cc", default=None,
                       help="Target C compiler")
    group.add_argument("--hostcc", default=None,
                       help="Host C compiler")
    group.add_argument("--cross-compile", default=None,
                       help="Kernel CROSS_COMPILE")
    group.add_argument("--clang-triple", default=None,
                       help="Kernel CLANG_TRIPLE")
    group.add_argument("--ld", default=None,
                       help="Kernel LD")
    group.add_argument("--kbuild-options", nargs="+", default=[],
                       help="Kernel config options to enable, that get added to EXTRA_CFLAGS too")
    group.add_argument("--extra-cflags", default="",
                       help="Options to add to EXTRA_CFLAGS as a string")
    group.add_argument("make_args", nargs=argparse.REMAINDER, default=[],
                       help="Make variables to be set")

    group = parser.add_argument_group("Dependency checking")
    group.add_argument("--include-dir", "-I", metavar="INCLUDE_DIR", action="append", default=[],
                       help="Include file search path")

    args = parser.parse_args()

    args.module_sources, args.extra_symbols = parse_source_list(args.sources)
    args.output = parse_output_list(parser, args.output)

    return args

def main():
    args = parse_args()

    # The build is run in $KDIR, rather than the usual build workdir, so
    # parameters need to be absolute so they are accessible with a different CWD.
    output_dir = os.path.dirname(args.output)
    abs_output_dir = os.path.abspath(output_dir)
    abs_kdir = os.path.abspath(args.kernel)
    search_path = [os.path.abspath(d) for d in args.include_dir]

    cross_compile = get_tool_abspath(args.cross_compile)
    target_cc = get_tool_abspath(args.cc)
    host_cc = get_tool_abspath(args.hostcc)
    make_command = get_tool_abspath(args.make_command)

    # Check the kernel ARCH
    arch = kernel_config_parser.get_arch(abs_kdir)
    if not arch:
        sys.exit(1)

    kbuild_cflags = []
    kbuild_conflicts = 0
    # Parse kbuild_options and make them cflags style
    for option, value in parse_kbuild_options(args.kbuild_options).items():
        kbuild_conflicts += check_kbuild_option_conflicts(abs_kdir, option, value)
        kbuild_cflags.append(kbuild_to_cflag(option, value))

    if kbuild_conflicts > 0:
        sys.exit(1)

    # Prepend EXTRA_CFLAGS with modified include paths
    includes = ["-I" + s for s in search_path]
    extra_cflags = " ".join(includes) + " " + args.extra_cflags + " " + \
                   " ".join(kbuild_cflags)

    deps = []

    # Add commonly needed search paths for copy_with_deps
    search_path.extend([str.format(d, kdir=abs_kdir, arch=arch) for d in kernel_search_paths])
    kconfig = os.path.join("linux", "kconfig.h")
    root = os.path.abspath(args.common_root)
    for src in args.module_sources:
        src_rel = os.path.relpath(os.path.abspath(src), root)
        if src_rel.startswith("../"):
            msg = "Source path: %s doesn't share common root directory: %s"
            logger.error(msg, src, args.common_root)
            sys.exit(1)

        dest = os.path.join(output_dir, src_rel)
        deps.extend(copy_with_deps.copy_with_deps(src, dest, search_path, [kconfig]))

    deps = sorted(set(deps))

    # Add a dependency on copy_with_deps.py, which won't have been set by Bob
    deps.append(os.path.join(os.path.dirname(sys.argv[0]), "copy_with_deps.py"))

    # Add a dependency on the test kernel Makefile. We do not attempt to add
    # dependencies on every part of the kernel's build system - this is just
    # enough to ensure that incremental builds of the Bob tests work OK.
    deps.append(os.path.join(abs_kdir, "Makefile"))

    copy_with_deps.write_depfile(args.depfile, args.output, deps)

    make_args = args.make_args
    make_args.extend(args.kbuild_options)
    make_args.append("ARCH=" + arch)

    # CROSS_COMPILE is still required with CC=clang
    if cross_compile:
        make_args.append("CROSS_COMPILE=" + cross_compile)
    if target_cc:
        make_args.append("CC=" + target_cc)
    if host_cc:
        make_args.append("HOSTCC=" + host_cc)
    if args.clang_triple:
        make_args.append("CLANG_TRIPLE=" + args.clang_triple)
    if args.ld:
        make_args.append("LD=" + args.ld)
    elif kernel_config_parser.option_enabled(abs_kdir, "CONFIG_LD_IS_LLD"):
        # Auto-set LD to `ld.lld` if LTO has been enabled
        make_args.append("LD=ld.lld")
    if args.extra_symbols:
        extra_symbols = [os.path.abspath(d) for d in args.extra_symbols]
        make_args.append("KBUILD_EXTRA_SYMBOLS=" + " ".join(extra_symbols))

    if args.jobs:
        make_args.append("-j" + str(args.jobs))
    else:
        # If the following env var is set, we are running in a build
        # farm where we should avoid increasing thread
        # count. Therefore leave make to run with a single core.
        # If not, build kernel modules with the number of CPUs we have.
        if os.getenv("MPDTI_BUILD_PARALLELISM") is None:
            make_args.append("-j" + str(multiprocessing.cpu_count()))

    module_ko = os.path.basename(args.output)
    abs_module_dir = os.path.abspath(args.module_dir)
    build_module(output_dir, module_ko, abs_kdir, abs_module_dir,
                 make_command, make_args, extra_cflags)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import logging
import time
l = logging.getLogger("angr_tests")

import nose
import pprint

try:
    import standard_logging
    import angr_debug
except ImportError:
    pass

import angr
import simuvex

# Load the tests
test_location = str(os.path.dirname(os.path.realpath(__file__)))
vfg_tests = {}

def setup_module():
    global scout_tests
    vfg_tests[0] = angr.Project(test_location + "/blob/x86_64/cfg_1",
                                use_sim_procedures=True,
                                default_analysis_mode='symbolic')

def test_vfg_0():
    print "CFG 0"
    start = time.time()
    vfg = vfg_tests[0].construct_vfg(start=0x40057c, context_sensitivity_level=2)
    end = time.time()
    duration = end - start
    print "Normal: Done in %f seconds." % duration
    bbl_dict = vfg.get_bbl_dict()
    graph = vfg.get_graph()
    print "Contains %d members in BBL dict." % len(bbl_dict)
    print graph.nodes()

    import ipdb; ipdb.set_trace()

if __name__ == "__main__":
    logging.getLogger("simuvex.plugins.abstract_memory").setLevel(logging.DEBUG)
    #logging.getLogger("simuvex.plugins.symbolic_memory").setLevel(logging.DEBUG)
    logging.getLogger("angr.vfg").setLevel(logging.DEBUG)
    # Temporarily disable the warnings of claripy backend
    logging.getLogger("claripy.backends.backend").setLevel(logging.ERROR)
    logging.getLogger("claripy.claripy").setLevel(logging.ERROR)
    setup_module()
    test_vfg_0()
<commit_msg>fix up test_vfg a bit<commit_after>#!/usr/bin/env python

import os
import logging
import time
l = logging.getLogger("angr_tests")

import nose
import pprint

try:
    import standard_logging
    import angr_debug
except ImportError:
    pass

import angr
import simuvex

# Load the tests
test_location = str(os.path.dirname(os.path.realpath(__file__)))
vfg_tests = {}

def setup_module():
    global scout_tests
    vfg_tests[0] = angr.Project(test_location + "/blob/x86_64/cfg_1",
                                use_sim_procedures=True,
                                default_analysis_mode='symbolic')

def test_vfg_0():
    print "CFG 0"
    start = time.time()
    vfg = vfg_tests[0].construct_vfg(start=0x40057c, context_sensitivity_level=2)
    end = time.time()
    duration = end - start
    print "Normal: Done in %f seconds." % duration
    bbl_dict = vfg.get_bbl_dict()
    graph = vfg.graph
    print "Contains %d members in BBL dict." % len(bbl_dict)
    print graph.nodes()

    import ipdb; ipdb.set_trace()

if __name__ == "__main__":
    logging.getLogger("simuvex.plugins.abstract_memory").setLevel(logging.DEBUG)
    #logging.getLogger("simuvex.plugins.symbolic_memory").setLevel(logging.DEBUG)
    logging.getLogger("angr.vfg").setLevel(logging.DEBUG)
    # Temporarily disable the warnings of claripy backend
    logging.getLogger("claripy.backends.backend").setLevel(logging.ERROR)
    logging.getLogger("claripy.claripy").setLevel(logging.ERROR)
    setup_module()
    test_vfg_0()
<|endoftext|>


<commit_before>#! /usr/bin/env python3
"""Launch the flask-forecaster application."""
import os

from flask_forecaster import app

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=int(os.getenv('PORT')))
<commit_msg>Allow running the app in debug mode from the command line<commit_after>#! /usr/bin/env python3
"""Launch the flask-forecaster application."""
import os
import sys

from flask_forecaster import app

if __name__ == '__main__':
    app.run(
        debug='debug' in sys.argv,
        host='0.0.0.0',
        port=int(os.getenv('PORT'))
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import os
import rdflib
import shutil
import subprocess
import sys
import xml.dom
import xml.dom.minidom

sys.path.append("./lv2specgen")
import lv2specgen

try:
    shutil.rmtree('build', 'ns')
except:
    pass

# Copy bundles (less build files) to build directory
shutil.copytree('ns', 'build/ns',
                ignore=shutil.ignore_patterns('.*', 'waf', 'wscript', '*.in'))

# Copy stylesheet to build directory
try:
    os.mkdir('build/aux')
except:
    pass

shutil.copy('lv2specgen/style.css', 'build/aux/style.css')

URIPREFIX  = 'http://lv2plug.in/ns/'
DOXPREFIX  = 'ns/doc/html/'
SPECGENDIR = os.path.abspath('lv2specgen')
STYLEPATH  = os.path.abspath('build/aux/style.css')
TAGFILE    = os.path.abspath('doclinks')
BUILDDIR   = os.path.abspath('build')

doap = rdflib.Namespace('http://usefulinc.com/ns/doap#')
lv2  = rdflib.Namespace('http://lv2plug.in/ns/lv2core#')
owl  = rdflib.Namespace('http://www.w3.org/2002/07/owl#')
rdf  = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')

devnull = open(os.devnull, 'w')

# Generate code (headers) documentation
print('** Generating header documentation')
print(' * Calling doxygen in ' + os.getcwd())
subprocess.call('doxygen', stdout=devnull)

def rescue_tags(in_path, out_path):
    "Rescue Doxygen tag file from XML hell."

    def getChildText(elt, tagname):
        "Return the content of the first child node with a certain tag name."
        elements = elt.getElementsByTagName(tagname)
        for e in elements:
            if e.parentNode == elt:
                return e.firstChild.nodeValue
        return ''

    tagdoc = xml.dom.minidom.parse(in_path)
    root = tagdoc.documentElement
    bettertags = open(out_path, 'w')
    for cn in root.childNodes:
        if cn.nodeType == xml.dom.Node.ELEMENT_NODE and cn.tagName == 'compound':
            if cn.getAttribute('kind') == 'page':
                continue
            name = getChildText(cn, 'name')
            filename = getChildText(cn, 'filename')
            # Sometimes the .html is there, sometimes it isn't...
            if not filename.endswith('.html'):
                filename += '.html'
            bettertags.write('%s %s%s\n' % (name, DOXPREFIX, filename))
            if cn.getAttribute('kind') == 'file':
                prefix = ''
            else:
                prefix = name + '::'
            members = cn.getElementsByTagName('member')
            for m in members:
                mname = prefix + getChildText(m, 'name')
                mafile = getChildText(m, 'anchorfile')
                manchor = getChildText(m, 'anchor')
                bettertags.write('%s %s%s#%s\n' % (mname, DOXPREFIX, \
                                                       mafile, manchor))
    bettertags.close()

rescue_tags('c_tags', TAGFILE)

def subst_file(template, output, dict):
    i = open(template, 'r')
    o = open(output, 'w')
    for line in i:
        for key in dict:
            line = line.replace(key, dict[key])
        o.write(line)
    i.close()
    o.close()

print('** Generating core documentation')

print("Entering directory `%s'" % os.path.abspath('build'))
oldcwd = os.getcwd()
os.chdir('build')

extensions = []

for root, dirs, files in os.walk('ns'):
    if '.svn' in dirs:
        dirs.remove('.svn')

    if root in ['ns', 'ns/ext', 'ns/extensions']:
        if 'doc' in dirs:
            dirs.remove('doc')
        continue

    abs_root = os.path.abspath(root)
    outdir = root
    bundle = root
    b = os.path.basename(root)

    print("** Generating %s documentation" % outdir)

    try:
        model = rdflib.ConjunctiveGraph()
        model.parse('%s/manifest.ttl' % bundle, format='n3')
        model.parse('%s/%s.ttl' % (bundle, b), format='n3')
    except:
        e = sys.exc_info()[1]
        print('error parsing %s: %s' % (bundle, str(e)))
        continue

    # Get extension URI
    ext_node = model.value(None, rdf.type, lv2.Specification)
    if not ext_node:
        continue

    ext = str(ext_node)

    # Get version
    minor = 0
    micro = 0
    try:
        minor = int(model.value(ext_node, lv2.minorVersion, None))
        micro = int(model.value(ext_node, lv2.microVersion, None))
    except Exception as e:
        print("warning: %s: failed to find version for %s" % (bundle, ext))

    # Get date
    date = None
    for r in model.triples([ext_node, doap.release, None]):
        revision = model.value(r[2], doap.revision, None)
        if revision == ("%d.%d" % (minor, micro)):
            date = model.value(r[2], doap.created, None)
            break

    # Verify that this date is the latest
    for r in model.triples([ext_node, doap.release, None]):
        revision = model.value(r[2], doap.revision, None)
        this_date = model.value(r[2], doap.created, None)
        if this_date > date:
            print("warning: revision %d.%d (%s) is not the latest release" % (
                minor, micro, date))
            break

    # Get short description
    shortdesc = model.value(ext_node, doap.shortdesc, None)

    if (os.access(outdir + '/%s.ttl' % b, os.R_OK)):
        print(' * Running lv2specgen for %s in %s' % (b, os.getcwd()))
        specdoc = lv2specgen.specgen(
            root + '/%s.ttl' % b,
            SPECGENDIR,
            os.path.relpath(os.path.join('ns', 'doc'), abs_root),
            os.path.relpath(STYLEPATH, abs_root),
            os.path.relpath(BUILDDIR, abs_root),
            TAGFILE,
            instances=True)

        lv2specgen.save(root + '/%s.html' % b, specdoc)

        # Name
        row = '
%s	' % (
            os.path.relpath(root, 'ns'), b)

        # Description
        if shortdesc:
            row += '' + str(shortdesc) + '	'
        else:
            row += '	'

        # Version
        version_str = '%s.%s' % (minor, micro)
        if minor == 0 or (micro % 2 != 0):
            row += '' + version_str + ' dev	'
        else:
            row += '' + version_str + '	'

        # Date
        row += '%s	' % (str(date) if date else '')

        # Status
        deprecated = model.value(ext_node, owl.deprecated, None)
        if minor == 0:
            row += 'Experimental	'
        elif deprecated and str(deprecated[2]) != "false":
                row += 'Deprecated	'
        elif micro % 2 == 0:
            row += 'Stable	'

        row += '
'
        extensions.append(row)

        subst_file('../doc/htaccess.in', outdir + '.htaccess',
                   { '@NAME@': b,
                     '@BASE@': '/ns/%s/%s' % (dir, b) })

index_rows = ''
extensions.sort()
for i in extensions:
    index_rows += i + '\n'

subst_file('../ns/index.html.in', 'ns/index.html',
           { '@ROWS@': index_rows,
             '@TIME@': datetime.datetime.utcnow().strftime('%F %H:%M UTC') })

print("Leaving directory `%s'" % os.path.abspath('build'))
os.chdir(oldcwd)

devnull.close()
<commit_msg>Bail early if there's no foo.ttl file.<commit_after>#!/usr/bin/env python

import datetime
import os
import rdflib
import shutil
import subprocess
import sys
import xml.dom
import xml.dom.minidom

sys.path.append("./lv2specgen")
import lv2specgen

try:
    shutil.rmtree('build', 'ns')
except:
    pass

# Copy bundles (less build files) to build directory
shutil.copytree('ns', 'build/ns',
                ignore=shutil.ignore_patterns('.*', 'waf', 'wscript', '*.in'))

# Copy stylesheet to build directory
try:
    os.mkdir('build/aux')
except:
    pass

shutil.copy('lv2specgen/style.css', 'build/aux/style.css')

URIPREFIX  = 'http://lv2plug.in/ns/'
DOXPREFIX  = 'ns/doc/html/'
SPECGENDIR = os.path.abspath('lv2specgen')
STYLEPATH  = os.path.abspath('build/aux/style.css')
TAGFILE    = os.path.abspath('doclinks')
BUILDDIR   = os.path.abspath('build')

doap = rdflib.Namespace('http://usefulinc.com/ns/doap#')
lv2  = rdflib.Namespace('http://lv2plug.in/ns/lv2core#')
owl  = rdflib.Namespace('http://www.w3.org/2002/07/owl#')
rdf  = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')

devnull = open(os.devnull, 'w')

# Generate code (headers) documentation
print('** Generating header documentation')
print(' * Calling doxygen in ' + os.getcwd())
subprocess.call('doxygen', stdout=devnull)

def rescue_tags(in_path, out_path):
    "Rescue Doxygen tag file from XML hell."

    def getChildText(elt, tagname):
        "Return the content of the first child node with a certain tag name."
        elements = elt.getElementsByTagName(tagname)
        for e in elements:
            if e.parentNode == elt:
                return e.firstChild.nodeValue
        return ''

    tagdoc = xml.dom.minidom.parse(in_path)
    root = tagdoc.documentElement
    bettertags = open(out_path, 'w')
    for cn in root.childNodes:
        if cn.nodeType == xml.dom.Node.ELEMENT_NODE and cn.tagName == 'compound':
            if cn.getAttribute('kind') == 'page':
                continue
            name = getChildText(cn, 'name')
            filename = getChildText(cn, 'filename')
            # Sometimes the .html is there, sometimes it isn't...
            if not filename.endswith('.html'):
                filename += '.html'
            bettertags.write('%s %s%s\n' % (name, DOXPREFIX, filename))
            if cn.getAttribute('kind') == 'file':
                prefix = ''
            else:
                prefix = name + '::'
            members = cn.getElementsByTagName('member')
            for m in members:
                mname = prefix + getChildText(m, 'name')
                mafile = getChildText(m, 'anchorfile')
                manchor = getChildText(m, 'anchor')
                bettertags.write('%s %s%s#%s\n' % (mname, DOXPREFIX, \
                                                       mafile, manchor))
    bettertags.close()

rescue_tags('c_tags', TAGFILE)

def subst_file(template, output, dict):
    i = open(template, 'r')
    o = open(output, 'w')
    for line in i:
        for key in dict:
            line = line.replace(key, dict[key])
        o.write(line)
    i.close()
    o.close()

print('** Generating core documentation')

print("Entering directory `%s'" % os.path.abspath('build'))
oldcwd = os.getcwd()
os.chdir('build')

extensions = []

for root, dirs, files in os.walk('ns'):
    if '.svn' in dirs:
        dirs.remove('.svn')

    if root in ['ns', 'ns/ext', 'ns/extensions']:
        if 'doc' in dirs:
            dirs.remove('doc')
        continue

    abs_root = os.path.abspath(root)
    outdir = root
    bundle = root
    b = os.path.basename(root)

    if not os.access(outdir + '/%s.ttl' % b, os.R_OK):
        print('warning: extension %s has no %s.ttl file' % (root, root))
        continue

    print(" * Generating %s documentation" % outdir)

    try:
        model = rdflib.ConjunctiveGraph()
        model.parse('%s/manifest.ttl' % bundle, format='n3')
        model.parse('%s/%s.ttl' % (bundle, b), format='n3')
    except:
        e = sys.exc_info()[1]
        print('error parsing %s: %s' % (bundle, str(e)))
        continue

    # Get extension URI
    ext_node = model.value(None, rdf.type, lv2.Specification)
    if not ext_node:
        continue

    ext = str(ext_node)

    # Get version
    minor = 0
    micro = 0
    try:
        minor = int(model.value(ext_node, lv2.minorVersion, None))
        micro = int(model.value(ext_node, lv2.microVersion, None))
    except Exception as e:
        print("warning: %s: failed to find version for %s" % (bundle, ext))

    # Get date
    date = None
    for r in model.triples([ext_node, doap.release, None]):
        revision = model.value(r[2], doap.revision, None)
        if revision == ("%d.%d" % (minor, micro)):
            date = model.value(r[2], doap.created, None)
            break

    # Verify that this date is the latest
    for r in model.triples([ext_node, doap.release, None]):
        revision = model.value(r[2], doap.revision, None)
        this_date = model.value(r[2], doap.created, None)
        if this_date > date:
            print("warning: revision %d.%d (%s) is not the latest release" % (
                minor, micro, date))
            break

    # Get short description
    shortdesc = model.value(ext_node, doap.shortdesc, None)

    specdoc = lv2specgen.specgen(
        root + '/%s.ttl' % b,
        SPECGENDIR,
        os.path.relpath(os.path.join('ns', 'doc'), abs_root),
        os.path.relpath(STYLEPATH, abs_root),
        os.path.relpath(BUILDDIR, abs_root),
        TAGFILE,
        instances=True)

    lv2specgen.save(root + '/%s.html' % b, specdoc)

    # Name
    row = '
%s	' % (
        os.path.relpath(root, 'ns'), b)

    # Description
    if shortdesc:
        row += '' + str(shortdesc) + '	'
    else:
        row += '	'

    # Version
    version_str = '%s.%s' % (minor, micro)
    if minor == 0 or (micro % 2 != 0):
        row += '' + version_str + ' dev	'
    else:
        row += '' + version_str + '	'

    # Date
    row += '%s	' % (str(date) if date else '')

    # Status
    deprecated = model.value(ext_node, owl.deprecated, None)
    if minor == 0:
        row += 'Experimental	'
    elif deprecated and str(deprecated[2]) != "false":
            row += 'Deprecated	'
    elif micro % 2 == 0:
        row += 'Stable	'

    row += '
'
    extensions.append(row)

    subst_file('../doc/htaccess.in', outdir + '.htaccess',
               { '@NAME@': b,
                 '@BASE@': '/ns/%s/%s' % (dir, b) })

index_rows = ''
extensions.sort()
for i in extensions:
    index_rows += i + '\n'

subst_file('../ns/index.html.in', 'ns/index.html',
           { '@ROWS@': index_rows,
             '@TIME@': datetime.datetime.utcnow().strftime('%F %H:%M UTC') })

print("Leaving directory `%s'" % os.path.abspath('build'))
os.chdir(oldcwd)

devnull.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import six
import subprocess

def parallel_link_jobs():
    try:
        import multiprocessing
        cpus = multiprocessing.cpu_count()
        return max(cpus / 4, 2)
    except:
        return 2

parser = argparse.ArgumentParser(
    description=
    "CMake configuration options are relatively verbose and remembering the "
    "ones you don't use that often can be a real pain. This scripts attempts "
    "to reduce some of the most commonly used options to a few easy to "
    "remember mnemonic arguments.")

parser.add_argument("source", help="LLVM source path", type=str)

parser.add_argument(
    '-s', '--shared', action='store_true', help="Build shared libraries")

parser.add_argument(
    '-c',
    '--clang-light',
    action='store_true',
    help=
    "Build a light version of clang, which means no ARC, Static Anaylzer or plugins"
)

parser.add_argument(
    '-r',
    '--ra',
    action='store_true',
    help="Release build with debug info and asserts")
parser.add_argument('-d', '--debug', action='store_true', help="Debug build")

parser.add_argument(
    '-m', '--modules', action='store_true', help="Enable modules")

parser.add_argument(
    '-x', '--x86', action='store_true', help="Only build x86 target")

parser.add_argument('--sanitizers', nargs='*', help="Sanitizers to enable")

parser.add_argument(
    '--system-debugserver',
    action='store_true',
    help="Use system debug server")

parser.add_argument(
    '--docs',
    action='store_true',
    help="Build the documentation")

parser.add_argument(
    '--expensive',
    action='store_true',
    help="Enable expensive checks")

parser.add_argument(
    '--launcher',
    help="Specify launcher",
    type=str)

parser.add_argument(
    '--projects',
    nargs='*',
    help="Project to enable when using the monorepo")

parser.add_argument(
    '--runtimes',
    nargs='*',
    help="Runtimes to enable when using the monorepo")

args = parser.parse_args()

cmake_cmd = [
    "cmake {}".format(args.source), "-G Ninja",
    "-DCMAKE_INSTALL_PREFIX='../install'"
]

cmake_cmd.append("-DLLVM_PARALLEL_LINK_JOBS:INT={}".format(
    parallel_link_jobs()))

if args.shared:
    cmake_cmd.append("-DBUILD_SHARED_LIBS:BOOL=ON")

if args.clang_light:
    cmake_cmd.append("-DCLANG_ENABLE_ARCMT:BOOL=OFF")
    cmake_cmd.append("-DCLANG_ENABLE_STATIC_ANALYZER:BOOL=OFF")
    cmake_cmd.append("-DCLANG_PLUGIN_SUPPORT:BOOL=OFF")

if args.ra:
    cmake_cmd.append("-DCMAKE_BUILD_TYPE='RelWithDebInfo'")
    cmake_cmd.append("-DLLVM_ENABLE_ASSERTIONS:BOOL=ON")

if args.debug:
    cmake_cmd.append("-DCMAKE_BUILD_TYPE='Debug'")
    cmake_cmd.append("-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON")

if args.modules:
    cmake_cmd.append("-DLLVM_ENABLE_MODULES:BOOL=ON")

if args.x86:
    cmake_cmd.append("-DLLVM_TARGETS_TO_BUILD='X86'")

if args.sanitizers:
    sanitizers = ';'.join(args.sanitizers)
    cmake_cmd.append("-DLLVM_USE_SANITIZER='{}'".format(sanitizers))

if args.system_debugserver:
    cmake_cmd.append("-DLLDB_USE_SYSTEM_DEBUGSERVER:BOOL=ON")

if args.docs:
    cmake_cmd.append("-DLLVM_ENABLE_SPHINX:BOOL=ON")

if args.expensive:
    cmake_cmd.append("-DLLVM_ENABLE_EXPENSIVE_CHECKS:BOOL=ON")
    cmake_cmd.append("-DLLVM_ENABLE_REVERSE_ITERATION:BOOL=ON")

if args.launcher:
    cmake_cmd.append("-DCMAKE_C_COMPILER_LAUNCHER='{}'".format(args.launcher))
    cmake_cmd.append("-DCMAKE_CXX_COMPILER_LAUNCHER='{}'".format(args.launcher))

if args.projects:
    projects = ';'.join(args.projects)
    cmake_cmd.append("-DLLVM_ENABLE_PROJECTS='{}'".format(projects))

if args.runtimes:
    runtimes = ';'.join(args.runtimes)
    cmake_cmd.append("-DLLVM_ENABLE_RUNTIMES='{}'".format(runtimes))

try:
    print(' \\\n    '.join(cmake_cmd))
    six.moves.input("Press Enter to run CMake or ^C to abort...")
except KeyboardInterrupt:
    exit(1)

subprocess.call(' '.join(cmake_cmd), shell=True)
subprocess.call('ninja', shell=True)
<commit_msg>[llvm-cmake] Add flags to pass extra C(XX) flags<commit_after>#!/usr/bin/env python

import argparse
import six
import subprocess

def parallel_link_jobs():
    try:
        import multiprocessing
        cpus = multiprocessing.cpu_count()
        return max(cpus / 4, 2)
    except:
        return 2

parser = argparse.ArgumentParser(
    description=
    "CMake configuration options are relatively verbose and remembering the "
    "ones you don't use that often can be a real pain. This scripts attempts "
    "to reduce some of the most commonly used options to a few easy to "
    "remember mnemonic arguments.")

parser.add_argument("source", help="LLVM source path", type=str)

parser.add_argument(
    '-s', '--shared', action='store_true', help="Build shared libraries")

parser.add_argument(
    '-c',
    '--clang-light',
    action='store_true',
    help=
    "Build a light version of clang, which means no ARC, Static Anaylzer or plugins"
)

parser.add_argument(
    '-r',
    '--ra',
    action='store_true',
    help="Release build with debug info and asserts")
parser.add_argument('-d', '--debug', action='store_true', help="Debug build")

parser.add_argument(
    '-m', '--modules', action='store_true', help="Enable modules")

parser.add_argument(
    '-x', '--x86', action='store_true', help="Only build x86 target")

parser.add_argument('--sanitizers', nargs='*', help="Sanitizers to enable")

parser.add_argument(
    '--system-debugserver',
    action='store_true',
    help="Use system debug server")

parser.add_argument(
    '--docs',
    action='store_true',
    help="Build the documentation")

parser.add_argument(
    '--expensive',
    action='store_true',
    help="Enable expensive checks")

parser.add_argument(
    '--launcher',
    help="Specify launcher",
    type=str)

parser.add_argument(
    '--extra',
    help="Specify extra C/CXX flags",
    type=str)

parser.add_argument(
    '--projects',
    nargs='*',
    help="Project to enable when using the monorepo")

parser.add_argument(
    '--runtimes',
    nargs='*',
    help="Runtimes to enable when using the monorepo")

args = parser.parse_args()

cmake_cmd = [
    "cmake {}".format(args.source), "-G Ninja",
    "-DCMAKE_INSTALL_PREFIX='../install'"
]

cmake_cmd.append("-DLLVM_PARALLEL_LINK_JOBS:INT={}".format(
    parallel_link_jobs()))

if args.shared:
    cmake_cmd.append("-DBUILD_SHARED_LIBS:BOOL=ON")

if args.clang_light:
    cmake_cmd.append("-DCLANG_ENABLE_ARCMT:BOOL=OFF")
    cmake_cmd.append("-DCLANG_ENABLE_STATIC_ANALYZER:BOOL=OFF")
    cmake_cmd.append("-DCLANG_PLUGIN_SUPPORT:BOOL=OFF")

if args.ra:
    cmake_cmd.append("-DCMAKE_BUILD_TYPE='RelWithDebInfo'")
    cmake_cmd.append("-DLLVM_ENABLE_ASSERTIONS:BOOL=ON")

if args.debug:
    cmake_cmd.append("-DCMAKE_BUILD_TYPE='Debug'")
    cmake_cmd.append("-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON")

if args.modules:
    cmake_cmd.append("-DLLVM_ENABLE_MODULES:BOOL=ON")

if args.x86:
    cmake_cmd.append("-DLLVM_TARGETS_TO_BUILD='X86'")

if args.sanitizers:
    sanitizers = ';'.join(args.sanitizers)
    cmake_cmd.append("-DLLVM_USE_SANITIZER='{}'".format(sanitizers))

if args.system_debugserver:
    cmake_cmd.append("-DLLDB_USE_SYSTEM_DEBUGSERVER:BOOL=ON")

if args.docs:
    cmake_cmd.append("-DLLVM_ENABLE_SPHINX:BOOL=ON")

if args.expensive:
    cmake_cmd.append("-DLLVM_ENABLE_EXPENSIVE_CHECKS:BOOL=ON")
    cmake_cmd.append("-DLLVM_ENABLE_REVERSE_ITERATION:BOOL=ON")

if args.launcher:
    cmake_cmd.append("-DCMAKE_C_COMPILER_LAUNCHER='{}'".format(args.launcher))
    cmake_cmd.append("-DCMAKE_CXX_COMPILER_LAUNCHER='{}'".format(args.launcher))

if args.extra:
    cmake_cmd.append("-DCMAKE_C_FLAGS='{}'".format(args.extra))
    cmake_cmd.append("-DCMAKE_CXX_FLAGS='{}'".format(args.extra))

if args.projects:
    projects = ';'.join(args.projects)
    cmake_cmd.append("-DLLVM_ENABLE_PROJECTS='{}'".format(projects))

if args.runtimes:
    runtimes = ';'.join(args.runtimes)
    cmake_cmd.append("-DLLVM_ENABLE_RUNTIMES='{}'".format(runtimes))

try:
    print(' \\\n    '.join(cmake_cmd))
    six.moves.input("Press Enter to run CMake or ^C to abort...")
except KeyboardInterrupt:
    exit(1)

subprocess.call(' '.join(cmake_cmd), shell=True)
subprocess.call('ninja', shell=True)
<|endoftext|>


<commit_before>import numpy as np
import networkx as nx
import random
import matplotlib.pyplot as plt
import scipy.signal
import pygraphviz
import pydot

#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.

# If it is not possible or desirable to put the notice in a particular
# file, then You may include the notice in a location (such as a LICENSE
# file in a relevant directory) where a recipient would be likely to look
# for such a notice.

# You may add additional accurate notices of copyright ownership.

# Written by Stefano Palmieri in August-September 2016

# This class stores the genome as a weighted directed graph representing
# a Compositional Adjacency Matrix Producing Network. The weighted directed
#  graph is produced from Nodes and Links. Nodes can output a constant
# matrix (tensor) or produce the matrix product, transpose, convolution, or
# Kronecker product of the input matrices. Links are floating point values
# that are use for scalar multiplication of matrices outputted from Nodes.
class Genome(object):

    # Probability of mutating a new Node
    P_NEW_NODE = 0.15

    # Probability of mutating a new Link
    P_NEW_LINK = 0.15

    # Probability of mutating existing node
    P_NODE = 0.3

    # Probability of mutating existing link
    P_LINK = 0.3

    # Weights of a new added link
    N_LINK_WEIGHT = 1.0

    # Dictionary for types of Nodes
    types = {
        0: 'constant',
        1: 'kronecker',
        2: 'elem_mult'
        }

    # Sizes for matrices in the constant node type
    CONSTANT_ROW_SIZE = 2
    CONSTANT_COL_SIZE = 2

    # Mean and standard deviation for weight mutations
    MUTATE_MEAN = 0.1
    MUTATE_STD = 0.1

    def __init__(self):
        self.network = nx.DiGraph()
        # node with 0 id is the output
        self.network.add_node(0, type='output',
                              constant=np.matrix([[0.0, 0.0], [0.0, 0.0]], np.float16))
        #self.network.add_node(1, type='kronecker')
        #self.network.add_edge(1, 0, weight=1.0, left=True)
        self.network.add_node(1, type='constant',
                              constant=np.matrix([[1.0, 0.0], [0.0, 1.0]], np.float16))
        self.network.add_edge(1, 0, weight=1.0, left=True)

    # Mutations can add a Node, add a Link, change the weight of a Link,
    # change the function of a Node.
    def mutate(self):
        # If mutation occurs, mutate a new node
        rand = random.random()
        if rand <= self.P_NEW_NODE:
            # print "new node"
            self.mutate_new_node()
        # If mutation occurs, mutate a new link
        rand = random.random()
        if rand <= self.P_NEW_LINK:
            # print "new link"
            self.mutate_new_link()
        # If mutation occurs, mutate an existing node
        rand = random.random()
        if rand <= self.P_NODE:
            # print "node"
            self.mutate_node()
        # If mutation occurs, mutate an existing link
        rand = random.random()
        if rand <= self.P_LINK:
            # print "link"
            self.mutate_link()

    # Randomly mutates one of the existing nodes in the CAMPN.
    def mutate_node(self):

        # keep picking a node until a constant node is found
        while True:
            # pick a random node that's not the output node
            node = random.randrange(1, self.network.number_of_nodes())
            if self.network.node[node]['type'] == 'constant':
                break

        # pick a type of function to mutate to
        # func = self.types[random.randrange(0, len(self.types))]

        # self.network.node[node]['type'] = func

        # mutate one of it's constant's elements
        constant = self.network.node[node]['constant']
        rand_row = random.randrange(0, self.CONSTANT_ROW_SIZE)
        rand_col = random.randrange(0, self.CONSTANT_COL_SIZE)

        # Mutate by Gaussian random variable
        change = np.random.normal(self.MUTATE_MEAN, self.MUTATE_STD)

        # apply the change
        constant[rand_row, rand_col] += change
        self.network.node[node]['constant'] = constant

    # Randomly mutate an existing link by modifying the weight
    def mutate_link(self):
        # Add a Gaussian random variable to existing weight
        weight_change = np.random.normal(self.MUTATE_MEAN, self.MUTATE_STD)

        # Select a random edge
        rand = random.randrange(0, len(self.network.edges()))
        start, end = self.network.edges()[rand]
        self.network[start][end]['weight'] += weight_change

    # Create a new random Node and add it to the Genome.
    def mutate_new_node(self):
        # Randomly select the function for this node
        # func = self.types[random.randrange(0, len(self.types))]

        # Randomly select a node from the existing network
        end = random.randrange(0, self.network.number_of_nodes())

        if not self.network.node[end]['type'] == 'constant':
            # if the end node is kronecker/output, add a constant as a predecessor
            matrix = np.random.rand(self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE).astype(np.float16)
            self.network.add_node(self.network.number_of_nodes(),
                                  type='constant', constant=matrix)
            self.network.add_edge(self.network.number_of_nodes()-1,
                                  end, weight=self.N_LINK_WEIGHT,
                                  left=bool(random.getrandbits(1)))

        elif self.network.node[end]['type'] == 'constant':
            # if the node is a constant, place a kronecker in between
            matrix = self.network.node[end]['constant']

            if random.random() < 0.5:
                self.network.node[end]['type'] = 'kronecker'
            else:
                self.network.node[end]['type'] = 'elem_mult'
            self.network.node[end]['constant'] = None
            self.network.add_node(self.network.number_of_nodes(),
                                  type='constant', constant=matrix)
            self.network.add_edge(self.network.number_of_nodes()-1,
                                  end, weight=self.N_LINK_WEIGHT,
                                  left=bool(random.getrandbits(1)))

    # Create a new Link and add it to the Genome.
    def mutate_new_link(self):
        # select a random node to start a link from, don't include output node
        start = random.randrange(1, self.network.number_of_nodes())

        # Keep picking a terminal node until one that is not the
        # start link and not a constant type is found
        while True:
            end = random.randrange(0, self.network.number_of_nodes())
            if not self.network.node[end]['type'] == 'constant':
                break

        # First check this link doesn't already exist so we don't add it again.
        # Otherwise we can consider this mutations as failed if it does exist
        # (no mutation occurred).
        if not (start, end) in self.network.edges():
            # If the end node has no incoming links, make link primary.
            self.network.add_edge(start, end, weight=self.N_LINK_WEIGHT,
                                  left=bool(random.getrandbits(1)))

            # attempt to add the link. Do this checking that
            # the number of simple cycles does not become one or greater.
            # Otherwise we have to remove the link to keep the CTPPN acyclic
            # and can consider this a mutation that failed (no mutation occurred).
            cycles = len(list(nx.simple_cycles(self.network)))

            if cycles >= 1 or start == end:
                self.network.remove_edge(start, end)

    # recursively compute output for a node
    def get_phenotype(self, node=None):

        if node is None:
            return self.get_phenotype(0)

        else:
            # optimization for constant node so don't have to calculate sub
            # phenotype
            if self.network.node[node]['type'] == 'constant':
                return self.network.node[node]['constant']

            predecessors = list(self.network.predecessors(node))

            left_nodes = self.left_nodes(node, predecessors)

            left_matrix = np.zeros((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE))
            logit = np.zeros((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE))

            # remove the left nodes from the predecessor list
            for left in left_nodes:
                predecessors.remove(left)

            # left will be weighted sum of left node outputs
            for left in left_nodes:
                weight = self.network[left][node]['weight']
                left_output = weight * self.get_phenotype(left)

                # rescale smaller matrix if need be.
                while np.ma.size(left_output, 0) != np.ma.size(left_matrix, 0):
                    if np.ma.size(left_matrix, 0) < np.ma.size(left_output, 0):
                        left_matrix = np.kron(left_matrix,
                                        np.ones((self.CONSTANT_ROW_SIZE,
                                                 self.CONSTANT_COL_SIZE)))
                    elif np.ma.size(left_matrix, 0) > np.ma.size(left_output, 0):
                        left_output = np.kron(left_output,
                                              np.ones((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE)))

                left_matrix = np.add(left_matrix, left_output)

            if len(predecessors) == 0 and node > 0:
                # weight = self.network[primary][node]['weight']
                return self.node_output(node, left_matrix, left_matrix)

            # logit will be weighted sum of predecessor outputs
            for predecessor in predecessors:
                weight = self.network[predecessor][node]['weight']
                predecessor_output = weight * self.get_phenotype(predecessor)

                # rescale smaller matrix if need be.
                while np.ma.size(predecessor_output, 0) != np.ma.size(logit, 0):
                    if np.ma.size(logit, 0) < np.ma.size(predecessor_output, 0):
                        logit = np.kron(logit,
                                        np.ones((self.CONSTANT_ROW_SIZE,
                                                 self.CONSTANT_COL_SIZE)))
                    elif np.ma.size(logit, 0) > np.ma.size(predecessor_output, 0):
                        predecessor_output = np.kron(predecessor_output,
                                                     np.ones((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE)))

                logit = np.add(logit, predecessor_output)

            if len(left_nodes) == 0 and node > 0:
                # weight = self.network[primary][node]['weight']
                return self.node_output(node, logit, logit)

            return self.node_output(node, left_matrix, logit)

    def node_output(self, node, left_matrix, logit):

        if self.network.node[node]['type'] == 'kronecker':
            return np.kron(left_matrix, logit)

        # rescale smaller matrix if need be.
        while np.ma.size(left_matrix, 0) != np.ma.size(logit, 0):
            if np.ma.size(logit, 0) < np.ma.size(left_matrix, 0):
                logit = np.kron(logit,
                                np.ones((self.CONSTANT_ROW_SIZE,
                                         self.CONSTANT_COL_SIZE)))
            elif np.ma.size(logit, 0) > np.ma.size(left_matrix, 0):
                left_matrix = np.kron(left_matrix,
                                            np.ones((self.CONSTANT_ROW_SIZE,
                                                     self.CONSTANT_COL_SIZE)))

        if node == 0:
            # output node acts just like a sum
            return np.add(left_matrix, logit)
        elif self.network.node[node]['type'] == 'elem_mult':
            return np.multiply(left_matrix, logit)

    def left_nodes(self, node, predecessors):

        left_nodes = []

        if self.network.node[node]['type'] == 'constant':
            return left_nodes

        for predecessor in predecessors:
            if self.network[predecessor][node]['left']:
                left_nodes.append(predecessor)

        # if there is no primary return None
        return left_nodes

# Testing Genotype stuff

genome = Genome()

for i in range(0, 1):
    genome.mutate()

node_size = 1000
node_color = 'white'
node_text_size = 15
edge_alpha = 0.3
edge_thickness = 1
edge_text_pos = 0.8
text_font = 'sans-serif'

labels = {}
for i in range(nx.number_of_nodes(genome.network)):
    if genome.network.node[i]['type'] == 'constant':
        word = "\n" + str(genome.network.node[i]['constant'])
    else:
        word = " "
    labels[i] = str(genome.network.node[i]['type']) + word

colorList = []

for i in genome.network.edges():
    a, b = i
    colorVal = genome.network.edge[a][b]['weight']
    colorList.append(colorVal)

pos = nx.drawing.nx_agraph.graphviz_layout(genome.network, prog='dot')

nx.draw_networkx_nodes(genome.network, pos, node_size=node_size, node_color=node_color)
nx.draw_networkx_edges(genome.network, pos, width=colorList, edge_color=colorList, edge_cmap=plt.cm.RdYlGn, arrows=True)
nx.draw_networkx_labels(genome.network, pos, labels=labels, font_size=node_text_size,
                        font_family=text_font)
nx.draw_networkx_edge_labels(genome.network, pos)
plt.show()

print "getting phenotype"
print genome.get_phenotype()
# print np.size(genome.get_phenotype(), 0)

<commit_msg>Changed to python3, changed how nodes are created<commit_after>import numpy as np
import networkx as nx
import random
from enum import Enum
import matplotlib.pyplot as plt
import tkinter
# import scipy.signal
# import pygraphviz
# import pydot

#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.

# If it is not possible or desirable to put the notice in a particular
# file, then You may include the notice in a location (such as a LICENSE
# file in a relevant directory) where a recipient would be likely to look
# for such a notice.

# You may add additional accurate notices of copyright ownership.

# Written by Stefano Palmieri in December 2016

# This class stores the genome as a weighted directed graph representing
# a Compositional Adjacency Matrix Producing Network. The weighted directed
#  graph is produced from Nodes and Links. Nodes can output a constant
# matrix (tensor) or produce the matrix product, transpose, convolution, or
# Kronecker product of the input matrices. Links are floating point values
# that are use for scalar multiplication of matrices outputted from Nodes.

# Enumerate node types
# If you want to add more node types, add after Hadamard
class NodeType(Enum):
    output = 0
    input = 1
    # actual matrix operations get placed under this line
    #############################################
    kronecker = 2
    hadamard = 3

class Genome(object):

    # Probability of mutating a new node
    P_NEW_NODE = 0.1

    # Probability of mutating a new input node
    P_NEW_INPUT_NODE = 0.1

    # Probability of mutating a new link
    P_NEW_LINK = 0.3

    # Probability of mutating an existing input node
    P_INPUT_NODE = 0.3

    # Probability of mutating existing link
    P_LINK = 0.3

    # Weights of a new added link
    N_LINK_WEIGHT = 1.0

    # Sizes for matrices in the input node type
    CONSTANT_ROW_SIZE = 4
    CONSTANT_COL_SIZE = 4

    # Mean and standard deviation for weight mutations
    MUTATE_MEAN = 0.1
    MUTATE_STD = 0.1

    def __init__(self):
        self.network = nx.MultiDiGraph()

        # node with 0 id is the output
        self.network.add_node(0, type=NodeType.output)
        self.network.add_node(1, type=NodeType.input,
                              constant=np.zeros((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE), dtype=np.float16))
        self.network.add_edge(1, 0, key=True, weight=1.0)

    # Mutations can add a Node, add a Link, change the weight of a Link,
    # change the function of a Node.
    def mutate(self):

        # If mutation occurs, mutate a new node
        if random.random() <= self.P_NEW_NODE:
            print("P_NEW_NODE")
            self.mutate_new_node()

        # If mutation occurs, mutate a new input node
        if random.random() <= self.P_NEW_INPUT_NODE:
            print("P_NEW_INPUT_NODE")
            self.mutate_new_input_node()

        # If mutation occurs, mutate an existing input node
        if random.random() <= self.P_INPUT_NODE:
            print("P_INPUT_NODE")
            self.mutate_node()

        # If mutation occurs, mutate a new link
        if random.random() <= self.P_NEW_LINK:
            self.mutate_new_link()

        # If mutation occurs, mutate an existing link
        if random.random() <= self.P_LINK:
            print("P_LINK")
            self.mutate_link()

    # Create a new random Node by selecting a link and splitting it
    def mutate_new_node(self):

        # Randomly select a link from the existing network
        start, end, key = random.choice(self.network.edges(keys=True))

        # Randomly select the function for this node but don't include Input or Output types as functions
        function = NodeType(random.randrange(NodeType.input.value + 1, len(NodeType)))

        # Create a new node with the function type
        node_num = self.network.number_of_nodes()
        self.network.add_node(node_num, type=function)

        # Add links for the new node
        self.network.add_edge(start, node_num, key=random.random() < 0.5, weight=self.N_LINK_WEIGHT)
        self.network.add_edge(node_num, end, key=key, weight=self.N_LINK_WEIGHT)

        # Remove old edge (in future versions this edge should be disabled rather than removed)
        self.network.remove_edge(start, end, key=key)

    # Create a new random Node by selecting a non-input node and creating a child
    def mutate_new_input_node(self):

        # Get list of non-input nodes
        non_input_nodes = [node for node, data in self.network.nodes(data=True) if data['type'] != NodeType.input]

        # randomly pick a node from the list
        node = random.choice(non_input_nodes)

        # Create a new input node
        node_num = self.network.number_of_nodes()
        self.network.add_node(node_num, type=NodeType.input,
                              constant=np.zeros((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE), dtype=np.float16))

        # Create a link from the new input node to existing non-input node
        self.network.add_edge(node_num, node, key=random.random() < 0.5, weight=self.N_LINK_WEIGHT)

    # Randomly mutates one of the existing input nodes in the CAMPN.
    def mutate_node(self):

        # get list of input nodes
        input_nodes = [node for node, data in self.network.nodes(data=True) if data['type'] == NodeType.input]

        # randomly pick a node from the list
        node = random.choice(input_nodes)

        # mutate one of it's constant's elements
        rand_row = random.randrange(0, self.CONSTANT_ROW_SIZE)
        rand_col = random.randrange(0, self.CONSTANT_COL_SIZE)

        # Mutate by Gaussian random variable
        change = np.random.normal(self.MUTATE_MEAN, self.MUTATE_STD)

        # apply the change
        self.network.node[node]['constant'][rand_row, rand_col] += change

    # Create a new Link and add it to the Genome.
    def mutate_new_link(self):
        # select a random node to start a link from and, don't include output node
        list = [node for node, data in self.network.nodes(data=True) if data['type'] != NodeType.output]
        start = random.choice(list)

        # select a random node to end link at, don't include input nodes
        list = [node for node, data in self.network.nodes(data=True) if data['type'] != NodeType.input and node != start]
        end = random.choice(list)

        # Create the link
        key = random.random() < 0.5
        self.network.add_edge(start, end, key=key, weight=self.N_LINK_WEIGHT)

        # Check for no cycles. If there are, remove this link
        try:
            nx.find_cycle(self.network, 1)
            self.network.remove_edge(start, end, key=key)
        except nx.exception.NetworkXNoCycle:
            return

    # Randomly mutate an existing link by modifying the weight
    def mutate_link(self):

        # Select a random link
        start, end, key = random.choice(self.network.edges(keys=True))

        # Add a Gaussian random variable to existing weight
        weight_change = np.random.normal(self.MUTATE_MEAN, self.MUTATE_STD)

        # Modify the weight on the link
        self.network[start][end][key]['weight'] += weight_change

    # recursively compute output for a node
    def get_phenotype(self, node=None):

        if node is None:
            return self.get_phenotype(0)

        if self.network.node[node]['type'] == NodeType.input:
            return self.network.node[node]['constant']

        left_logit = self.calculate_logit(node, True)
        right_logit = self.calculate_logit(node, False)

        return self.node_output(node, left_logit, right_logit)

    def calculate_logit(self, node, key):
        predecessors = list(self.network.predecessors(node))
        nodes = [predecessor for predecessor in predecessors if key in self.network[predecessor][node]]

        if nodes:
            logit = np.zeros((self.CONSTANT_ROW_SIZE, self.CONSTANT_COL_SIZE))
            for x in nodes:
                weight = self.network[x][node][key]['weight']
                term = self.get_phenotype(x)
                logit, term = self.same_size(logit, term)
                logit += weight * term
            return logit
        # if there are no links for this logit, return false
        else:
            return None

    # compute output for the node
    def node_output(self, node, left_logit, right_logit):

        # output node acts just like a sum
        if self.network.node[node]['type'] == NodeType.output:
            if left_logit is None:
                return right_logit
            elif right_logit is None:
                return left_logit
            else:
                left_logit, right_logit = self.same_size(left_logit, right_logit)
                return np.add(left_logit, right_logit)

        elif self.network.node[node]['type'] == NodeType.input:
            return self.network.node[node]['constant']

        elif self.network.node[node]['type'] == NodeType.kronecker:
            if left_logit is None:
                result = np.kron(right_logit, right_logit)
                return result
            elif right_logit is None:
                result = np.kron(left_logit, left_logit)
                return result
            else:
                return np.kron(left_logit, right_logit)

        elif self.network.node[node]['type'] == NodeType.hadamard:
            if left_logit is None:
                return right_logit
            elif right_logit is None:
                return left_logit
            else:
                left_logit, right_logit = self.same_size(left_logit, right_logit)
                return np.multiply(left_logit, right_logit)

    # returns left and right so that they are the same size, assuming both are square
    # and share some common factor of their sides
    def same_size(self, left, right):
        left_row_size = np.ma.size(left, 0)
        right_row_size = np.ma.size(right, 0)
        if left_row_size != right_row_size:
            if left_row_size < right_row_size:
                length = int(right_row_size / left_row_size)
                left = np.kron(left, np.ones((length, length)))
            else:
                length = int(left_row_size / right_row_size)
                right = np.kron(right, np.ones((length, length)))

        return left, right

# Testing Genotype stuff

genome = Genome()

for i in range(0, 40):
    genome.mutate()

print("printing nodes")
for node in genome.network.nodes(data=True):
    print(node)

print("printing edges")
for edge in genome.network.edges(data=True, keys=True):
    print(edge)

node_size = 1000
node_color = 'white'
node_text_size = 15
edge_alpha = 0.3
edge_thickness = 1
edge_text_pos = 0.8
text_font = 'sans-serif'

pos = nx.spring_layout(genome.network)

nx.draw_networkx(genome.network, pos, node_size=node_size, node_color=node_color)
#nx.draw_networkx_nodes(genome.network, pos, node_size=node_size, node_color=node_color)
#nx.draw_networkx_edges(genome.network, pos, width=colorList, edge_color=colorList, edge_cmap=plt.cm.RdYlGn, arrows=True)
#nx.draw_networkx_labels(genome.network, pos, labels=labels, font_size=node_text_size,
#                        font_family=text_font)
#nx.draw_networkx_edge_labels(genome.network, pos)
plt.show()

print("getting phenotype")
phenotype = genome.get_phenotype()
print(phenotype)
print(np.size(phenotype, 0))

plt.matshow(phenotype, fignum=100, cmap=plt.cm.gray)
plt.show()<|endoftext|>


<commit_before># Copyright (C) 2007-2010 Michael Foord
# E-mail: fuzzyman AT voidspace DOT org DOT uk
# http://www.voidspace.org.uk/python/mock/

from __future__ import with_statement

import os
import sys

info = sys.version_info
if info[:3] >= (3, 2, 0) or info[0] == 2 and info[1] >= 7:
    # for Python 2.7 and 3.2 ordinary unittest is fine
    import unittest as unittest2
else:
    import unittest2

this_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
if not this_dir in sys.path:
    sys.path.insert(0, this_dir)

if __name__ == '__main__':
    sys.modules['testwith'] = sys.modules['__main__']

if 'testwith' in sys.modules:
    # Fix for running tests under Wing
    import tests
    import testwith
    tests.testwith = testwith

from mock import Mock, patch, patch_object, sentinel

something  = sentinel.Something
something_else  = sentinel.SomethingElse

try:
    from contextlib import nested
except ImportError:
    from contextlib import contextmanager
    @contextmanager
    def nested(*managers):
        exits = []
        vars = []
        exc = (None, None, None)
        try:
            for mgr in managers:
                exit = mgr.__exit__
                enter = mgr.__enter__
                vars.append(enter())
                exits.append(exit)
            yield vars
        except:
            exc = sys.exc_info()
        finally:
            while exits:
                exit = exits.pop()
                try:
                    if exit(*exc):
                        exc = (None, None, None)
                except:
                    exc = sys.exc_info()
            if exc != (None, None, None):
                raise exc[1]

class WithTest(unittest2.TestCase):
    def testWithStatement(self):
        with patch('tests.testwith.something', sentinel.Something2):
            self.assertEquals(something, sentinel.Something2, "unpatched")
        self.assertEquals(something, sentinel.Something)

    def testWithStatementException(self):
        try:
            with patch('tests.testwith.something', sentinel.Something2):
                self.assertEquals(something, sentinel.Something2, "unpatched")
                raise Exception('pow')
        except Exception:
            pass
        else:
            self.fail("patch swallowed exception")
        self.assertEquals(something, sentinel.Something)

    def testWithStatementAs(self):
        with patch('tests.testwith.something') as mock_something:
            self.assertEquals(something, mock_something, "unpatched")
            self.assertTrue(isinstance(mock_something, Mock), "patching wrong type")
        self.assertEquals(something, sentinel.Something)

    def testPatchObjectWithStatementAs(self):
        mock = Mock()
        original = mock.something
        with patch_object(mock, 'something') as mock_something:
            self.assertNotEquals(mock.something, original, "unpatched")
        self.assertEquals(mock.something, original)

    def testWithStatementNested(self):
        with nested(patch('tests.testwith.something'),
                    patch('tests.testwith.something_else')) as (mock_something, mock_something_else):
            self.assertEquals(something, mock_something, "unpatched")
            self.assertEquals(something_else, mock_something_else, "unpatched")
        self.assertEquals(something, sentinel.Something)
        self.assertEquals(something_else, sentinel.SomethingElse)

    def testWithStatementSpecified(self):
        with patch('tests.testwith.something', sentinel.Patched) as mock_something:
            self.assertEquals(something, mock_something, "unpatched")
            self.assertEquals(mock_something, sentinel.Patched, "wrong patch")
        self.assertEquals(something, sentinel.Something)

    def testContextManagerMocking(self):
        mock = Mock()
        mock.__enter__ = Mock()
        mock.__exit__ = Mock()
        mock.__exit__.return_value = False

        with mock as m:
            self.assertEqual(m, mock.__enter__.return_value)
        mock.__enter__.assert_called_with()
        mock.__exit__.assert_called_with(None, None, None)

if __name__ == '__main__':
    unittest2.main()

<commit_msg>Test for MagicMock as a context manager<commit_after># Copyright (C) 2007-2010 Michael Foord
# E-mail: fuzzyman AT voidspace DOT org DOT uk
# http://www.voidspace.org.uk/python/mock/

from __future__ import with_statement

import os
import sys

info = sys.version_info
if info[:3] >= (3, 2, 0) or info[0] == 2 and info[1] >= 7:
    # for Python 2.7 and 3.2 ordinary unittest is fine
    import unittest as unittest2
else:
    import unittest2

this_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
if not this_dir in sys.path:
    sys.path.insert(0, this_dir)

if __name__ == '__main__':
    sys.modules['testwith'] = sys.modules['__main__']

if 'testwith' in sys.modules:
    # Fix for running tests under Wing
    import tests
    import testwith
    tests.testwith = testwith

from mock import MagicMock, Mock, patch, patch_object, sentinel

something  = sentinel.Something
something_else  = sentinel.SomethingElse

try:
    from contextlib import nested
except ImportError:
    from contextlib import contextmanager
    @contextmanager
    def nested(*managers):
        exits = []
        vars = []
        exc = (None, None, None)
        try:
            for mgr in managers:
                exit = mgr.__exit__
                enter = mgr.__enter__
                vars.append(enter())
                exits.append(exit)
            yield vars
        except:
            exc = sys.exc_info()
        finally:
            while exits:
                exit = exits.pop()
                try:
                    if exit(*exc):
                        exc = (None, None, None)
                except:
                    exc = sys.exc_info()
            if exc != (None, None, None):
                raise exc[1]

class WithTest(unittest2.TestCase):
    def testWithStatement(self):
        with patch('tests.testwith.something', sentinel.Something2):
            self.assertEquals(something, sentinel.Something2, "unpatched")
        self.assertEquals(something, sentinel.Something)

    def testWithStatementException(self):
        try:
            with patch('tests.testwith.something', sentinel.Something2):
                self.assertEquals(something, sentinel.Something2, "unpatched")
                raise Exception('pow')
        except Exception:
            pass
        else:
            self.fail("patch swallowed exception")
        self.assertEquals(something, sentinel.Something)

    def testWithStatementAs(self):
        with patch('tests.testwith.something') as mock_something:
            self.assertEquals(something, mock_something, "unpatched")
            self.assertTrue(isinstance(mock_something, Mock), "patching wrong type")
        self.assertEquals(something, sentinel.Something)

    def testPatchObjectWithStatementAs(self):
        mock = Mock()
        original = mock.something
        with patch_object(mock, 'something') as mock_something:
            self.assertNotEquals(mock.something, original, "unpatched")
        self.assertEquals(mock.something, original)

    def testWithStatementNested(self):
        with nested(patch('tests.testwith.something'),
                    patch('tests.testwith.something_else')) as (mock_something, mock_something_else):
            self.assertEquals(something, mock_something, "unpatched")
            self.assertEquals(something_else, mock_something_else, "unpatched")
        self.assertEquals(something, sentinel.Something)
        self.assertEquals(something_else, sentinel.SomethingElse)

    def testWithStatementSpecified(self):
        with patch('tests.testwith.something', sentinel.Patched) as mock_something:
            self.assertEquals(something, mock_something, "unpatched")
            self.assertEquals(mock_something, sentinel.Patched, "wrong patch")
        self.assertEquals(something, sentinel.Something)

    def testContextManagerMocking(self):
        mock = Mock()
        mock.__enter__ = Mock()
        mock.__exit__ = Mock()
        mock.__exit__.return_value = False

        with mock as m:
            self.assertEqual(m, mock.__enter__.return_value)
        mock.__enter__.assert_called_with()
        mock.__exit__.assert_called_with(None, None, None)

    def testContextManagerWithMagicMock(self):
        mock = MagicMock()

        with self.assertRaises(TypeError):
            with mock as m:
                'foo' + 3
        mock.__enter__.assert_called_with()
        self.assertTrue(mock.__exit__.called)

if __name__ == '__main__':
    unittest2.main()

<|endoftext|>