<commit_before>#!/usr/bin/env python

import os
import shutil
import subprocess
import sys
import paths
from optloader import load_options

options = {}

DEFAULT_BUNDLE_PREFIX = 'monitoring'
DEFAULT_BUNDLE_NAME = "%s.zip" % DEFAULT_BUNDLE_PREFIX
DEFAULT_SIGNATURE_NAME = "%s.zip.sig" % DEFAULT_BUNDLE_PREFIX
DEFAULT_BUNDLE_PATH = os.path.join(paths.root, '..', 'bundle')

def _get_bundle_filename(filename=DEFAULT_BUNDLE_NAME):
    return os.path.join(paths.output_path, filename)

def _get_signature_filename(filename=DEFAULT_SIGNATURE_NAME):
    return os.path.join(paths.output_path, filename)

def _get_version():
    # capture the current version
    cmd = 'python tools/version.py'
    print cmd
    p = subprocess.Popen(["python", "tools/version.py"], stdout=subprocess.PIPE)
    version, err = p.communicate()
    return version.strip()

def extra_env():
    env = {}
    if sys.platform.find('freebsd') == 0:
        env['CC'] = 'gcc48'
        env['CXX'] = 'g++48'
    return env

def _build(prod=False):
    env_str = ''
    if prod:
        env_str = 'PRODUCTION=1'
    if sys.platform.find('freebsd') == 0:
        cmd = '%s gmake -C %s' % (env_str, paths.root)
    elif sys.platform != "win32":
        cmd = '%s make -C %s' % (env_str, paths.root)
    else:
        build = 'Debug' if options['variables']['virgo_debug'] == 'true' else 'Release'
        if prod:
            env_str = 'Production'
        cmd = 'tools\win_build.bat %s %s' % (build, env_str)

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def build():
    _build()

def build_prod():
    _build(True)

def sig_gen(signingkey, filename, sigfilename):
    if not filename or not sigfilename:
        print "sig-gen requires additioanl parameters of <filename> <signature_filename>"
        sys.exit(1)
    cmd = '%s dgst -sha256 -sign %s %s > %s' % (
        options['variables']['OPENSSL'], signingkey, filename, sigfilename)
    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def pkg():
    if sys.platform.find('freebsd') == 0:
        cmd = 'BUILDTYPE=%s gmake -C %s pkg' % (paths.BUILDTYPE, paths.root)
    elif sys.platform != "win32":
        cmd = 'BUILDTYPE=%s make -C %s pkg' % (paths.BUILDTYPE, paths.root)
    else:
        build = 'Debug' if options['variables']['virgo_debug'] == 'true' else 'Release'
        cmd = 'tools\win_pkg.bat %s' % build

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def exe_sign():
    if sys.platform == "win32":
        cmd = 'tools\win_sign.py exe'
    else:
        print "Executable Signing is only supported on Win32"
        sys.exit(1)

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def pkg_sign():
    if sys.platform.find('freebsd') == 0:
        cmd = 'BUILDTYPE=%s gmake -C %s pkg-sign' % (paths.BUILDTYPE, paths.root)
    elif sys.platform != "win32":
        cmd = 'BUILDTYPE=%s make -C %s pkg-sign' % (paths.BUILDTYPE, paths.root)
    else:
        cmd = 'tools\win_sign.py pkg'
    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def test_cmd(additional=""):
    state_config = os.path.join(paths.root, 'contrib')
    monitoring_config = os.path.join(paths.root, 'agents', 'monitoring', 'tests', 'fixtures', 'monitoring-agent-localhost.cfg')

    return '%s -o -d -c %s -s %s %s' % (paths.agent, monitoring_config, state_config, additional)

def test(stdout=None, entry="tests", flags=None):
    agent_tests = _get_bundle_filename('monitoring-test.zip')
    cmd = test_cmd("--zip %s -e %s -o" % (agent_tests, entry))
    if flags:
        for flag in flags:
            cmd += " %s" % (flag)
    print cmd
    rc = 0
    if stdout is None:
        rc = subprocess.call(cmd, shell=True)
    else:
        rc = subprocess.call(cmd, shell=True, stdout=stdout)
    sys.exit(rc)

def test_std():
    test()

def test_pipe():
    test(subprocess.PIPE)

def test_file():
    stdout = open("stdout", "w+")
    test(stdout)

def crash():
    test(None, "crash", flags=["--production"])

def bundle(directory=DEFAULT_BUNDLE_PATH):
    bundle_filename = _get_bundle_filename()
    signature_filename = _get_signature_filename()
    version = _get_version()

    print "Removing %s" % (directory)
    shutil.rmtree(directory, True)
    print "mkdir %s" % (directory)
    os.mkdir(directory)

    # copy bundle
    dest_path = os.path.join(directory, "monitoring-%s.zip" % (version))
    print "Copying %s to %s" % (_get_bundle_filename(), dest_path)
    shutil.copy(_get_bundle_filename(), dest_path)

    # copy signature
    dest_path = os.path.join(directory, "monitoring-%s.zip.sig" % (version))
    print "Copying %s to %s" % (_get_signature_filename(), dest_path)
    shutil.copy(_get_signature_filename(), dest_path)

    # create VERSION file
    version_file_name = os.path.join(directory, 'VERSION')
    print "Creating %s" % (version_file_name)
    version_file = open(version_file_name, "w")
    version_file.write(version)
    version_file.write('\n')
    version_file.close()

commands = {
    'crash': crash,
    'bundle': bundle,
    'build': build,
    'build_prod': build_prod,
    'sig-gen': sig_gen,
    'pkg': pkg,
    'exe-sign': exe_sign,
    'pkg-sign': pkg_sign,
    'test': test_std,
    'test_pipe': test_pipe,
    'test_file': test_file,
}

def usage():
    print('Usage: build.py [%s]' % ', '.join(commands.keys()))
    sys.exit(1)

if len(sys.argv) < 2:
    usage()

ins = sys.argv[1]
if not ins in commands:
    print('Invalid command: %s' % ins)
    sys.exit(1)

extra_env = extra_env()

for key in extra_env.keys():
    os.environ[key] = extra_env[key]

options = load_options()

print('Running %s' % ins)
cmd = commands.get(ins)
cmd(*(sys.argv[2:]))
<commit_msg>hack version string to never be greater<commit_after>#!/usr/bin/env python

import os
import shutil
import subprocess
import sys
import paths
from optloader import load_options

options = {}

DEFAULT_BUNDLE_PREFIX = 'monitoring'
DEFAULT_BUNDLE_NAME = "%s.zip" % DEFAULT_BUNDLE_PREFIX
DEFAULT_SIGNATURE_NAME = "%s.zip.sig" % DEFAULT_BUNDLE_PREFIX
DEFAULT_BUNDLE_PATH = os.path.join(paths.root, '..', 'bundle')

def _get_bundle_filename(filename=DEFAULT_BUNDLE_NAME):
    return os.path.join(paths.output_path, filename)

def _get_signature_filename(filename=DEFAULT_SIGNATURE_NAME):
    return os.path.join(paths.output_path, filename)

def _get_version():
    # capture the current version
    cmd = 'python tools/version.py'
    print cmd
    p = subprocess.Popen(["python", "tools/version.py"], stdout=subprocess.PIPE)
    version, err = p.communicate()
    return version.strip()

def extra_env():
    env = {}
    if sys.platform.find('freebsd') == 0:
        env['CC'] = 'gcc48'
        env['CXX'] = 'g++48'
    return env

def _build(prod=False):
    env_str = ''
    if prod:
        env_str = 'PRODUCTION=1'
    if sys.platform.find('freebsd') == 0:
        cmd = '%s gmake -C %s' % (env_str, paths.root)
    elif sys.platform != "win32":
        cmd = '%s make -C %s' % (env_str, paths.root)
    else:
        build = 'Debug' if options['variables']['virgo_debug'] == 'true' else 'Release'
        if prod:
            env_str = 'Production'
        cmd = 'tools\win_build.bat %s %s' % (build, env_str)

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def build():
    _build()

def build_prod():
    _build(True)

def sig_gen(signingkey, filename, sigfilename):
    if not filename or not sigfilename:
        print "sig-gen requires additioanl parameters of <filename> <signature_filename>"
        sys.exit(1)
    cmd = '%s dgst -sha256 -sign %s %s > %s' % (
        options['variables']['OPENSSL'], signingkey, filename, sigfilename)
    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def pkg():
    if sys.platform.find('freebsd') == 0:
        cmd = 'BUILDTYPE=%s gmake -C %s pkg' % (paths.BUILDTYPE, paths.root)
    elif sys.platform != "win32":
        cmd = 'BUILDTYPE=%s make -C %s pkg' % (paths.BUILDTYPE, paths.root)
    else:
        build = 'Debug' if options['variables']['virgo_debug'] == 'true' else 'Release'
        cmd = 'tools\win_pkg.bat %s' % build

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def exe_sign():
    if sys.platform == "win32":
        cmd = 'tools\win_sign.py exe'
    else:
        print "Executable Signing is only supported on Win32"
        sys.exit(1)

    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def pkg_sign():
    if sys.platform.find('freebsd') == 0:
        cmd = 'BUILDTYPE=%s gmake -C %s pkg-sign' % (paths.BUILDTYPE, paths.root)
    elif sys.platform != "win32":
        cmd = 'BUILDTYPE=%s make -C %s pkg-sign' % (paths.BUILDTYPE, paths.root)
    else:
        cmd = 'tools\win_sign.py pkg'
    print cmd
    sys.exit(subprocess.call(cmd, shell=True))

def test_cmd(additional=""):
    state_config = os.path.join(paths.root, 'contrib')
    monitoring_config = os.path.join(paths.root, 'agents', 'monitoring', 'tests', 'fixtures', 'monitoring-agent-localhost.cfg')

    return '%s -o -d -c %s -s %s %s' % (paths.agent, monitoring_config, state_config, additional)

def test(stdout=None, entry="tests", flags=None):
    agent_tests = _get_bundle_filename('monitoring-test.zip')
    cmd = test_cmd("--zip %s -e %s -o" % (agent_tests, entry))
    if flags:
        for flag in flags:
            cmd += " %s" % (flag)
    print cmd
    rc = 0
    if stdout is None:
        rc = subprocess.call(cmd, shell=True)
    else:
        rc = subprocess.call(cmd, shell=True, stdout=stdout)
    sys.exit(rc)

def test_std():
    test()

def test_pipe():
    test(subprocess.PIPE)

def test_file():
    stdout = open("stdout", "w+")
    test(stdout)

def crash():
    test(None, "crash", flags=["--production"])

def bundle(directory=DEFAULT_BUNDLE_PATH):
    bundle_filename = _get_bundle_filename()
    signature_filename = _get_signature_filename()
    version = _get_version()

    print "Removing %s" % (directory)
    shutil.rmtree(directory, True)
    print "mkdir %s" % (directory)
    os.mkdir(directory)

    # copy bundle
    dest_path = os.path.join(directory, "monitoring-%s.zip" % (version))
    print "Copying %s to %s" % (_get_bundle_filename(), dest_path)
    shutil.copy(_get_bundle_filename(), dest_path)

    # copy signature
    dest_path = os.path.join(directory, "monitoring-%s.zip.sig" % (version))
    print "Copying %s to %s" % (_get_signature_filename(), dest_path)
    shutil.copy(_get_signature_filename(), dest_path)

    # create VERSION file
    version_file_name = os.path.join(directory, 'VERSION')
    print "Creating %s" % (version_file_name)
    version_file = open(version_file_name, "w")
    # version_file.write(version)
    version_file.write("0.0.0-0")
    version_file.write('\n')
    version_file.close()

commands = {
    'crash': crash,
    'bundle': bundle,
    'build': build,
    'build_prod': build_prod,
    'sig-gen': sig_gen,
    'pkg': pkg,
    'exe-sign': exe_sign,
    'pkg-sign': pkg_sign,
    'test': test_std,
    'test_pipe': test_pipe,
    'test_file': test_file,
}

def usage():
    print('Usage: build.py [%s]' % ', '.join(commands.keys()))
    sys.exit(1)

if len(sys.argv) < 2:
    usage()

ins = sys.argv[1]
if not ins in commands:
    print('Invalid command: %s' % ins)
    sys.exit(1)

extra_env = extra_env()

for key in extra_env.keys():
    os.environ[key] = extra_env[key]

options = load_options()

print('Running %s' % ins)
cmd = commands.get(ins)
cmd(*(sys.argv[2:]))
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import multiprocessing
import shutil
import subprocess
import sys
from os import makedirs, uname
from settings import *

BUILD_DIR = path.join(PROJECT_DIR, 'build')

def default_toolchain():
    (sysname, _, _, _, machine) = uname()
    toolchain = path.join(PROJECT_DIR, 'cmake', 'toolchain_%s_%s.cmake' % (sysname.lower(), machine.lower()))
    return toolchain if path.isfile(toolchain) else None

def get_arguments():
    devhelp_preparser = argparse.ArgumentParser(add_help=False)
    devhelp_preparser.add_argument('--devhelp', action='store_true', default=False, help='show help with all options (including those, which are useful for developers only)')

    devhelp_arguments, args = devhelp_preparser.parse_known_args()
    if devhelp_arguments.devhelp:
        args.append('--devhelp')

    def devhelp(help):
        return help if devhelp_arguments.devhelp else argparse.SUPPRESS

    parser = argparse.ArgumentParser(parents=[devhelp_preparser])
    parser.add_argument('--all-in-one', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='all-in-one build (%(choices)s; default: %(default)s)')
    parser.add_argument('--builddir', metavar='DIR', action='store', default=BUILD_DIR, help='specify output directory (default: %(default)s)')
    parser.add_argument('--clean', action='store_true', default=False, help='clean build')
    parser.add_argument('--cmake-param', metavar='OPT', action='append', default=[], help='add custom argument to CMake')
    parser.add_argument('--compile-flag', metavar='OPT', action='append', default=[], help='add custom compile flag')
    parser.add_argument('--cpointer-32bit', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable 32 bit compressed pointers (%(choices)s; default: %(default)s)')
    parser.add_argument('--debug', action='store_const', const='Debug', default='Release', dest='build_type', help='debug build')
    parser.add_argument('--error-messages', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable error messages (%(choices)s; default: %(default)s)')
    parser.add_argument('-j', '--jobs', metavar='N', action='store', type=int, default=multiprocessing.cpu_count() + 1, help='Allowed N build jobs at once (default: %(default)s)')
    parser.add_argument('--jerry-cmdline', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build jerry command line tool (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-cmdline-minimal', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='build minimal version of the jerry command line tool (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-libc', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build and use jerry-libc (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-libm', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build and use jerry-libm (%(choices)s; default: %(default)s)')
    parser.add_argument('--js-parser', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable js-parser (%(choices)s; default: %(default)s)')
    parser.add_argument('--link-lib', metavar='OPT', action='append', default=[], help='add custom library to be linked')
    parser.add_argument('--linker-flag', metavar='OPT', action='append', default=[], help='add custom linker flag')
    parser.add_argument('--lto', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable link-time optimizations (%(choices)s; default: %(default)s)')
    parser.add_argument('--profile', metavar='PROFILE', choices=['es5.1', 'minimal', 'es2015-subset'], default='es5.1', type=str.lower, help='specify the profile (%(choices)s; default: %(default)s)')
    parser.add_argument('--snapshot-exec', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable executing snapshot files (%(choices)s; default: %(default)s)')
    parser.add_argument('--snapshot-save', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable saving snapshot files (%(choices)s; default: %(default)s)')
    parser.add_argument('--static-link', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable static linking of binaries (%(choices)s; default: %(default)s)')
    parser.add_argument('--strip', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='strip release binaries (%(choices)s; default: %(default)s)')
    parser.add_argument('--toolchain', metavar='FILE', action='store', default=default_toolchain(), help='add toolchain file (default: %(default)s)')
    parser.add_argument('--unittests', action='store_const', const='ON', default='OFF', help='build unittests')
    parser.add_argument('-v', '--verbose', action='store_const', const='ON', default='OFF', help='increase verbosity')

    devgroup = parser.add_argument_group('developer options')
    devgroup.add_argument('--link-map', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable the generation of a link map file for jerry command line tool (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--mem-stats', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable memory statistics (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--mem-stress-test', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable mem-stress test (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--show-opcodes', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable parser byte-code dumps (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--show-regexp-opcodes', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable regexp byte-code dumps (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--valgrind', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable Valgrind support (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--valgrind-freya', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable Valgrind-Freya support (%(choices)s; default: %(default)s)'))

    arguments = parser.parse_args(args)
    if arguments.devhelp:
        parser.print_help()
        sys.exit(0)

    return arguments

def generate_build_options(arguments):
    build_options = []

    build_options.append('-DENABLE_ALL_IN_ONE=%s' % arguments.all_in_one)
    build_options.append('-DCMAKE_BUILD_TYPE=%s' % arguments.build_type)
    build_options.extend(arguments.cmake_param)
    build_options.append('-DEXTERNAL_COMPILE_FLAGS=' + ' '.join(arguments.compile_flag))
    build_options.append('-DFEATURE_CPOINTER_32_BIT=%s' % arguments.cpointer_32bit)
    build_options.append('-DFEATURE_ERROR_MESSAGES=%s' % arguments.error_messages)
    build_options.append('-DJERRY_CMDLINE=%s' % arguments.jerry_cmdline)
    build_options.append('-DJERRY_CMDLINE_MINIMAL=%s' % arguments.jerry_cmdline_minimal)
    build_options.append('-DJERRY_LIBC=%s' % arguments.jerry_libc)
    build_options.append('-DJERRY_LIBM=%s' % arguments.jerry_libm)
    build_options.append('-DFEATURE_JS_PARSER=%s' % arguments.js_parser)
    build_options.append('-DEXTERNAL_LINK_LIBS=' + ' '.join(arguments.link_lib))
    build_options.append('-DEXTERNAL_LINKER_FLAGS=' + ' '.join(arguments.linker_flag))
    build_options.append('-DENABLE_LTO=%s' % arguments.lto)
    build_options.append('-DFEATURE_PROFILE=%s' % arguments.profile)
    build_options.append('-DFEATURE_SNAPSHOT_EXEC=%s' % arguments.snapshot_exec)
    build_options.append('-DFEATURE_SNAPSHOT_SAVE=%s' % arguments.snapshot_save)
    build_options.append('-DENABLE_STATIC_LINK=%s' % arguments.static_link)
    build_options.append('-DENABLE_STRIP=%s' % arguments.strip)

    if arguments.toolchain:
      build_options.append('-DCMAKE_TOOLCHAIN_FILE=%s' % arguments.toolchain)

    build_options.append('-DUNITTESTS=%s' % arguments.unittests)
    build_options.append('-DCMAKE_VERBOSE_MAKEFILE=%s' % arguments.verbose)

    # developer options
    build_options.append('-DENABLE_LINK_MAP=%s' % arguments.link_map)
    build_options.append('-DFEATURE_MEM_STATS=%s' % arguments.mem_stats)
    build_options.append('-DFEATURE_MEM_STRESS_TEST=%s' % arguments.mem_stress_test)
    build_options.append('-DFEATURE_PARSER_DUMP=%s' % arguments.show_opcodes)
    build_options.append('-DFEATURE_REGEXP_DUMP=%s' % arguments.show_regexp_opcodes)
    build_options.append('-DFEATURE_VALGRIND=%s' % arguments.valgrind)
    build_options.append('-DFEATURE_VALGRIND_FREYA=%s' % arguments.valgrind_freya)

    return build_options

def configure_output_dir(arguments):
    global BUILD_DIR

    if path.isabs(arguments.builddir):
        BUILD_DIR = arguments.builddir
    else:
        BUILD_DIR = path.join(PROJECT_DIR, arguments.builddir)

    if arguments.clean and path.exists(BUILD_DIR):
        shutil.rmtree(BUILD_DIR)

    if not path.exists(BUILD_DIR):
        makedirs(BUILD_DIR)

def configure_build(arguments):
    configure_output_dir(arguments)

    build_options = generate_build_options(arguments)

    cmake_cmd = ['cmake', '-B' + BUILD_DIR, '-H' + PROJECT_DIR]
    cmake_cmd.extend(build_options)

    return subprocess.call(cmake_cmd)

def build_jerry(arguments):
    return subprocess.call(['make', '--no-print-directory','-j', str(arguments.jobs), '-C', BUILD_DIR])

def print_result(ret):
    print('=' * 30)
    if ret:
        print('Build failed with exit code: %s' % (ret))
    else:
        print('Build succeeded!')
    print('=' * 30)

def main():
    arguments = get_arguments()
    ret = configure_build(arguments)

    if not ret:
        ret = build_jerry(arguments)

    print_result(ret)
    sys.exit(ret)

if __name__ == "__main__":
    main()
<commit_msg>Set `PORT_DIR` in the build script. (#1492)<commit_after>#!/usr/bin/env python

# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import multiprocessing
import shutil
import subprocess
import sys
from os import makedirs, uname
from settings import *

BUILD_DIR = path.join(PROJECT_DIR, 'build')
DEFAULT_PORT_DIR = path.join(PROJECT_DIR, 'targets/default')

def default_toolchain():
    (sysname, _, _, _, machine) = uname()
    toolchain = path.join(PROJECT_DIR, 'cmake', 'toolchain_%s_%s.cmake' % (sysname.lower(), machine.lower()))
    return toolchain if path.isfile(toolchain) else None

def get_arguments():
    devhelp_preparser = argparse.ArgumentParser(add_help=False)
    devhelp_preparser.add_argument('--devhelp', action='store_true', default=False, help='show help with all options (including those, which are useful for developers only)')

    devhelp_arguments, args = devhelp_preparser.parse_known_args()
    if devhelp_arguments.devhelp:
        args.append('--devhelp')

    def devhelp(help):
        return help if devhelp_arguments.devhelp else argparse.SUPPRESS

    parser = argparse.ArgumentParser(parents=[devhelp_preparser])
    parser.add_argument('--all-in-one', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='all-in-one build (%(choices)s; default: %(default)s)')
    parser.add_argument('--builddir', metavar='DIR', action='store', default=BUILD_DIR, help='specify output directory (default: %(default)s)')
    parser.add_argument('--clean', action='store_true', default=False, help='clean build')
    parser.add_argument('--cmake-param', metavar='OPT', action='append', default=[], help='add custom argument to CMake')
    parser.add_argument('--compile-flag', metavar='OPT', action='append', default=[], help='add custom compile flag')
    parser.add_argument('--cpointer-32bit', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable 32 bit compressed pointers (%(choices)s; default: %(default)s)')
    parser.add_argument('--debug', action='store_const', const='Debug', default='Release', dest='build_type', help='debug build')
    parser.add_argument('--error-messages', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable error messages (%(choices)s; default: %(default)s)')
    parser.add_argument('-j', '--jobs', metavar='N', action='store', type=int, default=multiprocessing.cpu_count() + 1, help='Allowed N build jobs at once (default: %(default)s)')
    parser.add_argument('--jerry-cmdline', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build jerry command line tool (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-cmdline-minimal', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='build minimal version of the jerry command line tool (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-libc', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build and use jerry-libc (%(choices)s; default: %(default)s)')
    parser.add_argument('--jerry-libm', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='build and use jerry-libm (%(choices)s; default: %(default)s)')
    parser.add_argument('--js-parser', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable js-parser (%(choices)s; default: %(default)s)')
    parser.add_argument('--link-lib', metavar='OPT', action='append', default=[], help='add custom library to be linked')
    parser.add_argument('--linker-flag', metavar='OPT', action='append', default=[], help='add custom linker flag')
    parser.add_argument('--lto', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable link-time optimizations (%(choices)s; default: %(default)s)')
    parser.add_argument('--port-dir', metavar='DIR', action='store', default=DEFAULT_PORT_DIR, help='add port directory (default: %(default)s)')
    parser.add_argument('--profile', metavar='PROFILE', choices=['es5.1', 'minimal', 'es2015-subset'], default='es5.1', type=str.lower, help='specify the profile (%(choices)s; default: %(default)s)')
    parser.add_argument('--snapshot-exec', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable executing snapshot files (%(choices)s; default: %(default)s)')
    parser.add_argument('--snapshot-save', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help='enable saving snapshot files (%(choices)s; default: %(default)s)')
    parser.add_argument('--static-link', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='enable static linking of binaries (%(choices)s; default: %(default)s)')
    parser.add_argument('--strip', metavar='X', choices=['ON', 'OFF'], default='ON', type=str.upper, help='strip release binaries (%(choices)s; default: %(default)s)')
    parser.add_argument('--toolchain', metavar='FILE', action='store', default=default_toolchain(), help='add toolchain file (default: %(default)s)')
    parser.add_argument('--unittests', action='store_const', const='ON', default='OFF', help='build unittests')
    parser.add_argument('-v', '--verbose', action='store_const', const='ON', default='OFF', help='increase verbosity')

    devgroup = parser.add_argument_group('developer options')
    devgroup.add_argument('--link-map', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable the generation of a link map file for jerry command line tool (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--mem-stats', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable memory statistics (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--mem-stress-test', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable mem-stress test (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--show-opcodes', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable parser byte-code dumps (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--show-regexp-opcodes', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable regexp byte-code dumps (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--valgrind', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable Valgrind support (%(choices)s; default: %(default)s)'))
    devgroup.add_argument('--valgrind-freya', metavar='X', choices=['ON', 'OFF'], default='OFF', type=str.upper, help=devhelp('enable Valgrind-Freya support (%(choices)s; default: %(default)s)'))

    arguments = parser.parse_args(args)
    if arguments.devhelp:
        parser.print_help()
        sys.exit(0)

    return arguments

def generate_build_options(arguments):
    build_options = []

    build_options.append('-DENABLE_ALL_IN_ONE=%s' % arguments.all_in_one)
    build_options.append('-DCMAKE_BUILD_TYPE=%s' % arguments.build_type)
    build_options.extend(arguments.cmake_param)
    build_options.append('-DEXTERNAL_COMPILE_FLAGS=' + ' '.join(arguments.compile_flag))
    build_options.append('-DFEATURE_CPOINTER_32_BIT=%s' % arguments.cpointer_32bit)
    build_options.append('-DFEATURE_ERROR_MESSAGES=%s' % arguments.error_messages)
    build_options.append('-DJERRY_CMDLINE=%s' % arguments.jerry_cmdline)
    build_options.append('-DJERRY_CMDLINE_MINIMAL=%s' % arguments.jerry_cmdline_minimal)
    build_options.append('-DJERRY_LIBC=%s' % arguments.jerry_libc)
    build_options.append('-DJERRY_LIBM=%s' % arguments.jerry_libm)
    build_options.append('-DFEATURE_JS_PARSER=%s' % arguments.js_parser)
    build_options.append('-DEXTERNAL_LINK_LIBS=' + ' '.join(arguments.link_lib))
    build_options.append('-DEXTERNAL_LINKER_FLAGS=' + ' '.join(arguments.linker_flag))
    build_options.append('-DENABLE_LTO=%s' % arguments.lto)
    build_options.append('-DPORT_DIR=%s' % arguments.port_dir)
    build_options.append('-DFEATURE_PROFILE=%s' % arguments.profile)
    build_options.append('-DFEATURE_SNAPSHOT_EXEC=%s' % arguments.snapshot_exec)
    build_options.append('-DFEATURE_SNAPSHOT_SAVE=%s' % arguments.snapshot_save)
    build_options.append('-DENABLE_STATIC_LINK=%s' % arguments.static_link)
    build_options.append('-DENABLE_STRIP=%s' % arguments.strip)

    if arguments.toolchain:
      build_options.append('-DCMAKE_TOOLCHAIN_FILE=%s' % arguments.toolchain)

    build_options.append('-DUNITTESTS=%s' % arguments.unittests)
    build_options.append('-DCMAKE_VERBOSE_MAKEFILE=%s' % arguments.verbose)

    # developer options
    build_options.append('-DENABLE_LINK_MAP=%s' % arguments.link_map)
    build_options.append('-DFEATURE_MEM_STATS=%s' % arguments.mem_stats)
    build_options.append('-DFEATURE_MEM_STRESS_TEST=%s' % arguments.mem_stress_test)
    build_options.append('-DFEATURE_PARSER_DUMP=%s' % arguments.show_opcodes)
    build_options.append('-DFEATURE_REGEXP_DUMP=%s' % arguments.show_regexp_opcodes)
    build_options.append('-DFEATURE_VALGRIND=%s' % arguments.valgrind)
    build_options.append('-DFEATURE_VALGRIND_FREYA=%s' % arguments.valgrind_freya)

    return build_options

def configure_output_dir(arguments):
    global BUILD_DIR

    if path.isabs(arguments.builddir):
        BUILD_DIR = arguments.builddir
    else:
        BUILD_DIR = path.join(PROJECT_DIR, arguments.builddir)

    if arguments.clean and path.exists(BUILD_DIR):
        shutil.rmtree(BUILD_DIR)

    if not path.exists(BUILD_DIR):
        makedirs(BUILD_DIR)

def configure_build(arguments):
    configure_output_dir(arguments)

    build_options = generate_build_options(arguments)

    cmake_cmd = ['cmake', '-B' + BUILD_DIR, '-H' + PROJECT_DIR]
    cmake_cmd.extend(build_options)

    return subprocess.call(cmake_cmd)

def build_jerry(arguments):
    return subprocess.call(['make', '--no-print-directory','-j', str(arguments.jobs), '-C', BUILD_DIR])

def print_result(ret):
    print('=' * 30)
    if ret:
        print('Build failed with exit code: %s' % (ret))
    else:
        print('Build succeeded!')
    print('=' * 30)

def main():
    arguments = get_arguments()
    ret = configure_build(arguments)

    if not ret:
        ret = build_jerry(arguments)

    print_result(ret)
    sys.exit(ret)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from typing import Optional

trace_icon      = ":fontawesome-solid-route:"
debug_icon      = ":fontawesome-solid-bug:"
info_icon       = ":fontawesome-solid-circle-info:"
warning_icon    = ":fontawesome-solid-triangle-exclamation:"
error_icon      = ":fontawesome-solid-circle-exclamation:"
stop_icon       = ":fontawesome-solid-circle-xmark:"
true_icon       = ":fontawesome-solid-square-check:"
false_icon      = ":fontawesome-solid-circle-times:"

windows_icon    = ":fontawesome-brands-windows:"
macos_icon      = ":fontawesome-brands-apple:"
linux_icon      = ":fontawesome-brands-linux:"

artifacts_url_base = 'https://artifacts.metaborg.org'
artifacts_releases_url = f'{artifacts_url_base}/content/repositories/releases/org/metaborg/'

def artifacts_download(repo, artifact, classifier=None, packaging="jar", version="LATEST", group="org.metaborg"):
    return f'{artifacts_url_base}/service/local/artifact/maven/redirect?r={repo}&g={group}' \
           f'&a={artifact}{f"&c={classifier}" if classifier is not None else ""}&p={packaging}&v={version}'

def eclipse_lwb_artifacts_download(repo: str, variant: str, packaging: str, version: str):
    return artifacts_download(repo, "org.metaborg.spoofax.eclipse.dist", variant, packaging, version)

buildfarm_url_base = 'https://buildfarm.metaborg.org'

def buildfarm_download(path: str):
    return f'{buildfarm_url_base}/job/metaborg/job/spoofax-releng/job/master/lastSuccessfulBuild/artifact/{path}'

def eclipse_lwb_buildfarm_download(variant: str, packaging: str):
    return buildfarm_download(f'dist/spoofax/eclipse/spoofax-{variant}.{packaging}')

macos_64_jre_variant = 'macosx-x64-jre'
linux_64_jre_variant = 'linux-x64-jre'
win_64_jre_variant = 'windows-x64-jre'
win_32_jre_variant = "windows-x86-jre"
macos_64_variant = 'macosx-x64'
linux_64_variant = "linux-x64"
win_64_variant = 'windows-x64'
win_32_variant = 'windows-x86'

macos_archive_ext = 'tar.gz'
linux_archive_ext = 'tar.gz'
win_archive_ext = 'zip'

def download_link(icon: str, name: str, link: str):
    return f"{icon} [{name}]({link})"

def fill_vars_with_release(vars, env_version: str, version: str, download_version: str, date: Optional[str]):
    repo = 'snapshots' if 'SNAPSHOT' in version else 'releases'

    if 'SNAPSHOT' in version:
        macos_64_jre = eclipse_lwb_buildfarm_download(macos_64_jre_variant, macos_archive_ext)
        linux_64_jre = eclipse_lwb_buildfarm_download(linux_64_jre_variant, linux_archive_ext)
        windows_64_jre = eclipse_lwb_buildfarm_download(win_64_jre_variant, win_archive_ext)
        windows_32_jre = eclipse_lwb_buildfarm_download(win_32_jre_variant, win_archive_ext)
        macos_64 = eclipse_lwb_buildfarm_download(macos_64_variant, macos_archive_ext)
        linux_64 = eclipse_lwb_buildfarm_download(linux_64_variant, linux_archive_ext)
        windows_64 = eclipse_lwb_buildfarm_download(win_64_variant, win_archive_ext)
        windows_32 = eclipse_lwb_buildfarm_download(win_32_variant, win_archive_ext)
        eclipse_repo = buildfarm_download('dist/spoofax/eclipse/site/')
        eclipse_repo_archive = None
        intellij_update_site = buildfarm_download('dist/spoofax/intellij/plugin.zip')
        sunshine_jar = None
        spt_testrunner_jar = None
        spoofax_core_uber_jar = None
        strategoxt_distrib = buildfarm_download('dist/strategoxt/distrib.tar')
        strategoxt_jar = buildfarm_download('dist/strategoxt/strategoxt.jar')
    else:
        macos_64_jre = eclipse_lwb_artifacts_download(repo, macos_64_jre_variant, macos_archive_ext, download_version)
        linux_64_jre = eclipse_lwb_artifacts_download(repo, linux_64_jre_variant, linux_archive_ext, download_version)
        windows_64_jre = eclipse_lwb_artifacts_download(repo, win_64_jre_variant, win_archive_ext, download_version)
        windows_32_jre = eclipse_lwb_artifacts_download(repo, win_32_jre_variant, win_archive_ext, download_version)
        macos_64 = eclipse_lwb_artifacts_download(repo, macos_64_variant, macos_archive_ext, download_version)
        linux_64 = eclipse_lwb_artifacts_download(repo, linux_64_variant, linux_archive_ext, download_version)
        windows_64 = eclipse_lwb_artifacts_download(repo, win_64_variant, win_archive_ext, download_version)
        windows_32 = eclipse_lwb_artifacts_download(repo, win_32_variant, win_archive_ext, download_version)
        eclipse_repo = f'https://artifacts.metaborg.org/content/unzip/releases-unzipped/org/metaborg/org.metaborg.spoofax.eclipse.updatesite/{version}/org.metaborg.spoofax.eclipse.updatesite-{version}-assembly.zip-unzip/'
        eclipse_repo_archive = artifacts_download(repo, 'org.metaborg.spoofax.eclipse.updatesite',
                                                  classifier='assembly', packaging='zip', version=download_version)
        intellij_update_site = artifacts_download(repo, 'org.metaborg.intellij.dist', packaging='zip',
                                                  version=download_version)
        sunshine_jar = artifacts_download(repo, 'org.metaborg.sunshine2', version=download_version)
        spt_testrunner_jar = artifacts_download(repo, 'org.metaborg.spt.cmd', version=download_version)
        spoofax_core_uber_jar = artifacts_download(repo, 'org.metaborg.spoofax.core.uber', version=download_version)
        strategoxt_distrib = artifacts_download(repo, 'strategoxt-distrib', classifier='bin', packaging='tar',
                                                version=download_version)
        strategoxt_jar = artifacts_download(repo, 'strategoxt-jar', version=download_version)

    vars.release[env_version] = dict(
        date=date,
        version=version,
        eclipse=dict(
            install=dict(
                jvm=dict(
                    link=dict(
                        macos_64=download_link(macos_icon, "macOS 64-bit with embedded JVM", macos_64_jre),
                        linux_64=download_link(linux_icon, "Linux 64-bit with embedded JVM", linux_64_jre),
                        windows_64=download_link(windows_icon, "Windows 64-bit with embedded JVM", windows_64_jre),
                        windows_32=download_link(windows_icon, "Windows 32-bit with embedded JVM", windows_32_jre),
                    ),
                    macos_64=macos_64_jre,
                    linux_64=linux_64_jre,
                    windows_64=windows_64_jre,
                    windows_32=windows_32_jre,
                ), link=dict(
                    macos_64=download_link(macos_icon, "macOS 64-bit", macos_64_jre),
                    linux_64=download_link(linux_icon, "Linux 64-bit", linux_64_jre),
                    windows_64=download_link(windows_icon, "Windows 64-bit", windows_64_jre),
                    windows_32=download_link(windows_icon, "Windows 32-bit", windows_32_jre),
                ),
                macos_64=macos_64,
                linux_64=linux_64,
                windows_64=windows_64,
                windows_32=windows_32,
            ),
            repository=eclipse_repo,
            repository_archive=eclipse_repo_archive,
            repository_archive_link=f'[Eclipse update site archive]({eclipse_repo_archive})'
        ),
        intellij=dict(
            update_site=intellij_update_site,
            update_site_link=f'[IntelliJ update site archive]({intellij_update_site})'
        ),
        sunshine_jar=sunshine_jar,
        sunshine_jar_link=f'[Sunshine JAR]({sunshine_jar})',
        spt_testrunner_jar=spt_testrunner_jar,
        spt_testrunner_jar_link=f'[SPT testrunner JAR]({spt_testrunner_jar})',
        spoofax_core_uber_jar=spoofax_core_uber_jar,
        spoofax_core_uber_jar_link=f'[Spoofax Core Uber JAR]({spoofax_core_uber_jar})',
        spoofax_core_uber_jar_artifact=f'org.metaborg:org.metaborg.spoofax.core.uber:{download_version}',
        strategoxt_distrib=strategoxt_distrib,
        strategoxt_distrib_link=f'[Stratego/XT distribution]({strategoxt_distrib})',
        strategoxt_jar=strategoxt_jar,
        strategoxt_jar_link=f'[Stratego/XT JAR]({strategoxt_jar})',
    )

release_versions = {
    '2.5.17': '06-07-2022',
    '2.5.16': '04-06-2021',
    '2.5.15': '11-05-2021',
    '2.5.14': '16-12-2020',
    '2.5.13': '20-11-2020',
    '2.5.12': '08-10-2020',
    '2.5.11': '17-07-2020',
    '2.5.10': '07-07-2020',
    '2.5.9': '08-05-2020',
    '2.5.8': '28-04-2020',
    '2.5.7': '26-06-2019',
    '2.5.6': '24-05-2019',
    '2.5.5': '23-05-2019',
    '2.5.4': '08-05-2019',
    '2.5.3': '02-05-2019',
    '2.5.2': '12-03-2019',
    '2.5.1': '02-10-2018',
    '2.5.0': '11-09-2018',
    '2.4.1': '29-01-2018',
    '2.4.0': '09-01-2018',
    '2.3.0': '29-09-2017',
    '2.2.1': '04-05-2017',
    '2.2.0': '18-04-2017',
    '2.1.0': '10-01-2017',
    '2.0.0': '08-07-2016',
    '2.0.0-beta1': '07-04-2016',
    '1.5.0': '18-12-2015',
    '1.4.0': '06-03-2015',
    '1.3.1': '09-12-2014',
    '1.3.0': '12-11-2014',
    '1.2.0': '13-08-2014',
    '1.1.0': '25-03-2013',
    '1.0.2': '15-02-2012',
    '1.0.0': '28-12-2011',
}
development_version = '2.6.0-SNAPSHOT'

def on_pre_page_macros(env):
    define_macros(env.conf['extra'])

def define_env(env):
    define_macros(env.variables)

def define_macros(vars):
    vars['trace'] = f"{trace_icon}{{.trace}}"
    vars['debug'] = f"{debug_icon}{{.debug}}"
    vars['info'] = f"{info_icon}{{.info}}"
    vars['warning'] = f"{warning_icon}{{.warning}}"
    vars['error'] = f"{error_icon}{{.error}}"
    vars['stop'] = f"{stop_icon}{{.stop}}"

    # NOTE: `true` and `false` are reserved keywords
    vars['yes'] = f"{true_icon}{{.true}}"
    vars['no'] = f"{false_icon}{{.false}}"

    vars.os = dict(
        windows=f'{windows_icon} Windows',
        linux=f'{linux_icon} Linux',
        macos=f'{macos_icon} macOS'
    )
    vars.artifacts_releases = artifacts_releases_url
    vars.artifacts_releases_link = f'[artifact server]({artifacts_releases_url})'

    vars.release = {}
    for version, date in release_versions.items():
        fill_vars_with_release(vars, version, version, version, date)
    latest_rel_version, latest_rel_date = next(iter(release_versions.items()))
    fill_vars_with_release(vars, 'rel', latest_rel_version, latest_rel_version, latest_rel_date)
    fill_vars_with_release(vars, 'dev', development_version, "LATEST", None)

<commit_msg>Fix release notes not linking to non-JRE versions<commit_after>from typing import Optional

trace_icon      = ":fontawesome-solid-route:"
debug_icon      = ":fontawesome-solid-bug:"
info_icon       = ":fontawesome-solid-circle-info:"
warning_icon    = ":fontawesome-solid-triangle-exclamation:"
error_icon      = ":fontawesome-solid-circle-exclamation:"
stop_icon       = ":fontawesome-solid-circle-xmark:"
true_icon       = ":fontawesome-solid-square-check:"
false_icon      = ":fontawesome-solid-circle-times:"

windows_icon    = ":fontawesome-brands-windows:"
macos_icon      = ":fontawesome-brands-apple:"
linux_icon      = ":fontawesome-brands-linux:"

artifacts_url_base = 'https://artifacts.metaborg.org'
artifacts_releases_url = f'{artifacts_url_base}/content/repositories/releases/org/metaborg/'

def artifacts_download(repo, artifact, classifier=None, packaging="jar", version="LATEST", group="org.metaborg"):
    return f'{artifacts_url_base}/service/local/artifact/maven/redirect?r={repo}&g={group}' \
           f'&a={artifact}{f"&c={classifier}" if classifier is not None else ""}&p={packaging}&v={version}'

def eclipse_lwb_artifacts_download(repo: str, variant: str, packaging: str, version: str):
    return artifacts_download(repo, "org.metaborg.spoofax.eclipse.dist", variant, packaging, version)

buildfarm_url_base = 'https://buildfarm.metaborg.org'

def buildfarm_download(path: str):
    return f'{buildfarm_url_base}/job/metaborg/job/spoofax-releng/job/master/lastSuccessfulBuild/artifact/{path}'

def eclipse_lwb_buildfarm_download(variant: str, packaging: str):
    return buildfarm_download(f'dist/spoofax/eclipse/spoofax-{variant}.{packaging}')

macos_64_jre_variant = 'macosx-x64-jre'
linux_64_jre_variant = 'linux-x64-jre'
win_64_jre_variant = 'windows-x64-jre'
win_32_jre_variant = "windows-x86-jre"
macos_64_variant = 'macosx-x64'
linux_64_variant = "linux-x64"
win_64_variant = 'windows-x64'
win_32_variant = 'windows-x86'

macos_archive_ext = 'tar.gz'
linux_archive_ext = 'tar.gz'
win_archive_ext = 'zip'

def download_link(icon: str, name: str, link: str):
    return f"{icon} [{name}]({link})"

def fill_vars_with_release(vars, env_version: str, version: str, download_version: str, date: Optional[str]):
    repo = 'snapshots' if 'SNAPSHOT' in version else 'releases'

    if 'SNAPSHOT' in version:
        macos_64_jre = eclipse_lwb_buildfarm_download(macos_64_jre_variant, macos_archive_ext)
        linux_64_jre = eclipse_lwb_buildfarm_download(linux_64_jre_variant, linux_archive_ext)
        windows_64_jre = eclipse_lwb_buildfarm_download(win_64_jre_variant, win_archive_ext)
        windows_32_jre = eclipse_lwb_buildfarm_download(win_32_jre_variant, win_archive_ext)
        macos_64 = eclipse_lwb_buildfarm_download(macos_64_variant, macos_archive_ext)
        linux_64 = eclipse_lwb_buildfarm_download(linux_64_variant, linux_archive_ext)
        windows_64 = eclipse_lwb_buildfarm_download(win_64_variant, win_archive_ext)
        windows_32 = eclipse_lwb_buildfarm_download(win_32_variant, win_archive_ext)
        eclipse_repo = buildfarm_download('dist/spoofax/eclipse/site/')
        eclipse_repo_archive = None
        intellij_update_site = buildfarm_download('dist/spoofax/intellij/plugin.zip')
        sunshine_jar = None
        spt_testrunner_jar = None
        spoofax_core_uber_jar = None
        strategoxt_distrib = buildfarm_download('dist/strategoxt/distrib.tar')
        strategoxt_jar = buildfarm_download('dist/strategoxt/strategoxt.jar')
    else:
        macos_64_jre = eclipse_lwb_artifacts_download(repo, macos_64_jre_variant, macos_archive_ext, download_version)
        linux_64_jre = eclipse_lwb_artifacts_download(repo, linux_64_jre_variant, linux_archive_ext, download_version)
        windows_64_jre = eclipse_lwb_artifacts_download(repo, win_64_jre_variant, win_archive_ext, download_version)
        windows_32_jre = eclipse_lwb_artifacts_download(repo, win_32_jre_variant, win_archive_ext, download_version)
        macos_64 = eclipse_lwb_artifacts_download(repo, macos_64_variant, macos_archive_ext, download_version)
        linux_64 = eclipse_lwb_artifacts_download(repo, linux_64_variant, linux_archive_ext, download_version)
        windows_64 = eclipse_lwb_artifacts_download(repo, win_64_variant, win_archive_ext, download_version)
        windows_32 = eclipse_lwb_artifacts_download(repo, win_32_variant, win_archive_ext, download_version)
        eclipse_repo = f'https://artifacts.metaborg.org/content/unzip/releases-unzipped/org/metaborg/org.metaborg.spoofax.eclipse.updatesite/{version}/org.metaborg.spoofax.eclipse.updatesite-{version}-assembly.zip-unzip/'
        eclipse_repo_archive = artifacts_download(repo, 'org.metaborg.spoofax.eclipse.updatesite',
                                                  classifier='assembly', packaging='zip', version=download_version)
        intellij_update_site = artifacts_download(repo, 'org.metaborg.intellij.dist', packaging='zip',
                                                  version=download_version)
        sunshine_jar = artifacts_download(repo, 'org.metaborg.sunshine2', version=download_version)
        spt_testrunner_jar = artifacts_download(repo, 'org.metaborg.spt.cmd', version=download_version)
        spoofax_core_uber_jar = artifacts_download(repo, 'org.metaborg.spoofax.core.uber', version=download_version)
        strategoxt_distrib = artifacts_download(repo, 'strategoxt-distrib', classifier='bin', packaging='tar',
                                                version=download_version)
        strategoxt_jar = artifacts_download(repo, 'strategoxt-jar', version=download_version)

    vars.release[env_version] = dict(
        date=date,
        version=version,
        eclipse=dict(
            install=dict(
                jvm=dict(
                    link=dict(
                        macos_64=download_link(macos_icon, "macOS 64-bit with embedded JVM", macos_64_jre),
                        linux_64=download_link(linux_icon, "Linux 64-bit with embedded JVM", linux_64_jre),
                        windows_64=download_link(windows_icon, "Windows 64-bit with embedded JVM", windows_64_jre),
                        windows_32=download_link(windows_icon, "Windows 32-bit with embedded JVM", windows_32_jre),
                    ),
                    macos_64=macos_64_jre,
                    linux_64=linux_64_jre,
                    windows_64=windows_64_jre,
                    windows_32=windows_32_jre,
                ), link=dict(
                    macos_64=download_link(macos_icon, "macOS 64-bit", macos_64),
                    linux_64=download_link(linux_icon, "Linux 64-bit", linux_64),
                    windows_64=download_link(windows_icon, "Windows 64-bit", windows_64),
                    windows_32=download_link(windows_icon, "Windows 32-bit", windows_32),
                ),
                macos_64=macos_64,
                linux_64=linux_64,
                windows_64=windows_64,
                windows_32=windows_32,
            ),
            repository=eclipse_repo,
            repository_archive=eclipse_repo_archive,
            repository_archive_link=f'[Eclipse update site archive]({eclipse_repo_archive})'
        ),
        intellij=dict(
            update_site=intellij_update_site,
            update_site_link=f'[IntelliJ update site archive]({intellij_update_site})'
        ),
        sunshine_jar=sunshine_jar,
        sunshine_jar_link=f'[Sunshine JAR]({sunshine_jar})',
        spt_testrunner_jar=spt_testrunner_jar,
        spt_testrunner_jar_link=f'[SPT testrunner JAR]({spt_testrunner_jar})',
        spoofax_core_uber_jar=spoofax_core_uber_jar,
        spoofax_core_uber_jar_link=f'[Spoofax Core Uber JAR]({spoofax_core_uber_jar})',
        spoofax_core_uber_jar_artifact=f'org.metaborg:org.metaborg.spoofax.core.uber:{download_version}',
        strategoxt_distrib=strategoxt_distrib,
        strategoxt_distrib_link=f'[Stratego/XT distribution]({strategoxt_distrib})',
        strategoxt_jar=strategoxt_jar,
        strategoxt_jar_link=f'[Stratego/XT JAR]({strategoxt_jar})',
    )

release_versions = {
    '2.5.17': '06-07-2022',
    '2.5.16': '04-06-2021',
    '2.5.15': '11-05-2021',
    '2.5.14': '16-12-2020',
    '2.5.13': '20-11-2020',
    '2.5.12': '08-10-2020',
    '2.5.11': '17-07-2020',
    '2.5.10': '07-07-2020',
    '2.5.9': '08-05-2020',
    '2.5.8': '28-04-2020',
    '2.5.7': '26-06-2019',
    '2.5.6': '24-05-2019',
    '2.5.5': '23-05-2019',
    '2.5.4': '08-05-2019',
    '2.5.3': '02-05-2019',
    '2.5.2': '12-03-2019',
    '2.5.1': '02-10-2018',
    '2.5.0': '11-09-2018',
    '2.4.1': '29-01-2018',
    '2.4.0': '09-01-2018',
    '2.3.0': '29-09-2017',
    '2.2.1': '04-05-2017',
    '2.2.0': '18-04-2017',
    '2.1.0': '10-01-2017',
    '2.0.0': '08-07-2016',
    '2.0.0-beta1': '07-04-2016',
    '1.5.0': '18-12-2015',
    '1.4.0': '06-03-2015',
    '1.3.1': '09-12-2014',
    '1.3.0': '12-11-2014',
    '1.2.0': '13-08-2014',
    '1.1.0': '25-03-2013',
    '1.0.2': '15-02-2012',
    '1.0.0': '28-12-2011',
}
development_version = '2.6.0-SNAPSHOT'

def on_pre_page_macros(env):
    define_macros(env.conf['extra'])

def define_env(env):
    define_macros(env.variables)

def define_macros(vars):
    vars['trace'] = f"{trace_icon}{{.trace}}"
    vars['debug'] = f"{debug_icon}{{.debug}}"
    vars['info'] = f"{info_icon}{{.info}}"
    vars['warning'] = f"{warning_icon}{{.warning}}"
    vars['error'] = f"{error_icon}{{.error}}"
    vars['stop'] = f"{stop_icon}{{.stop}}"

    # NOTE: `true` and `false` are reserved keywords
    vars['yes'] = f"{true_icon}{{.true}}"
    vars['no'] = f"{false_icon}{{.false}}"

    vars.os = dict(
        windows=f'{windows_icon} Windows',
        linux=f'{linux_icon} Linux',
        macos=f'{macos_icon} macOS'
    )
    vars.artifacts_releases = artifacts_releases_url
    vars.artifacts_releases_link = f'[artifact server]({artifacts_releases_url})'

    vars.release = {}
    for version, date in release_versions.items():
        fill_vars_with_release(vars, version, version, version, date)
    latest_rel_version, latest_rel_date = next(iter(release_versions.items()))
    fill_vars_with_release(vars, 'rel', latest_rel_version, latest_rel_version, latest_rel_date)
    fill_vars_with_release(vars, 'dev', development_version, "LATEST", None)

<|endoftext|>


<commit_before>from copy import deepcopy
from slm_lab.lib import logger, util
from slm_lab.spec import spec_util
import numpy as np
import pydash as ps
import random
import ray
import ray.tune as tune
import torch

logger = logger.get_logger(__name__)

def build_config_space(spec):
    '''
    Build ray config space from flattened spec.search
    Specify a config space in spec using `"{key}__{space_type}": {v}`.
    Where `{space_type}` is `grid_search` of `ray.tune`, or any function name of `np.random`:
    - `grid_search`: str/int/float. v = list of choices
    - `choice`: str/int/float. v = list of choices
    - `randint`: int. v = [low, high)
    - `uniform`: float. v = [low, high)
    - `normal`: float. v = [mean, stdev)

    For example:
    - `"explore_anneal_epi__randint": [10, 60],` will sample integers uniformly from 10 to 60 for `explore_anneal_epi`,
    - `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using `np.random.uniform(0.001, 0.1)`

    If any key uses `grid_search`, it will be combined exhaustively in combination with other random sampling.
    '''
    space_types = ('grid_search', 'choice', 'randint', 'uniform', 'normal')
    config_space = {}
    for k, v in util.flatten_dict(spec['search']).items():
        key, space_type = k.split('__')
        assert space_type in space_types, f'Please specify your search variable as {key}__<space_type> in one of {space_types}'
        if space_type == 'grid_search':
            config_space[key] = tune.grid_search(v)
        elif space_type == 'choice':
            config_space[key] = tune.sample_from(lambda spec, v=v: random.choice(v))
        else:
            np_fn = getattr(np.random, space_type)
            config_space[key] = tune.sample_from(lambda spec, v=v: np_fn(*v))
    return config_space

def infer_trial_resources(spec):
    '''Infer the resources_per_trial for ray from spec'''
    meta_spec = spec['meta']
    num_cpus = min(util.NUM_CPUS, meta_spec['max_session'])

    use_gpu = any(agent_spec['net'].get('gpu') for agent_spec in spec['agent'])
    requested_gpu = meta_spec['max_session'] if use_gpu else 0
    gpu_count = torch.cuda.device_count() if torch.cuda.is_available() else 0
    num_gpus = min(gpu_count, requested_gpu)
    resources_per_trial = {'cpu': num_cpus, 'gpu': num_gpus}
    return resources_per_trial

def inject_config(spec, config):
    '''Inject flattened config into SLM Lab spec.'''
    spec = deepcopy(spec)
    spec.pop('search', None)
    for k, v in config.items():
        ps.set_(spec, k, v)
    return spec

def ray_trainable(config, reporter):
    '''
    Create an instance of a trainable function for ray: https://ray.readthedocs.io/en/latest/tune-usage.html#training-api
    Lab needs a spec and a trial_index to be carried through config, pass them with config in ray.run() like so:
    config = {
        'spec': spec,
        'trial_index': tune.sample_from(lambda spec: gen_trial_index()),
        ... # normal ray config with sample, grid search etc.
    }
    '''
    from slm_lab.experiment.control import Trial
    import os
    # restore data carried from ray.run() config
    spec = config.pop('spec')
    spec = inject_config(spec, config)
    # tick trial_index with proper offset
    trial_index = config.pop('trial_index')
    spec['meta']['trial'] = trial_index - 1
    spec_util.tick(spec, 'trial')
    # run SLM Lab trial
    os.environ.pop('CUDA_VISIBLE_DEVICES', None)  # remove CUDA id restriction from ray
    metrics = Trial(spec).run()
    metrics.update(config)  # carry config for analysis too
    # ray report to carry data in ray trial.last_result
    reporter(trial_data={trial_index: metrics})

def run_ray_search(spec):
    '''
    Method to run ray search from experiment. Uses RandomSearch now.
    TODO support for other ray search algorithms: https://ray.readthedocs.io/en/latest/tune-searchalg.html
    '''
    logger.info(f'Running ray search for spec {spec["name"]}')
    # generate trial index to pass into Lab Trial
    global trial_index  # make gen_trial_index passable into ray.run
    trial_index = -1

    def gen_trial_index():
        global trial_index
        trial_index += 1
        return trial_index

    ray.init()

    ray_trials = tune.run(
        ray_trainable,
        name=spec['name'],
        config={
            'spec': spec,
            'trial_index': tune.sample_from(lambda spec: gen_trial_index()),
            **build_config_space(spec)
        },
        resources_per_trial=infer_trial_resources(spec),
        num_samples=spec['meta']['max_trial'],
        reuse_actors=True,
    )
    trial_data_dict = {}  # data for Lab Experiment to analyze
    for ray_trial in ray_trials:
        ray_trial_data = ray_trial.last_result['trial_data']
        trial_data_dict.update(ray_trial_data)

    ray.shutdown()
    return trial_data_dict

def run_param_specs(param_specs):
    '''Run the given param_specs in parallel trials using ray. Used for benchmarking.'''
    ray.init()
    ray_trials = tune.run(
        ray_trainable,
        name='param_specs',
        config={
            'spec': tune.grid_search(param_specs),
            'trial_index': 0,
        },
        resources_per_trial=infer_trial_resources(param_specs[0]),
        num_samples=1,
        reuse_actors=True,
    )
    ray.shutdown()
<commit_msg>dont reuse actor<commit_after>from copy import deepcopy
from slm_lab.lib import logger, util
from slm_lab.spec import spec_util
import numpy as np
import pydash as ps
import random
import ray
import ray.tune as tune
import torch

logger = logger.get_logger(__name__)

def build_config_space(spec):
    '''
    Build ray config space from flattened spec.search
    Specify a config space in spec using `"{key}__{space_type}": {v}`.
    Where `{space_type}` is `grid_search` of `ray.tune`, or any function name of `np.random`:
    - `grid_search`: str/int/float. v = list of choices
    - `choice`: str/int/float. v = list of choices
    - `randint`: int. v = [low, high)
    - `uniform`: float. v = [low, high)
    - `normal`: float. v = [mean, stdev)

    For example:
    - `"explore_anneal_epi__randint": [10, 60],` will sample integers uniformly from 10 to 60 for `explore_anneal_epi`,
    - `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using `np.random.uniform(0.001, 0.1)`

    If any key uses `grid_search`, it will be combined exhaustively in combination with other random sampling.
    '''
    space_types = ('grid_search', 'choice', 'randint', 'uniform', 'normal')
    config_space = {}
    for k, v in util.flatten_dict(spec['search']).items():
        key, space_type = k.split('__')
        assert space_type in space_types, f'Please specify your search variable as {key}__<space_type> in one of {space_types}'
        if space_type == 'grid_search':
            config_space[key] = tune.grid_search(v)
        elif space_type == 'choice':
            config_space[key] = tune.sample_from(lambda spec, v=v: random.choice(v))
        else:
            np_fn = getattr(np.random, space_type)
            config_space[key] = tune.sample_from(lambda spec, v=v: np_fn(*v))
    return config_space

def infer_trial_resources(spec):
    '''Infer the resources_per_trial for ray from spec'''
    meta_spec = spec['meta']
    num_cpus = min(util.NUM_CPUS, meta_spec['max_session'])

    use_gpu = any(agent_spec['net'].get('gpu') for agent_spec in spec['agent'])
    requested_gpu = meta_spec['max_session'] if use_gpu else 0
    gpu_count = torch.cuda.device_count() if torch.cuda.is_available() else 0
    num_gpus = min(gpu_count, requested_gpu)
    resources_per_trial = {'cpu': num_cpus, 'gpu': num_gpus}
    return resources_per_trial

def inject_config(spec, config):
    '''Inject flattened config into SLM Lab spec.'''
    spec = deepcopy(spec)
    spec.pop('search', None)
    for k, v in config.items():
        ps.set_(spec, k, v)
    return spec

def ray_trainable(config, reporter):
    '''
    Create an instance of a trainable function for ray: https://ray.readthedocs.io/en/latest/tune-usage.html#training-api
    Lab needs a spec and a trial_index to be carried through config, pass them with config in ray.run() like so:
    config = {
        'spec': spec,
        'trial_index': tune.sample_from(lambda spec: gen_trial_index()),
        ... # normal ray config with sample, grid search etc.
    }
    '''
    from slm_lab.experiment.control import Trial
    import os
    # restore data carried from ray.run() config
    spec = config.pop('spec')
    spec = inject_config(spec, config)
    # tick trial_index with proper offset
    trial_index = config.pop('trial_index')
    spec['meta']['trial'] = trial_index - 1
    spec_util.tick(spec, 'trial')
    # run SLM Lab trial
    os.environ.pop('CUDA_VISIBLE_DEVICES', None)  # remove CUDA id restriction from ray
    metrics = Trial(spec).run()
    metrics.update(config)  # carry config for analysis too
    # ray report to carry data in ray trial.last_result
    reporter(trial_data={trial_index: metrics})

def run_ray_search(spec):
    '''
    Method to run ray search from experiment. Uses RandomSearch now.
    TODO support for other ray search algorithms: https://ray.readthedocs.io/en/latest/tune-searchalg.html
    '''
    logger.info(f'Running ray search for spec {spec["name"]}')
    # generate trial index to pass into Lab Trial
    global trial_index  # make gen_trial_index passable into ray.run
    trial_index = -1

    def gen_trial_index():
        global trial_index
        trial_index += 1
        return trial_index

    ray.init()

    ray_trials = tune.run(
        ray_trainable,
        name=spec['name'],
        config={
            'spec': spec,
            'trial_index': tune.sample_from(lambda spec: gen_trial_index()),
            **build_config_space(spec)
        },
        resources_per_trial=infer_trial_resources(spec),
        num_samples=spec['meta']['max_trial'],
        reuse_actors=False,
    )
    trial_data_dict = {}  # data for Lab Experiment to analyze
    for ray_trial in ray_trials:
        ray_trial_data = ray_trial.last_result['trial_data']
        trial_data_dict.update(ray_trial_data)

    ray.shutdown()
    return trial_data_dict

def run_param_specs(param_specs):
    '''Run the given param_specs in parallel trials using ray. Used for benchmarking.'''
    ray.init()
    ray_trials = tune.run(
        ray_trainable,
        name='param_specs',
        config={
            'spec': tune.grid_search(param_specs),
            'trial_index': 0,
        },
        resources_per_trial=infer_trial_resources(param_specs[0]),
        num_samples=1,
        reuse_actors=False,
    )
    ray.shutdown()
<|endoftext|>


<commit_before>from __future__ import print_function
from collections import defaultdict
from functools import partial
import gc, inspect, os, sys

import numpy as np
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

import h2o
from h2o.sklearn.wrapper import H2OConnectionMonitorMixin

sys.path.insert(1, os.path.join("..",".."))
from tests import pyunit_utils, Namespace as ns

"""
This test suite creates a default sklearn classification estimator for each H2O estimator.
Then, it feeds them with H2O frames (more efficient and ensures compatibility with old API.)
or with numpy arrays to provide the simplest approach for users wanting to use H2O like any sklearn estimator.
"""

seed = 2019
init_connection_args = dict(strict_version_check=False, show_progress=True)

scores = defaultdict(dict)

def _get_data(format='numpy', n_classes=2):
    X, y = make_classification(n_samples=100, n_features=10, n_informative=5, n_classes=n_classes, random_state=seed)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=seed)
    data = ns(X_train=X_train, X_test=X_test, y_train=y_train, y_test=y_test)
    if format == 'h2o':
        for k, v in data.__dict__.items():
            setattr(data, k, h2o.H2OFrame(v))
    return data

def _get_default_args(estimator_cls):
    defaults = dict(
        H2OCoxProportionalHazardsClassifier=dict(),
        H2ODeepLearningClassifier=dict(seed=seed, reproducible=True),
        H2OGeneralizedAdditiveClassifier=dict(family='binomial', seed=seed, gam_columns=["C1"]),
        H2OGeneralizedLinearClassifier=dict(family='binomial', seed=seed),
    )
    return defaults.get(estimator_cls.__name__, dict(seed=seed))

def _get_custom_behaviour(estimator_cls):
    custom = dict(
        # H2ODeepLearningClassifier=dict(scores_may_differ=True),
    )
    return custom.get(estimator_cls.__name__, dict())

def test_estimator_with_h2o_frames(estimator_cls):
    args = _get_default_args(estimator_cls)
    estimator = estimator_cls(**args)

    data = _get_data(format='h2o', n_classes=2)
    assert isinstance(data.X_train, h2o.H2OFrame)
    estimator.fit(data.X_train, data.y_train)
    preds = estimator.predict(data.X_test)
    print(preds)
    assert isinstance(preds, h2o.H2OFrame)
    if _get_custom_behaviour(estimator_cls).get('preds_as_vector', True):
        assert preds.dim == [len(data.X_test), 1], "got {}".format(preds.dim)
    else:
        assert preds.dim[0] == len(data.X_test)

    probs = estimator.predict_proba(data.X_test)
    print(probs)
    assert probs.dim == [len(data.X_test), 2], "got {}".format(probs.dim)
    assert np.allclose(np.sum(probs.as_data_frame().values, axis=1), 1.), "`predict_proba` didn't return probabilities"

    score = estimator.score(data.X_test, data.y_test)
    assert isinstance(score, float)
    skl_score = accuracy_score(data.y_test.as_data_frame().values, preds.as_data_frame().values)
    assert abs(score - skl_score) < 1e-6, "score={}, skl_score={}".format(score, skl_score)
    scores[estimator_cls].update(with_h2o_frames=score)

def test_estimator_with_numpy_arrays(estimator_cls):
    estimator = estimator_cls(init_connection_args=init_connection_args, **_get_default_args(estimator_cls))

    data = _get_data(format='numpy', n_classes=2)
    assert isinstance(data.X_train, np.ndarray)

    with estimator:
        estimator.fit(data.X_train, data.y_train)
        preds = estimator.predict(data.X_test)
        print(preds)
        assert isinstance(preds, np.ndarray)
        if _get_custom_behaviour(estimator_cls).get('preds_as_vector', True):
            assert preds.shape == (len(data.X_test),), "got {}".format(preds.shape)
        else:
            assert preds.shape[0] == len(data.X_test)

        probs = estimator.predict_proba(data.X_test)
        print(probs)
        assert probs.shape == (len(data.X_test), 2)
        assert np.allclose(np.sum(probs, axis=1), 1.), "`predict_proba` didn't return probabilities"

        score = estimator.score(data.X_test, data.y_test)
        assert isinstance(score, float)
        skl_score = accuracy_score(data.y_test, preds)
        assert abs(score - skl_score) < 1e-6
        scores[estimator_cls].update(with_numpy_arrays=score)

def test_scores_are_equivalent(estimator_cls):
    try:
        lk, rk = ('with_h2o_frames', 'with_numpy_arrays')
        est_scores = scores[estimator_cls]
        if lk in est_scores and rk in est_scores:
            assert abs(est_scores[lk] - abs(est_scores[rk])) < 1e-6, \
                "expected equivalent scores but got {lk}={lscore} and {rk}={rscore}" \
                    .format(lk=lk, rk=rk, lscore=est_scores[lk], rscore=est_scores[rk])
        elif lk not in est_scores:
            print("no scores for {}".format(estimator_cls.__name__+' '+lk))
        else:
            print("no scores for {}".format(estimator_cls.__name__+' '+rk))
    except AssertionError as e:
        if _get_custom_behaviour(estimator_cls).get('scores_may_differ', False):
            print("ERROR !!! "+str(e))
        else:
            raise e

def make_test(test, classifier):
    bound_test = partial(test, classifier)
    bound_test.__name__ = test.__name__
    pyunit_utils.tag_test(bound_test, classifier.__name__)
    return bound_test

def make_tests(classifier):
    return list(map(lambda test: make_test(test, classifier), [
        test_estimator_with_h2o_frames,
        test_estimator_with_numpy_arrays,
        test_scores_are_equivalent
    ]))

failing = [
    'H2OStackedEnsembleClassifier', 'H2OUpliftRandomForestClassifier'  # needs a separate test (requires models as parameters)
]
classifiers = [cls for name, cls in inspect.getmembers(h2o.sklearn, inspect.isclass)
               if name.endswith('Classifier') and name not in ['H2OAutoMLClassifier']+failing]
pyunit_utils.run_tests([make_tests(c) for c in classifiers])
<commit_msg>Adding a single argument to fit function call to test that the fix is working<commit_after>from __future__ import print_function
from collections import defaultdict
from functools import partial
import gc, inspect, os, sys

import numpy as np
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

import h2o
from h2o.sklearn.wrapper import H2OConnectionMonitorMixin

sys.path.insert(1, os.path.join("..",".."))
from tests import pyunit_utils, Namespace as ns

"""
This test suite creates a default sklearn classification estimator for each H2O estimator.
Then, it feeds them with H2O frames (more efficient and ensures compatibility with old API.)
or with numpy arrays to provide the simplest approach for users wanting to use H2O like any sklearn estimator.
"""

seed = 2019
init_connection_args = dict(strict_version_check=False, show_progress=True)

scores = defaultdict(dict)

def _get_data(format='numpy', n_classes=2):
    X, y = make_classification(n_samples=100, n_features=10, n_informative=5, n_classes=n_classes, random_state=seed)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=seed)
    data = ns(X_train=X_train, X_test=X_test, y_train=y_train, y_test=y_test)
    if format == 'h2o':
        for k, v in data.__dict__.items():
            setattr(data, k, h2o.H2OFrame(v))
    return data

def _get_default_args(estimator_cls):
    defaults = dict(
        H2OCoxProportionalHazardsClassifier=dict(),
        H2ODeepLearningClassifier=dict(seed=seed, reproducible=True),
        H2OGeneralizedAdditiveClassifier=dict(family='binomial', seed=seed, gam_columns=["C1"]),
        H2OGeneralizedLinearClassifier=dict(family='binomial', seed=seed),
    )
    return defaults.get(estimator_cls.__name__, dict(seed=seed))

def _get_custom_behaviour(estimator_cls):
    custom = dict(
        # H2ODeepLearningClassifier=dict(scores_may_differ=True),
    )
    return custom.get(estimator_cls.__name__, dict())

def test_estimator_with_h2o_frames(estimator_cls):
    args = _get_default_args(estimator_cls)
    estimator = estimator_cls(**args)

    data = _get_data(format='h2o', n_classes=2)
    assert isinstance(data.X_train, h2o.H2OFrame)
    estimator.fit(data.X_train, data.y_train, verbose=True)  # Adding verbose to test fit_params integration
    preds = estimator.predict(data.X_test)
    print(preds)
    assert isinstance(preds, h2o.H2OFrame)
    if _get_custom_behaviour(estimator_cls).get('preds_as_vector', True):
        assert preds.dim == [len(data.X_test), 1], "got {}".format(preds.dim)
    else:
        assert preds.dim[0] == len(data.X_test)

    probs = estimator.predict_proba(data.X_test)
    print(probs)
    assert probs.dim == [len(data.X_test), 2], "got {}".format(probs.dim)
    assert np.allclose(np.sum(probs.as_data_frame().values, axis=1), 1.), "`predict_proba` didn't return probabilities"

    score = estimator.score(data.X_test, data.y_test)
    assert isinstance(score, float)
    skl_score = accuracy_score(data.y_test.as_data_frame().values, preds.as_data_frame().values)
    assert abs(score - skl_score) < 1e-6, "score={}, skl_score={}".format(score, skl_score)
    scores[estimator_cls].update(with_h2o_frames=score)

def test_estimator_with_numpy_arrays(estimator_cls):
    estimator = estimator_cls(init_connection_args=init_connection_args, **_get_default_args(estimator_cls))

    data = _get_data(format='numpy', n_classes=2)
    assert isinstance(data.X_train, np.ndarray)

    with estimator:
        estimator.fit(data.X_train, data.y_train)
        preds = estimator.predict(data.X_test)
        print(preds)
        assert isinstance(preds, np.ndarray)
        if _get_custom_behaviour(estimator_cls).get('preds_as_vector', True):
            assert preds.shape == (len(data.X_test),), "got {}".format(preds.shape)
        else:
            assert preds.shape[0] == len(data.X_test)

        probs = estimator.predict_proba(data.X_test)
        print(probs)
        assert probs.shape == (len(data.X_test), 2)
        assert np.allclose(np.sum(probs, axis=1), 1.), "`predict_proba` didn't return probabilities"

        score = estimator.score(data.X_test, data.y_test)
        assert isinstance(score, float)
        skl_score = accuracy_score(data.y_test, preds)
        assert abs(score - skl_score) < 1e-6
        scores[estimator_cls].update(with_numpy_arrays=score)

def test_scores_are_equivalent(estimator_cls):
    try:
        lk, rk = ('with_h2o_frames', 'with_numpy_arrays')
        est_scores = scores[estimator_cls]
        if lk in est_scores and rk in est_scores:
            assert abs(est_scores[lk] - abs(est_scores[rk])) < 1e-6, \
                "expected equivalent scores but got {lk}={lscore} and {rk}={rscore}" \
                    .format(lk=lk, rk=rk, lscore=est_scores[lk], rscore=est_scores[rk])
        elif lk not in est_scores:
            print("no scores for {}".format(estimator_cls.__name__+' '+lk))
        else:
            print("no scores for {}".format(estimator_cls.__name__+' '+rk))
    except AssertionError as e:
        if _get_custom_behaviour(estimator_cls).get('scores_may_differ', False):
            print("ERROR !!! "+str(e))
        else:
            raise e

def make_test(test, classifier):
    bound_test = partial(test, classifier)
    bound_test.__name__ = test.__name__
    pyunit_utils.tag_test(bound_test, classifier.__name__)
    return bound_test

def make_tests(classifier):
    return list(map(lambda test: make_test(test, classifier), [
        test_estimator_with_h2o_frames,
        test_estimator_with_numpy_arrays,
        test_scores_are_equivalent
    ]))

failing = [
    'H2OStackedEnsembleClassifier', 'H2OUpliftRandomForestClassifier'  # needs a separate test (requires models as parameters)
]
classifiers = [cls for name, cls in inspect.getmembers(h2o.sklearn, inspect.isclass)
               if name.endswith('Classifier') and name not in ['H2OAutoMLClassifier']+failing]
pyunit_utils.run_tests([make_tests(c) for c in classifiers])
<|endoftext|>


<commit_before>'''
Main wrapper script for Genomel Recalibration workflow
'''
import os
import time
import argparse
import logging
import sys
import uuid
import tempfile
import utils.s3
import utils.pipeline
import datetime
import socket
import json

import postgres.status
import postgres.utils
import postgres.mixins

from sqlalchemy.exc import NoSuchTableError

def is_nat(x):
    '''
    Checks that a value is a natural number.
    '''
    if int(x) > 0:
        return int(x)
    raise argparse.ArgumentTypeError('%s must be positive, non-zero' % x)

def get_args():
    '''
    Loads the parser
    '''
    # Main parser
    parser = argparse.ArgumentParser(description="Genomel Recalibration Pipeline")
    # Args
    required = parser.add_argument_group("Required input parameters")
    # Metadata from input table
    required.add_argument("--input_id", default=None, help="INPUT_ID, internal production id.")
    required.add_argument("--project", default=None, help="PROJECT, PDC project name.")
    required.add_argument("--md5", default=None, help="MD5, md5 of input file.")
    required.add_argument("--s3_url", default=None, help="S3_URL, s3 url of the input.")
    required.add_argument("--s3_profile", required=True, help="S3 profile name for project tenant.")
    required.add_argument("--s3_endpoint", required=True, help="S3 endpoint url for project tenant.")
    # Parameters for pipeline
    required.add_argument("--basedir", default="/mnt/SCRATCH/", help="Base directory for computations.")
    required.add_argument("--refdir", required=True, help="Path to reference directory.")
    required.add_argument("--cwl", required=True, help="Path to Genomel Recalibration CWL workflow yaml.")
    required.add_argument("--s3dir", default="s3://", help="S3 dir for uploading output files.")

    required.add_argument('--java_heap', required=True, help='Java heap memory limit.')
    required.add_argument('--thread_count', type=is_nat, default=8, help='Threads count.')

    return parser.parse_args()

def run_pipeline(args, statusclass, metricsclass):
    '''
    Executes the CWL pipeline and record status/metrics tables
    '''
    if not os.path.isdir(args.basedir):
        raise Exception("Could not find path to base directory: %s" %args.basedir)

    # Generate a uuid
    output_id = uuid.uuid4()
    # Get hostname
    hostname = socket.gethostname()
    # Get datetime start
    datetime_start = str(datetime.datetime.now())

    # Create directory structure
    jobdir = tempfile.mkdtemp(prefix="hc_%s_" % str(output_id), dir=args.basedir)
    workdir = tempfile.mkdtemp(prefix="workdir_", dir=jobdir)
    inputdir = tempfile.mkdtemp(prefix="input_", dir=jobdir)
    resultdir = tempfile.mkdtemp(prefix="result_", dir=jobdir)
    refdir = args.refdir

    # Setup logger
    log_file = os.path.join(resultdir, "%s.genomel.hc.cwl.log" % str(output_id))
    logger = utils.pipeline.setup_logging(logging.INFO, str(output_id), log_file)

    # Logging inputs
    logger.info("hostname: %s" % (hostname))
    logger.info("input_id: %s" % (args.input_id))
    logger.info("project: %s" % (args.project))
    logger.info("md5: %s" % (args.md5))
    logger.info("datetime_start: %s" % (datetime_start))

    # Setup start point
    cwl_start = time.time()

    # Getting refs
    logger.info("getting resources")
    reference_data        = utils.pipeline.load_reference_json('etc/reference.json')
    reference_fasta_path  = os.path.join(refdir, reference_data["reference_fasta"])
    reference_dbsnp       = os.path.join(refdir, reference_data["reference_dbsnp"])
    reference_intervals   = os.path.join(refdir, reference_data["reference_intervals"])
    postgres_config       = os.path.join(refdir, reference_data["pg_config"])

    # Logging pipeline info
    cwl_version    = reference_data["cwl_version"]
    docker_version = reference_data["docker_version"]
    logger.info("cwl_version: %s" % (cwl_version))
    logger.info("docker_version: %s" % (docker_version))

    # Download input bam and index
    input_url = args.s3_url
    input_bam = os.path.join(inputdir, os.path.basename(input_url))
    input_index_url = input_url + '.bai'
    project_s3_profile = args.s3_profile
    project_s3_endpoint_url = args.s3_endpoint
    download_exit_code = utils.s3.aws_s3_get(logger, input_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)
    download_exit_code = utils.s3.aws_s3_get(logger, input_index_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)

    download_end_time = time.time()
    download_time = download_end_time - cwl_start
    if not (download_exit_code != 0 or str(utils.pipeline.get_md5(input_bam)) != args.md5):
        logger.info("Download input %s successfully in %s, and md5 matches. Input bam is %s." % (args.input_id, download_time, input_bam))
    else:
        cwl_elapsed = download_time
        datetime_end = str(datetime.datetime.now())
        engine = postgres.utils.get_db_engine(postgres_config)
        postgres.utils.set_download_error(download_exit_code, logger, engine,
                                          output_id, [args.input_id], output_id,
                                          datetime_start, datetime_end,
                                          hostname, cwl_version, docker_version,
                                          download_time, cwl_elapsed, statusclass, metricsclass)
        # Exit
        sys.exit(download_exit_code)

    # Define output
    output_gvcf = os.path.join(workdir, os.path.basename(input_bam)).replace('.bam', '.g.vcf.gz')

    # Create input json
    input_json_file = os.path.join(resultdir, '{0}.genomel.hc.inputs.json'.format(str(output_id)))
    input_json_data = {
      "java_opts": args.java_heap,
      "input_bam_path": {"class": "File", "path": input_bam},
      "reference_fasta_path": {"class": "File", "path": reference_fasta_path},
      "reference_snp_path": {"class": "File", "path": reference_dbsnp},
      "interval_file_path": {"class": "File", "path": reference_intervals},
      "output_gvcf_name": output_gvcf
    }
    with open(input_json_file, 'wt') as o:
        json.dump(input_json_data, o, indent=4)
    logger.info("Preparing input json: %s" % (input_json_file))

    # Run CWL
    os.chdir(workdir)
    logger.info('Running CWL workflow')
    cmd = ['/usr/bin/time', '-v',
           '/home/ubuntu/.virtualenvs/p2/bin/cwltool',
           "--debug",
           "--tmpdir-prefix", inputdir,
           "--tmp-outdir-prefix", workdir,
           args.cwl,
           input_json_file]
    cwl_exit = utils.pipeline.run_command(cmd, logger)
    cwl_failure = False
    if cwl_exit:
        cwl_failure = True

    # Get md5 and file size
    md5 = utils.pipeline.get_md5(output_gvcf)
    file_size = utils.pipeline.get_file_size(output_gvcf)

    # Upload output
    upload_start = time.time()
    logger.info("Uploading workflow output to %s" % (upload_bam_location))
    upload_dir_location = os.path.join(args.s3dir, str(output_id))
    upload_gvcf_location = os.path.join(upload_dir_location, os.path.basename(output_gvcf))
    upload_exit  = utils.s3.aws_s3_put(logger, upload_gvcf_location, output_gvcf, args.s3_profile, args.s3_endpoint, recursive=False)

    # Establish connection with database
    engine = postgres.utils.get_db_engine(postgres_config)

    # Calculate times
    cwl_end = time.time()
    upload_time = cwl_end - upload_start
    cwl_elapsed = cwl_end - cwl_start
    datetime_end = str(datetime.datetime.now())
    logger.info("datetime_end: %s" % (datetime_end))

    # Get status info
    logger.info("Get status/metrics info")
    status, loc = postgres.status.get_status(upload_exit, cwl_exit, upload_bam_location, upload_dir_location, logger)

    # Get metrics info
    time_metrics = utils.pipeline.get_time_metrics(log_file)

    # Set status table
    logger.info("Updating status")
    postgres.utils.add_pipeline_status(engine, output_id, [args.input_id], output_id,
                                       status, loc, datetime_start, datetime_end,
                                       md5, file_size, hostname, cwl_version, docker_version, statusclass)
    # Set metrics table
    logger.info("Updating metrics")
    postgres.utils.add_pipeline_metrics(engine, output_id, [args.input_id], download_time,
                                        upload_time, args.thread_count, cwl_elapsed,
                                        time_metrics['system_time'],
                                        time_metrics['user_time'],
                                        time_metrics['wall_clock'],
                                        time_metrics['percent_of_cpu'],
                                        time_metrics['maximum_resident_set_size'],
                                        status, metricsclass)

    # Remove job directories, upload final log file
    logger.info("Uploading main log file")
    utils.s3.aws_s3_put(logger, upload_dir_location + '/' + os.path.basename(log_file), log_file, args.s3_profile, args.s3_endpoint, recursive=False)
    utils.pipeline.remove_dir(jobdir)

if __name__ == '__main__':

    # Get args
    args = get_args()
    project = args.project.lower()

    # Setup postgres classes for tables
    class HaplotypeCallerStatus(postgres.mixins.StatusTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_status'
    class HaplotypeCallerMetrics(postgres.mixins.MetricsTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_metrics'

    # Run pipeline
    run_pipeline(args, RecalibrationStatus, RecalibrationMetrics)
<commit_msg>fix(slurm): fix table class names<commit_after>'''
Main wrapper script for Genomel Recalibration workflow
'''
import os
import time
import argparse
import logging
import sys
import uuid
import tempfile
import utils.s3
import utils.pipeline
import datetime
import socket
import json

import postgres.status
import postgres.utils
import postgres.mixins

from sqlalchemy.exc import NoSuchTableError

def is_nat(x):
    '''
    Checks that a value is a natural number.
    '''
    if int(x) > 0:
        return int(x)
    raise argparse.ArgumentTypeError('%s must be positive, non-zero' % x)

def get_args():
    '''
    Loads the parser
    '''
    # Main parser
    parser = argparse.ArgumentParser(description="Genomel Recalibration Pipeline")
    # Args
    required = parser.add_argument_group("Required input parameters")
    # Metadata from input table
    required.add_argument("--input_id", default=None, help="INPUT_ID, internal production id.")
    required.add_argument("--project", default=None, help="PROJECT, PDC project name.")
    required.add_argument("--md5", default=None, help="MD5, md5 of input file.")
    required.add_argument("--s3_url", default=None, help="S3_URL, s3 url of the input.")
    required.add_argument("--s3_profile", required=True, help="S3 profile name for project tenant.")
    required.add_argument("--s3_endpoint", required=True, help="S3 endpoint url for project tenant.")
    # Parameters for pipeline
    required.add_argument("--basedir", default="/mnt/SCRATCH/", help="Base directory for computations.")
    required.add_argument("--refdir", required=True, help="Path to reference directory.")
    required.add_argument("--cwl", required=True, help="Path to Genomel Recalibration CWL workflow yaml.")
    required.add_argument("--s3dir", default="s3://", help="S3 dir for uploading output files.")

    required.add_argument('--java_heap', required=True, help='Java heap memory limit.')
    required.add_argument('--thread_count', type=is_nat, default=8, help='Threads count.')

    return parser.parse_args()

def run_pipeline(args, statusclass, metricsclass):
    '''
    Executes the CWL pipeline and record status/metrics tables
    '''
    if not os.path.isdir(args.basedir):
        raise Exception("Could not find path to base directory: %s" %args.basedir)

    # Generate a uuid
    output_id = uuid.uuid4()
    # Get hostname
    hostname = socket.gethostname()
    # Get datetime start
    datetime_start = str(datetime.datetime.now())

    # Create directory structure
    jobdir = tempfile.mkdtemp(prefix="hc_%s_" % str(output_id), dir=args.basedir)
    workdir = tempfile.mkdtemp(prefix="workdir_", dir=jobdir)
    inputdir = tempfile.mkdtemp(prefix="input_", dir=jobdir)
    resultdir = tempfile.mkdtemp(prefix="result_", dir=jobdir)
    refdir = args.refdir

    # Setup logger
    log_file = os.path.join(resultdir, "%s.genomel.hc.cwl.log" % str(output_id))
    logger = utils.pipeline.setup_logging(logging.INFO, str(output_id), log_file)

    # Logging inputs
    logger.info("hostname: %s" % (hostname))
    logger.info("input_id: %s" % (args.input_id))
    logger.info("project: %s" % (args.project))
    logger.info("md5: %s" % (args.md5))
    logger.info("datetime_start: %s" % (datetime_start))

    # Setup start point
    cwl_start = time.time()

    # Getting refs
    logger.info("getting resources")
    reference_data        = utils.pipeline.load_reference_json('etc/reference.json')
    reference_fasta_path  = os.path.join(refdir, reference_data["reference_fasta"])
    reference_dbsnp       = os.path.join(refdir, reference_data["reference_dbsnp"])
    reference_intervals   = os.path.join(refdir, reference_data["reference_intervals"])
    postgres_config       = os.path.join(refdir, reference_data["pg_config"])

    # Logging pipeline info
    cwl_version    = reference_data["cwl_version"]
    docker_version = reference_data["docker_version"]
    logger.info("cwl_version: %s" % (cwl_version))
    logger.info("docker_version: %s" % (docker_version))

    # Download input bam and index
    input_url = args.s3_url
    input_bam = os.path.join(inputdir, os.path.basename(input_url))
    input_index_url = input_url + '.bai'
    project_s3_profile = args.s3_profile
    project_s3_endpoint_url = args.s3_endpoint
    download_exit_code = utils.s3.aws_s3_get(logger, input_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)
    download_exit_code = utils.s3.aws_s3_get(logger, input_index_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)

    download_end_time = time.time()
    download_time = download_end_time - cwl_start
    if not (download_exit_code != 0 or str(utils.pipeline.get_md5(input_bam)) != args.md5):
        logger.info("Download input %s successfully in %s, and md5 matches. Input bam is %s." % (args.input_id, download_time, input_bam))
    else:
        cwl_elapsed = download_time
        datetime_end = str(datetime.datetime.now())
        engine = postgres.utils.get_db_engine(postgres_config)
        postgres.utils.set_download_error(download_exit_code, logger, engine,
                                          output_id, [args.input_id], output_id,
                                          datetime_start, datetime_end,
                                          hostname, cwl_version, docker_version,
                                          download_time, cwl_elapsed, statusclass, metricsclass)
        # Exit
        sys.exit(download_exit_code)

    # Define output
    output_gvcf = os.path.join(workdir, os.path.basename(input_bam)).replace('.bam', '.g.vcf.gz')

    # Create input json
    input_json_file = os.path.join(resultdir, '{0}.genomel.hc.inputs.json'.format(str(output_id)))
    input_json_data = {
      "java_opts": args.java_heap,
      "input_bam_path": {"class": "File", "path": input_bam},
      "reference_fasta_path": {"class": "File", "path": reference_fasta_path},
      "reference_snp_path": {"class": "File", "path": reference_dbsnp},
      "interval_file_path": {"class": "File", "path": reference_intervals},
      "output_gvcf_name": output_gvcf
    }
    with open(input_json_file, 'wt') as o:
        json.dump(input_json_data, o, indent=4)
    logger.info("Preparing input json: %s" % (input_json_file))

    # Run CWL
    os.chdir(workdir)
    logger.info('Running CWL workflow')
    cmd = ['/usr/bin/time', '-v',
           '/home/ubuntu/.virtualenvs/p2/bin/cwltool',
           "--debug",
           "--tmpdir-prefix", inputdir,
           "--tmp-outdir-prefix", workdir,
           args.cwl,
           input_json_file]
    cwl_exit = utils.pipeline.run_command(cmd, logger)
    cwl_failure = False
    if cwl_exit:
        cwl_failure = True

    # Get md5 and file size
    md5 = utils.pipeline.get_md5(output_gvcf)
    file_size = utils.pipeline.get_file_size(output_gvcf)

    # Upload output
    upload_start = time.time()
    logger.info("Uploading workflow output to %s" % (upload_bam_location))
    upload_dir_location = os.path.join(args.s3dir, str(output_id))
    upload_gvcf_location = os.path.join(upload_dir_location, os.path.basename(output_gvcf))
    upload_exit  = utils.s3.aws_s3_put(logger, upload_gvcf_location, output_gvcf, args.s3_profile, args.s3_endpoint, recursive=False)

    # Establish connection with database
    engine = postgres.utils.get_db_engine(postgres_config)

    # Calculate times
    cwl_end = time.time()
    upload_time = cwl_end - upload_start
    cwl_elapsed = cwl_end - cwl_start
    datetime_end = str(datetime.datetime.now())
    logger.info("datetime_end: %s" % (datetime_end))

    # Get status info
    logger.info("Get status/metrics info")
    status, loc = postgres.status.get_status(upload_exit, cwl_exit, upload_bam_location, upload_dir_location, logger)

    # Get metrics info
    time_metrics = utils.pipeline.get_time_metrics(log_file)

    # Set status table
    logger.info("Updating status")
    postgres.utils.add_pipeline_status(engine, output_id, [args.input_id], output_id,
                                       status, loc, datetime_start, datetime_end,
                                       md5, file_size, hostname, cwl_version, docker_version, statusclass)
    # Set metrics table
    logger.info("Updating metrics")
    postgres.utils.add_pipeline_metrics(engine, output_id, [args.input_id], download_time,
                                        upload_time, args.thread_count, cwl_elapsed,
                                        time_metrics['system_time'],
                                        time_metrics['user_time'],
                                        time_metrics['wall_clock'],
                                        time_metrics['percent_of_cpu'],
                                        time_metrics['maximum_resident_set_size'],
                                        status, metricsclass)

    # Remove job directories, upload final log file
    logger.info("Uploading main log file")
    utils.s3.aws_s3_put(logger, upload_dir_location + '/' + os.path.basename(log_file), log_file, args.s3_profile, args.s3_endpoint, recursive=False)
    utils.pipeline.remove_dir(jobdir)

if __name__ == '__main__':

    # Get args
    args = get_args()
    project = args.project.lower()

    # Setup postgres classes for tables
    class CallerStatus(postgres.mixins.StatusTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_status'
    class CallerMetrics(postgres.mixins.MetricsTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_metrics'

    # Run pipeline
    run_pipeline(args, CallerStatus, CallerMetrics)
<|endoftext|>