<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 Martine Lenders <mail@martine-lenders.eu>
#
# Distributed under terms of the MIT license.

from __future__ import print_function
import os
import sys
import random
import subprocess
import time
import types
import pexpect

DEFAULT_TIMEOUT = 5

class Strategy(object):
    def __init__(self, func=None):
        if func is not None:
            if sys.version_info < (3,):
                self.__class__.execute = types.MethodType(func, self, self.__class__)
            else:
                self.__class__.execute = types.MethodType(func, self)

    def execute(self, *args, **kwargs):
        raise NotImplementedError()

class ApplicationStrategy(Strategy):
    def __init__(self, app_dir=os.getcwd(), func=None):
        super(ApplicationStrategy, self).__init__(func)
        self.app_dir = app_dir

class BoardStrategy(Strategy):
    def __init__(self, board, func=None):
        super(BoardStrategy, self).__init__(func)
        self.board = board

    def __run_make(self, application, make_targets, env=None):
        env = os.environ.copy()
        if env is not None:
            env.update(env)
        env.update(self.board.to_env())
        cmd = ("make", "-C", application) + make_targets
        print(' '.join(cmd))
        print(subprocess.check_output(cmd, env=env))

    def execute(self, application):
        super(BoardStrategy, self).execute(application)

class CleanStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(CleanStrategy, self).__run_make(application, ("-B", "clean"), env)

class BuildStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(BuildStrategy, self).__run_make(application, ("all",), env)

class FlashStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(FlashStrategy, self).__run_make(application, ("all",), env)

class ResetStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(ResetStrategy, self).__run_make(application, ("reset",), env)

class Board(object):
    def __init__(self, name, port=None, serial=None, clean=None,
                 build=None, flash=None,
                 reset=None, term=None):
        def _reset_native_execute(obj, application, env=None, *args, **kwargs):
            pass

        if (name == "native") and (reset is None):
            reset = _reset_native_execute

        self.name = name
        self.port = port
        self.serial = serial
        self.clean_strategy = CleanStrategy(self, clean)
        self.build_strategy = BuildStrategy(self, build)
        self.flash_strategy = FlashStrategy(self, flash)
        self.reset_strategy = ResetStrategy(self, reset)

    def __len__(self):
        return 1

    def __iter__(self):
        return self

    def next(self):
        raise StopIteration()

    def __repr__(self):
        return ("<Board %s,port=%s,serial=%s>" %
                (repr(self.name), repr(self.port), repr(self.serial)))

    def to_env(self):
        env = {}
        if self.name:
            env['BOARD'] = self.name
        if self.port:
            env['PORT'] = self.port
        if self.serial:
            env['SERIAL'] = self.serial
        return env

    def clean(self, application=os.getcwd(), env=None):
        self.build_strategy.execute(application, env)

    def build(self, application=os.getcwd(), env=None):
        self.build_strategy.execute(application, env)

    def flash(self, application=os.getcwd(), env=None):
        self.flash_strategy.execute(application, env)

    def reset(self, application=os.getcwd(), env=None):
        self.reset_strategy.execute(application, env)

class BoardGroup(object):
    def __init__(self, boards):
        self.boards = boards

    def __len__(self):
        return len(self.boards)

    def __iter__(self):
        return iter(self.boards)

    def __repr__(self):
        return str(self.boards)

    def clean(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.clean(application, env)

    def build(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.build(application, env)

    def flash(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.flash(application, env)

    def reset(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.reset(application, env)

def default_test_case(board_group, application, env=None):
    for board in board_group:
        env = os.environ.copy()
        if env is not None:
            env.update(env)
        env.update(board.to_env())
        with pexpect.spawnu("make", ["-C", application, "term"], env=env,
                            timeout=DEFAULT_TIMEOUT,
                            logfile=sys.stdout) as spawn:
            spawn.expect("TEST: SUCCESS")

class TestStrategy(ApplicationStrategy):
    def execute(self, board_groups, test_cases=[default_test_case],
                timeout=DEFAULT_TIMEOUT, env=None):
        for board_group in board_groups:
            print("Testing for %s: " % board_group)
            for test_case in test_cases:
                board_group.reset()
                test_case(board_group, self.app_dir, env=None)
                sys.stdout.write('.')
                sys.stdout.flush()
            print()

def get_ipv6_address(spawn):
    spawn.sendline(u"ifconfig")
    spawn.expect(r"[A-Za-z0-9]{2}_[0-9]+:  inet6 (fe80::[0-9a-f:]+)\s")
    return spawn.match.group(1)

def test_ipv6_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        ipprot = random.randint(0x00, 0xff)
        receiver_ip = get_ipv6_address(receiver)
        receiver.sendline(u"ip server start %d" % ipprot)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"ip send %s %d 01:23:45:67:89:ab:cd:ef" % (receiver_ip, ipprot))
        sender.expect_exact(u"Success: send 8 byte over IPv6 to %s (next header: %d)" %
                            (receiver_ip, ipprot))
        receiver.expect(u"00000000  01  23  45  67  89  AB  CD  EF")

def test_udpv6_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        port = random.randint(0x0000, 0xffff)
        receiver_ip = get_ipv6_address(receiver)

        receiver.sendline(u"udp server start %d" % port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"udp send %s %d ab:cd:ef" % (receiver_ip, port))
        sender.expect_exact("Success: send 3 byte over UDP to [{}]:{}"
                            .format(receiver_ip, port))
        receiver.expect(u"00000000  AB  CD  EF")

def test_tcpv6_send(board_group, application, env=None):
    env_client = os.environ.copy()
    if env is not None:
        env_client.update(env)
    env_client.update(board_group.boards[0].to_env())
    env_server = os.environ.copy()
    if env is not None:
        env_server.update(env)
    env_server.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_client,
                        timeout=DEFAULT_TIMEOUT) as client, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_server,
                       timeout=DEFAULT_TIMEOUT) as server:
        port = random.randint(0x0000, 0xffff)
        server_ip = get_ipv6_address(server)
        client_ip = get_ipv6_address(client)

        server.sendline(u"tcp server start %d" % port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        client.sendline(u"tcp connect %s %d" % (server_ip, port))
        server.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % client_ip)
        client.sendline(u"tcp send affe:abe")
        client.expect_exact(u"Success: send 4 byte over TCP to server")
        server.expect(u"00000000  AF  FE  AB  E0")
        client.sendline(u"tcp disconnect")
        client.sendline(u"tcp send affe:abe")
        client.expect_exact(u"could not send")

def test_triple_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        udp_port = random.randint(0x0000, 0xffff)
        tcp_port = random.randint(0x0000, 0xffff)
        ipprot = random.randint(0x00, 0xff)
        receiver_ip = get_ipv6_address(receiver)
        sender_ip = get_ipv6_address(sender)

        receiver.sendline(u"ip server start %d" % ipprot)
        receiver.sendline(u"udp server start %d" % udp_port)
        receiver.sendline(u"tcp server start %d" % tcp_port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"udp send %s %d 01:23" % (receiver_ip, udp_port))
        sender.expect_exact(u"Success: send 2 byte over UDP to [%s]:%d" %
                            (receiver_ip, udp_port))
        receiver.expect(u"00000000  01  23")

        sender.sendline(u"ip send %s %d 01:02:03:04" % (receiver_ip, ipprot))
        sender.expect_exact(u"Success: send 4 byte over IPv6 to %s (next header: %d)" %
                            (receiver_ip, ipprot))
        receiver.expect(u"00000000  01  02  03  04")
        sender.sendline(u"tcp connect %s %d" % (receiver_ip, tcp_port))
        receiver.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % sender_ip)
        sender.sendline(u"tcp send dead:beef")
        sender.expect_exact(u"Success: send 4 byte over TCP to server")
        receiver.expect(u"00000000  DE  AD  BE  EF")

if __name__ == "__main__":
    TestStrategy().execute([BoardGroup((Board("native", "tap0"),
                            Board("native", "tap1")))],
                           [test_ipv6_send, test_udpv6_send, test_tcpv6_send,
                            test_triple_send])
<commit_msg>tests/lwip: add regression test for multiple simultaneous connect case<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 Martine Lenders <mail@martine-lenders.eu>
#
# Distributed under terms of the MIT license.

from __future__ import print_function
import os
import sys
import random
import subprocess
import time
import types
import pexpect
import socket

DEFAULT_TIMEOUT = 5

class Strategy(object):
    def __init__(self, func=None):
        if func is not None:
            if sys.version_info < (3,):
                self.__class__.execute = types.MethodType(func, self, self.__class__)
            else:
                self.__class__.execute = types.MethodType(func, self)

    def execute(self, *args, **kwargs):
        raise NotImplementedError()

class ApplicationStrategy(Strategy):
    def __init__(self, app_dir=os.getcwd(), func=None):
        super(ApplicationStrategy, self).__init__(func)
        self.app_dir = app_dir

class BoardStrategy(Strategy):
    def __init__(self, board, func=None):
        super(BoardStrategy, self).__init__(func)
        self.board = board

    def __run_make(self, application, make_targets, env=None):
        env = os.environ.copy()
        if env is not None:
            env.update(env)
        env.update(self.board.to_env())
        cmd = ("make", "-C", application) + make_targets
        print(' '.join(cmd))
        print(subprocess.check_output(cmd, env=env))

    def execute(self, application):
        super(BoardStrategy, self).execute(application)

class CleanStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(CleanStrategy, self).__run_make(application, ("-B", "clean"), env)

class BuildStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(BuildStrategy, self).__run_make(application, ("all",), env)

class FlashStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(FlashStrategy, self).__run_make(application, ("all",), env)

class ResetStrategy(BoardStrategy):
    def execute(self, application, env=None):
        super(ResetStrategy, self).__run_make(application, ("reset",), env)

class Board(object):
    def __init__(self, name, port=None, serial=None, clean=None,
                 build=None, flash=None,
                 reset=None, term=None):
        def _reset_native_execute(obj, application, env=None, *args, **kwargs):
            pass

        if (name == "native") and (reset is None):
            reset = _reset_native_execute

        self.name = name
        self.port = port
        self.serial = serial
        self.clean_strategy = CleanStrategy(self, clean)
        self.build_strategy = BuildStrategy(self, build)
        self.flash_strategy = FlashStrategy(self, flash)
        self.reset_strategy = ResetStrategy(self, reset)

    def __len__(self):
        return 1

    def __iter__(self):
        return self

    def next(self):
        raise StopIteration()

    def __repr__(self):
        return ("<Board %s,port=%s,serial=%s>" %
                (repr(self.name), repr(self.port), repr(self.serial)))

    def to_env(self):
        env = {}
        if self.name:
            env['BOARD'] = self.name
        if self.port:
            env['PORT'] = self.port
        if self.serial:
            env['SERIAL'] = self.serial
        return env

    def clean(self, application=os.getcwd(), env=None):
        self.build_strategy.execute(application, env)

    def build(self, application=os.getcwd(), env=None):
        self.build_strategy.execute(application, env)

    def flash(self, application=os.getcwd(), env=None):
        self.flash_strategy.execute(application, env)

    def reset(self, application=os.getcwd(), env=None):
        self.reset_strategy.execute(application, env)

class BoardGroup(object):
    def __init__(self, boards):
        self.boards = boards

    def __len__(self):
        return len(self.boards)

    def __iter__(self):
        return iter(self.boards)

    def __repr__(self):
        return str(self.boards)

    def clean(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.clean(application, env)

    def build(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.build(application, env)

    def flash(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.flash(application, env)

    def reset(self, application=os.getcwd(), env=None):
        for board in self.boards:
            board.reset(application, env)

def default_test_case(board_group, application, env=None):
    for board in board_group:
        env = os.environ.copy()
        if env is not None:
            env.update(env)
        env.update(board.to_env())
        with pexpect.spawnu("make", ["-C", application, "term"], env=env,
                            timeout=DEFAULT_TIMEOUT,
                            logfile=sys.stdout) as spawn:
            spawn.expect("TEST: SUCCESS")

class TestStrategy(ApplicationStrategy):
    def execute(self, board_groups, test_cases=[default_test_case],
                timeout=DEFAULT_TIMEOUT, env=None):
        for board_group in board_groups:
            print("Testing for %s: " % board_group)
            for test_case in test_cases:
                board_group.reset()
                test_case(board_group, self.app_dir, env=None)
                sys.stdout.write('.')
                sys.stdout.flush()
            print()

def get_ipv6_address(spawn):
    spawn.sendline(u"ifconfig")
    spawn.expect(r"[A-Za-z0-9]{2}_[0-9]+:  inet6 (fe80::[0-9a-f:]+)\s")
    return spawn.match.group(1)

def test_ipv6_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        ipprot = random.randint(0x00, 0xff)
        receiver_ip = get_ipv6_address(receiver)
        receiver.sendline(u"ip server start %d" % ipprot)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"ip send %s %d 01:23:45:67:89:ab:cd:ef" % (receiver_ip, ipprot))
        sender.expect_exact(u"Success: send 8 byte over IPv6 to %s (next header: %d)" %
                            (receiver_ip, ipprot))
        receiver.expect(u"00000000  01  23  45  67  89  AB  CD  EF")

def test_udpv6_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        port = random.randint(0x0000, 0xffff)
        receiver_ip = get_ipv6_address(receiver)

        receiver.sendline(u"udp server start %d" % port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"udp send %s %d ab:cd:ef" % (receiver_ip, port))
        sender.expect_exact("Success: send 3 byte over UDP to [{}]:{}"
                            .format(receiver_ip, port))
        receiver.expect(u"00000000  AB  CD  EF")

def test_tcpv6_send(board_group, application, env=None):
    env_client = os.environ.copy()
    if env is not None:
        env_client.update(env)
    env_client.update(board_group.boards[0].to_env())
    env_server = os.environ.copy()
    if env is not None:
        env_server.update(env)
    env_server.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_client,
                        timeout=DEFAULT_TIMEOUT) as client, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_server,
                       timeout=DEFAULT_TIMEOUT) as server:
        port = random.randint(0x0000, 0xffff)
        server_ip = get_ipv6_address(server)
        client_ip = get_ipv6_address(client)

        server.sendline(u"tcp server start %d" % port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        client.sendline(u"tcp connect %s %d" % (server_ip, port))
        server.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % client_ip)
        client.sendline(u"tcp send affe:abe")
        client.expect_exact(u"Success: send 4 byte over TCP to server")
        server.expect(u"00000000  AF  FE  AB  E0")
        client.sendline(u"tcp disconnect")
        client.sendline(u"tcp send affe:abe")
        client.expect_exact(u"could not send")

def test_tcpv6_multiconnect(board_group, application, env=None):
    if any(b.name != "native" for b in board_group.boards):
        # run test only with native
        print("SKIP_TEST INFO found non-native board")
        return
    env_client = os.environ.copy()
    if env is not None:
        env_client.update(env)
    env_client.update(board_group.boards[0].to_env())
    env_server = os.environ.copy()
    if env is not None:
        env_server.update(env)
    env_server.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_client,
                        timeout=DEFAULT_TIMEOUT) as client, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_server,
                       timeout=DEFAULT_TIMEOUT) as server:
        port = random.randint(0x0000, 0xffff)
        server_ip = get_ipv6_address(server)
        client_ip = get_ipv6_address(client)

        try:
            connect_addr = socket.getaddrinfo(
                "%s%%tapbr0" % server_ip, port)[0][4]
        except socket.gaierror as e:
            print("SKIP_TEST INFO", e)
            return
        server.sendline(u"tcp server start %d" % port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        client.sendline(u"tcp connect %s %d" % (server_ip, port))
        server.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % client_ip)
        with socket.socket(socket.AF_INET6) as sock:
            sock.connect(connect_addr)
            server.expect(u"Error on TCP accept \\[-[0-9]+\\]")
        client.sendline(u"tcp disconnect")
        server.expect(u"TCP connection to \\[%s\\]:[0-9]+ reset" % client_ip)
        client.sendline(u"tcp connect %s %d" % (server_ip, port))
        server.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % client_ip)
        client.sendline(u"tcp disconnect")
        server.expect(u"TCP connection to \\[%s\\]:[0-9]+ reset" % client_ip)
        with socket.socket(socket.AF_INET6) as sock:
            sock.connect(connect_addr)
            server.expect(u"TCP client \\[[0-9a-f:]+\\]:[0-9]+ connected")
        server.expect(u"TCP connection to \\[[0-9a-f:]+\\]:[0-9]+ reset")

def test_triple_send(board_group, application, env=None):
    env_sender = os.environ.copy()
    if env is not None:
        env_sender.update(env)
    env_sender.update(board_group.boards[0].to_env())
    env_receiver = os.environ.copy()
    if env is not None:
        env_receiver.update(env)
    env_receiver.update(board_group.boards[1].to_env())
    with pexpect.spawnu("make", ["-C", application, "term"], env=env_sender,
                        timeout=DEFAULT_TIMEOUT) as sender, \
        pexpect.spawnu("make", ["-C", application, "term"], env=env_receiver,
                       timeout=DEFAULT_TIMEOUT) as receiver:
        udp_port = random.randint(0x0000, 0xffff)
        tcp_port = random.randint(0x0000, 0xffff)
        ipprot = random.randint(0x00, 0xff)
        receiver_ip = get_ipv6_address(receiver)
        sender_ip = get_ipv6_address(sender)

        receiver.sendline(u"ip server start %d" % ipprot)
        receiver.sendline(u"udp server start %d" % udp_port)
        receiver.sendline(u"tcp server start %d" % tcp_port)
        # wait for neighbor discovery to be done
        time.sleep(5)
        sender.sendline(u"udp send %s %d 01:23" % (receiver_ip, udp_port))
        sender.expect_exact(u"Success: send 2 byte over UDP to [%s]:%d" %
                            (receiver_ip, udp_port))
        receiver.expect(u"00000000  01  23")

        sender.sendline(u"ip send %s %d 01:02:03:04" % (receiver_ip, ipprot))
        sender.expect_exact(u"Success: send 4 byte over IPv6 to %s (next header: %d)" %
                            (receiver_ip, ipprot))
        receiver.expect(u"00000000  01  02  03  04")
        sender.sendline(u"tcp connect %s %d" % (receiver_ip, tcp_port))
        receiver.expect(u"TCP client \\[%s\\]:[0-9]+ connected" % sender_ip)
        sender.sendline(u"tcp send dead:beef")
        sender.expect_exact(u"Success: send 4 byte over TCP to server")
        receiver.expect(u"00000000  DE  AD  BE  EF")

if __name__ == "__main__":
    TestStrategy().execute([BoardGroup((Board("native", "tap0"),
                            Board("native", "tap1")))],
                           [test_ipv6_send, test_udpv6_send, test_tcpv6_send,
                            test_tcpv6_multiconnect, test_triple_send])
<|endoftext|>


<commit_before><commit_msg>exp2b conv1 layer<commit_after>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

# Functions for initializing neural nets parameters
def init_weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float64)
  return tf.Variable(initial)

def init_bias_variable(shape):
  initial = tf.constant(0.1, shape=shape, dtype=tf.float64)
  return tf.Variable(initial)

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID')

print('==> Experiment 2b')
filepath = '../taylorswift_out/s71d7.mat'
print('==> Loading data from {}'.format(filepath))
# benchmark
t_start = time.time()

# reading data
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))
print('--Time elapsed for loading data: {t:.2f} \
    seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of test samples: {}'.format(X_val.shape[0]))

# Neural-network model set-up

'''
	CNN config parameters
'''
total_features = X_train.shape[1]
num_freq = 121
num_frames = int(total_features / num_freq)
hidden_layer_size = 20
num_classes = int(max(y_train.max(), y_test.max()) + 1)
k1 = 16
k2 = 32
l = 5

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_test_OHEnc = tf.one_hot(y_test.copy(), num_classes)

# Set-up input and output label
x = tf.placeholder(tf.float64, [None, num_freq])
y_ = tf.placeholder(tf.float32, [None, num_classes])

# first convolutional layer
W_conv1 = init_weight_variable([num_freq, 1, 1, k1])
b_conv1 = init_bias_variable([k1])
x_image = tf.reshape(x, [-1, num_freq, num_frames, 1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)

# second layer

print('==> Done.')

<|endoftext|>


<commit_before>from .tests_discoverer import load_tests
<commit_msg>please the flake gods<commit_after># flake8: noqa
from .tests_discoverer import load_tests
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

# system arg
num_layers = 2
fac = 0.5
FAST_FLAG = 1
try:
	fac = float(sys.argv[1])
	num_layers = int(sys.argv[2])
	assert(num_layers >= 2 and num_layers <= 4)
	FAST_FLAG = int(sys.argv[3])
except Exception, e:
	print('-- {}'.format(e))
print('-- Decreasing factor = {}'.format(fac))
print('-- Number of layers = {}'.format(num_layers))
print('-- FAST FLAG: {}'.format(FAST_FLAG))

# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 4i: Early Stopping...')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if FAST_FLAG:
	filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

size_list = []
for num in range(num_layers + 1):
	size_list.append(int(total_features * np.power(fac, num)))
print('-- Layer sizes = {}'.format(size_list))

batch_size = 1000

num_epochs = 500
print_freq = 10
if FAST_FLAG:
	num_epochs = 5
	print_freq = 1

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

W_ae_list = [init_weight_variable([size_list[i], size_list[i + 1]]) \
	for i in range(num_layers)]
b_ae_list = [init_bias_variable([size_list[i + 1]])\
	for i in range(num_layers)]
a_list = [tf.nn.relu(batch_nm(tf.matmul(x, W_ae_list[0]) + b_ae_list[0]))]
for i in range(num_layers - 1):
	# batch normalization for
	a_i = tf.nn.relu(batch_nm(tf.matmul(a_list[-1], W_ae_list[i + 1]) + b_ae_list[i + 1]))
	a_list.append(a_i)

# dropout
keep_prob = tf.placeholder(tf.float32)
a_drop = tf.nn.dropout(a_list[-1], keep_prob)
W_sm = init_weight_variable([size_list[-1], num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(a_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_val = sess.run(y_val_OHEnc)[:, 0, :]

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# saver setup
varsave_list = W_ae_list + b_ae_list + [W_sm, b_sm]
saver = tf.train.Saver(varsave_list)
save_path = '4imodel_{}+{}.ckpt'
opt_val_err = np.inf
opt_epoch = -1
earlystop_counter = 0
max_counter = 100

print('==> Training the full network...')
t_start = time.time()
for epoch in range(num_epochs):
	for i in range(0, num_training_vec, batch_size):
		batch_end_point = min(i + batch_size, num_training_vec)
		train_batch_data = X_train[i : batch_end_point]
		train_batch_label = y_train[i : batch_end_point]
		train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
	if (epoch + 1) % print_freq == 0:
		# evaluate metrics
		train_acc = accuracy.eval(feed_dict={x:train_batch_data, y_: train_batch_label, keep_prob: 1.0})
		train_acc_list.append(train_acc)
		val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_acc_list.append(val_acc)
		train_err = cross_entropy.eval(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 1.0})
		train_err_list.append(train_err)
		val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_err_list.append(val_err)
		print("-- epoch: %d, training error %g, validation error %g"%(epoch + 1, train_err, val_err))
		# save screenshot of the model
		if val_err < opt_val_err:
			step_counter = 0
			saver.save(sess, save_path)
			print('==> New optimal validation error found. Model saved.')
			opt_val_err, opt_epoch = val_err, epoch + 1
	if step_counter > max_counter:
		break
	step_counter += 1
t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Restore model & Evaluations
# ==============================================
saver.restore(sess, save_path)
print('==> Model restored to epoch {}'.format(opt_epoch))

train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train, keep_prob: 1.0})
val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
print('-- Training accuracy: {:.4f}'.format(train_acc))
print('-- Validation accuracy: {:.4f}'.format(val_acc))
print('-- Training error: {:.4E}'.format(train_err))
print('-- Validation error: {:.4E}'.format(val_err))

print('-- Training accuracy --')
print([float('{:.4f}'.format(x)) for x in train_acc_list])
print('-- Validation accuracy --')
print([float('{:.4f}'.format(x)) for x in val_acc_list])
print('-- Training error --')
print([float('{:.4E}'.format(x)) for x in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(x)) for x in val_err_list])

# print('==> Generating error plot...')
# x_list = range(0, print_freq * len(train_acc_list), print_freq)
# train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
# val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
# plt.xlabel('Number of epochs')
# plt.ylabel('Cross-Entropy Error')
# plt.title('Error vs Number of Epochs with {} Layers and Decreasing Factor {}'.format(num_layers, fac))
# plt.legend(loc='best')
# plt.savefig('exp4i_L{}F{}.png'.format(num_layers, fac), format='png')
# plt.close()

print('==> Finished!')
<commit_msg>adding usage exp4i<commit_after>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

# usage: python exp4i.py 0.5 2 1
# system arg
num_layers = 2
fac = 0.5
FAST_FLAG = 1
try:
	fac = float(sys.argv[1])
	num_layers = int(sys.argv[2])
	assert(num_layers >= 2 and num_layers <= 4)
	FAST_FLAG = int(sys.argv[3])
except Exception, e:
	print('-- {}'.format(e))
print('-- Decreasing factor = {}'.format(fac))
print('-- Number of layers = {}'.format(num_layers))
print('-- FAST FLAG: {}'.format(FAST_FLAG))

# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 4i: Early Stopping...')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if FAST_FLAG:
	filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

size_list = []
for num in range(num_layers + 1):
	size_list.append(int(total_features * np.power(fac, num)))
print('-- Layer sizes = {}'.format(size_list))

batch_size = 1000

num_epochs = 500
print_freq = 10
if FAST_FLAG:
	num_epochs = 5
	print_freq = 1

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

W_ae_list = [init_weight_variable([size_list[i], size_list[i + 1]]) \
	for i in range(num_layers)]
b_ae_list = [init_bias_variable([size_list[i + 1]])\
	for i in range(num_layers)]
a_list = [tf.nn.relu(batch_nm(tf.matmul(x, W_ae_list[0]) + b_ae_list[0]))]
for i in range(num_layers - 1):
	# batch normalization for
	a_i = tf.nn.relu(batch_nm(tf.matmul(a_list[-1], W_ae_list[i + 1]) + b_ae_list[i + 1]))
	a_list.append(a_i)

# dropout
keep_prob = tf.placeholder(tf.float32)
a_drop = tf.nn.dropout(a_list[-1], keep_prob)
W_sm = init_weight_variable([size_list[-1], num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(a_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_val = sess.run(y_val_OHEnc)[:, 0, :]

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# saver setup
varsave_list = W_ae_list + b_ae_list + [W_sm, b_sm]
saver = tf.train.Saver(varsave_list)
save_path = '4imodel_{}+{}.ckpt'.format(num_layers, fac)
opt_val_err = np.inf
opt_epoch = -1
earlystop_counter = 0
max_counter = 100

print('==> Training the full network...')
t_start = time.time()
for epoch in range(num_epochs):
	for i in range(0, num_training_vec, batch_size):
		batch_end_point = min(i + batch_size, num_training_vec)
		train_batch_data = X_train[i : batch_end_point]
		train_batch_label = y_train[i : batch_end_point]
		train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
	if (epoch + 1) % print_freq == 0:
		# evaluate metrics
		train_acc = accuracy.eval(feed_dict={x:train_batch_data, y_: train_batch_label, keep_prob: 1.0})
		train_acc_list.append(train_acc)
		val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_acc_list.append(val_acc)
		train_err = cross_entropy.eval(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 1.0})
		train_err_list.append(train_err)
		val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_err_list.append(val_err)
		print("-- epoch: %d, training error %g, validation error %g"%(epoch + 1, train_err, val_err))
		# save screenshot of the model
		if val_err < opt_val_err:
			step_counter = 0
			saver.save(sess, save_path)
			print('==> New optimal validation error found. Model saved.')
			opt_val_err, opt_epoch = val_err, epoch + 1
	if step_counter > max_counter:
		break
	step_counter += 1
t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Restore model & Evaluations
# ==============================================
saver.restore(sess, save_path)
print('==> Model restored to epoch {}'.format(opt_epoch))

train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train, keep_prob: 1.0})
val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
print('-- Training accuracy: {:.4f}'.format(train_acc))
print('-- Validation accuracy: {:.4f}'.format(val_acc))
print('-- Training error: {:.4E}'.format(train_err))
print('-- Validation error: {:.4E}'.format(val_err))

print('-- Training accuracy --')
print([float('{:.4f}'.format(x)) for x in train_acc_list])
print('-- Validation accuracy --')
print([float('{:.4f}'.format(x)) for x in val_acc_list])
print('-- Training error --')
print([float('{:.4E}'.format(x)) for x in train_err_list])
print('-- Validation error --')
print([float('{:.4E}'.format(x)) for x in val_err_list])

# print('==> Generating error plot...')
# x_list = range(0, print_freq * len(train_acc_list), print_freq)
# train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
# val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
# plt.xlabel('Number of epochs')
# plt.ylabel('Cross-Entropy Error')
# plt.title('Error vs Number of Epochs with {} Layers and Decreasing Factor {}'.format(num_layers, fac))
# plt.legend(loc='best')
# plt.savefig('exp4i_L{}F{}.png'.format(num_layers, fac), format='png')
# plt.close()

print('==> Finished!')
<|endoftext|>


<commit_before>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, ip,
                                    s_abranch, s_dependent, s_force, s_trigger,
                                    hash_upload_to_master,
                                    hash_download_from_master,
                                    clean_unpack, mkdir)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

CODEBASE = make_cb(['polli-sb', 'benchbuild', 'polli'])
FORCE_CODEBASE = make_new_cb(['polli-sb', 'benchbuild', 'polli'])

P = util.Property
BF = util.BuildFactory

ACCEPTED_BUILDERS = \
    slaves.get_hostlist(slaves.infosun,
                        predicate=lambda host: host["host"] in {'debussy', 'ligeti'})

def get_sb_debug_steps(c):
    return [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            ip('-DCMAKE_CXX_COMPILER=%(prop:cxx)s'),
            ip('-DCMAKE_C_COMPILER=%(prop:cc)s'),
            '-DCMAKE_BUILD_TYPE=Debug',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
    ]

def configure(c):
    sb_steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            ip('-DCMAKE_CXX_COMPILER=%(prop:cxx)s'),
            ip('-DCMAKE_C_COMPILER=%(prop:cc)s'),
            '-DCMAKE_BUILD_TYPE=Release',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    sb_steps.extend(upload_pj)

    download_pj = hash_download_from_master("public_html/polyjit_sb.tar.gz",
                                            "polyjit_sb.tar.gz", "polyjit")
    slurm_steps = [
        define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/"))
    ]
    slurm_steps.extend(download_pj)
    slurm_steps.extend(clean_unpack("polyjit_sb.tar.gz", "llvm"))
    slurm_steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),
    ])
    slurm_steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('bb_src', ip('%(prop:scratch)s/benchbuild')),

        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install',
            P("BENCHBUILD_ROOT")),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', env={
            'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
            'BB_TMP_DIR': '/scratch/pjtest/src/',
            'BB_TEST_DIR': P("testinputs"),
            'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
            'BB_SLURM_PARTITION': 'anywhere',
            'BB_SLURM_NODE_DIR': '/local/hdd/buildbot-polyjit/',
            'BB_SLURM_ACCOUNT': 'anywhere',
            'BB_SLURM_TIMELIMIT': '24:00:00',
            'BB_CONTAINER_MOUNTS': ip('["%(prop:llvm)s", "%(prop:bb_src)s"]'),
            'BB_CONTAINER_PREFIXES': '["/opt/benchbuild", "/", "/usr", "/usr/local"]',
            'BB_ENV_PATH':
                ip('["/scratch/pjtest/opt/papi-5.5.1/install/bin", "%(prop:llvm)s/bin", "/scratch/pjtest/erlent/build"]'),
            'BB_ENV_LD_LIBRARY_PATH':
                ip('["/scratch/pjtest/opt/papi-5.5.1/install/lib", "%(prop:llvm)s/lib", "/scratch/pjtest/erlent/build"]'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:bb_src)s'),
            'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
            'BB_UNIONFS_ENABLE': 'false'
            },
            workdir=P('scratch')),
        # This only works on infosun machines
        cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
            ip("%(prop:bb_src)s")),
        mkdir(ip("%(prop:scratch)s/results"))
    ])

    c['builders'].append(
        builder("polyjit-superbuild-debug", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(get_sb_debug_steps(c))))
    c['builders'].append(
        builder("polyjit-superbuild", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(sb_steps)))
    c['builders'].append(
        builder("polyjit-superbuild-slurm", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(slurm_steps)))

def schedule(c):
    superbuild_sched = s_abranch("bs_polyjit-superbuild",
                                 CODEBASE, [
                                     "polyjit-superbuild",
                                     "polyjit-superbuild-debug"],
                                 treeStableTimer=10 * 60)
    c['schedulers'].extend([
        superbuild_sched,
        s_force("fs_polyjit-superbuild", FORCE_CODEBASE,
                ["polyjit-superbuild"]),
        s_trigger("ts_polyjit-superbuild", CODEBASE,
                  ["polyjit-superbuild"]),

        s_force("fs_polyjit-superbuild-debug", FORCE_CODEBASE,
                ["polyjit-superbuild-debug"]),
        s_trigger("ts_polyjit-superbuild-debug", CODEBASE,
                  ["polyjit-superbuild-debug"]),

        s_dependent("ds_polyjit-superbuild-slurm",
                    superbuild_sched,
                    ["polyjit-superbuild-slurm"]),
        s_force("fs_polyjit-superbuild-slurm", FORCE_CODEBASE,
                ["polyjit-superbuild-slurm"])
    ])

register(sys.modules[__name__])
<commit_msg>polyjit-meta: use build-llvm in path<commit_after>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, ip,
                                    s_abranch, s_dependent, s_force, s_trigger,
                                    hash_upload_to_master,
                                    hash_download_from_master,
                                    clean_unpack, mkdir)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

CODEBASE = make_cb(['polli-sb', 'benchbuild', 'polli'])
FORCE_CODEBASE = make_new_cb(['polli-sb', 'benchbuild', 'polli'])

P = util.Property
BF = util.BuildFactory

ACCEPTED_BUILDERS = \
    slaves.get_hostlist(slaves.infosun,
                        predicate=lambda host: host["host"] in {'debussy', 'ligeti'})

def get_sb_debug_steps(c):
    return [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            ip('-DCMAKE_CXX_COMPILER=%(prop:cxx)s'),
            ip('-DCMAKE_C_COMPILER=%(prop:cc)s'),
            '-DCMAKE_BUILD_TYPE=Debug',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
    ]

def configure(c):
    sb_steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            ip('-DCMAKE_CXX_COMPILER=%(prop:cxx)s'),
            ip('-DCMAKE_C_COMPILER=%(prop:cc)s'),
            '-DCMAKE_BUILD_TYPE=Release',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            env={"LD_LIBRARY_PATH": P("llvm_libs")},
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    sb_steps.extend(upload_pj)

    download_pj = hash_download_from_master("public_html/polyjit_sb.tar.gz",
                                            "polyjit_sb.tar.gz", "polyjit")
    slurm_steps = [
        define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/"))
    ]
    slurm_steps.extend(download_pj)
    slurm_steps.extend(clean_unpack("polyjit_sb.tar.gz", "llvm"))
    slurm_steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),
    ])
    slurm_steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('bb_src', ip('%(prop:scratch)s/benchbuild')),

        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install',
            P("BENCHBUILD_ROOT")),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', env={
            'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
            'BB_TMP_DIR': '/scratch/pjtest/src/',
            'BB_TEST_DIR': P("testinputs"),
            'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
            'BB_SLURM_PARTITION': 'anywhere',
            'BB_SLURM_NODE_DIR': '/local/hdd/buildbot-polyjit/',
            'BB_SLURM_ACCOUNT': 'anywhere',
            'BB_SLURM_TIMELIMIT': '24:00:00',
            'BB_CONTAINER_MOUNTS': ip('["%(prop:llvm)s", "%(prop:bb_src)s"]'),
            'BB_CONTAINER_PREFIXES':
                '["/opt/benchbuild", '
                '"/", '
                '"/usr", '
                '"/usr/local"]',
            'BB_ENV_PATH':
                ip('["/scratch/pjtest/opt/papi-5.5.1/install/bin", '
                   '"%(prop:llvm)s/bin", '
                   '"%(prop:llvm_prefix)s/bin, '
                   '"/scratch/pjtest/erlent/build"]'),
            'BB_ENV_LD_LIBRARY_PATH':
                ip('["/scratch/pjtest/opt/papi-5.5.1/install/lib", '
                   '"%(prop:llvm)s/lib", '
                   '"%(prop:llvm_libs)s, '
                   '"/scratch/pjtest/erlent/build"]'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:bb_src)s'),
            'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
            'BB_UNIONFS_ENABLE': 'false'
            },
            workdir=P('scratch')),
        # This only works on infosun machines
        cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
            ip("%(prop:bb_src)s")),
        mkdir(ip("%(prop:scratch)s/results"))
    ])

    c['builders'].append(
        builder("polyjit-superbuild-debug", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(get_sb_debug_steps(c))))
    c['builders'].append(
        builder("polyjit-superbuild", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(sb_steps)))
    c['builders'].append(
        builder("polyjit-superbuild-slurm", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(slurm_steps)))

def schedule(c):
    superbuild_sched = s_abranch("bs_polyjit-superbuild",
                                 CODEBASE, [
                                     "polyjit-superbuild",
                                     "polyjit-superbuild-debug"],
                                 treeStableTimer=10 * 60)
    c['schedulers'].extend([
        superbuild_sched,
        s_force("fs_polyjit-superbuild", FORCE_CODEBASE,
                ["polyjit-superbuild"]),
        s_trigger("ts_polyjit-superbuild", CODEBASE,
                  ["polyjit-superbuild"]),

        s_force("fs_polyjit-superbuild-debug", FORCE_CODEBASE,
                ["polyjit-superbuild-debug"]),
        s_trigger("ts_polyjit-superbuild-debug", CODEBASE,
                  ["polyjit-superbuild-debug"]),

        s_dependent("ds_polyjit-superbuild-slurm",
                    superbuild_sched,
                    ["polyjit-superbuild-slurm"]),
        s_force("fs_polyjit-superbuild-slurm", FORCE_CODEBASE,
                ["polyjit-superbuild-slurm"])
    ])

register(sys.modules[__name__])
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os

from doit import get_var
from ruamel import yaml

from api.autocert.config import _update_config, CONFIG_YML, DOT_CONFIG_YML

DOIT_CONFIG = {
    'default_tasks': ['deploy', 'rmimages', 'rmvolumes'],
    'verbosity': 2,
}

USER = os.getenv('USER')

MINIMUM_DOCKER_COMPOSE_VERSION = '1.6'

LOG_LEVELS = [
    'DEBUG',
    'INFO',
    'WARNING',
    'ERROR',
    'CRITICAL',
]

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

def get_pkgmgr():
    def check_hash(program):
        from subprocess import check_call, CalledProcessError
        try:
            check_call('hash {program}'.format(**locals()), shell=True)
            return True
        except CalledProcessError as cpe:
            return False
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    raise UnknownPkgmgrError

def task_checkreqs():
    '''
    check for required software
    '''
    DEBS = [
        'docker-engine',
        #'libpq-dev',
    ]
    RPMS = [
        'docker-engine',
        #'postgresql-devel',
    ]
    return {
        'deb': {
            'actions': ['dpkg -s ' + deb for deb in DEBS],
        },
        'rpm': {
            'actions': ['rpm -q ' + rpm for rpm in RPMS],
        }
    }[get_pkgmgr()]

def task_dockercompose():
    '''
    assert docker-compose version ({0}) or higher
    '''
    from utils.function import format_docstr
    format_docstr(task_dockercompose, MINIMUM_DOCKER_COMPOSE_VERSION)
    def check_docker_compose():
        import re
        from subprocess import check_output
        from packaging.version import parse as version_parse
        pattern = '(docker-compose version) ([0-9.]+)(, build [0-9]+)'
        output = check_output('docker-compose --version', shell=True).decode('utf-8').strip()
        regex = re.compile(pattern)
        match = regex.search(output)
        version = match.groups()[1]
        assert version_parse(version) >= version_parse(MINIMUM_DOCKER_COMPOSE_VERSION)

    return {
        'actions': [
            check_docker_compose,
        ],
    }

def task_noroot():
    '''
    make sure script isn't run as root
    '''
    then = 'echo "   DO NOT RUN AS ROOT!"; echo; exit 1'
    bash = 'if [[ $(id -u) -eq 0 ]]; then {0}; fi'.format(then)
    return {
        'actions': [
            'bash -c \'{0}\''.format(bash),
        ],
    }

def task_test():
    '''
    setup venv and run pytest
    '''
    return {
        'task_dep': [
            'noroot',
            'config',
        ],
        'actions': [
            'virtualenv --python=$(which python3) venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip install -r api/requirements.txt',
            'venv/bin/pip install -r tests/requirements.txt',
            'venv/bin/pytest -v tests/',
        ],
    }

def task_version():
    '''
    write git describe to VERSION file
    '''
    return {
        'actions': [
            'git describe | xargs echo -n > api/VERSION',
        ],
    }

def task_deploy():
    '''
    deloy flask app via docker-compose
    '''
    return {
        'task_dep': [
            'noroot',
            'checkreqs',
            'version',
            'test',
            'config',
            'dockercompose'
        ],
        'actions': [
            'docker-compose build',
            'docker-compose up --remove-orphans -d',
        ],
    }

def task_rmimages():
    '''
    remove dangling docker images
    '''
    query = '`docker images -q -f dangling=true`'
    return {
        'actions': [
            'docker rmi {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_rmvolumes():
    '''
    remove dangling docker volumes
    '''
    query = '`docker volume ls -q -f dangling=true`'
    return {
        'actions': [
            'docker volume rm {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_config():
    '''
    write config.yml -> .config.yml
    '''
    log_level = 'WARNING'
    filename = '{0}/LOG_LEVEL'.format(os.path.dirname(__file__))
    if os.path.isfile(filename):
        log_level = open(filename).read().strip()
    log_level = get_var('LOG_LEVEL', log_level)
    if log_level not in LOG_LEVELS:
        raise UnknownLogLevelError(log_level)
    punch = '''
    logging:
        loggers:
            api:
                level: {log_level}
        handlers:
            console:
                level: {log_level}
    '''.format(**locals())
    return {
        'actions': [
            'echo "cp {CONFIG_YML}\n-> {DOT_CONFIG_YML}"'.format(**globals()),
            'echo "setting LOG_LEVEL={log_level}"'.format(**locals()),
            'cp {CONFIG_YML} {DOT_CONFIG_YML}'.format(**globals()),
            lambda: _update_config(DOT_CONFIG_YML, yaml.safe_load(punch)),
        ]
    }

def task_example():
    '''
    cp|strip config.yml -> config.yml.example
    '''
    apikey = '82_CHAR_APIKEY'
    punch = '''
    authorities:
        digicert:
            apikey: {apikey}
    destinations:
        zeus:
            apikey: {apikey}
    '''.format(**locals())
    return {
        'actions': [
            'cp {CONFIG_YML}.example {CONFIG_YML}.bak'.format(**globals()),
            'cp {CONFIG_YML} {CONFIG_YML}.example'.format(**globals()),
            lambda: _update_config(CONFIG_YML+'.example', yaml.safe_load(punch)),
        ],
    }

def task_tidy():
    '''
    delete cached files
    '''
    TIDY_FILES = [
        '.doit.db/',
        'venv/',
        'api/VERSION',
    ]
    return {
        'actions': [
            'rm -rf ' + ' '.join(TIDY_FILES),
            'find . | grep -E "(__pycache__|\.pyc$)" | xargs rm -rf',
        ],
    }

def task_nuke():
    '''
    git clean and reset
    '''
    return {
        'task_dep': ['tidy'],
        'actions': [
            'docker-compose kill',
            'docker-compose rm -f',
            'git clean -fd',
            'git reset --hard HEAD',
        ],
    }

def task_setup():
    '''
    setup venv
    '''
    from utils.version import version
    return {
        'actions': [
            'rm -rf auto_cert_cli.egg-info/ venv/ dist/ __pycache__/',
            'virtualenv --python=python3 venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip3 install -r cli/requirements.txt',
            'venv/bin/python3 ./setup.py install',
            'unzip -l venv/lib/python3.5/site-packages/auto_cert_cli-{0}-py3.5.egg'.format(version()),
        ],
    }

if __name__ == '__main__':
    print('should be run with doit installed')
    import doit
    doit.run(globals())
<commit_msg>added count task and run it by default if cloc is installed<commit_after>#!/usr/bin/env python3

import os

from doit import get_var
from ruamel import yaml

from api.autocert.config import _update_config, CONFIG_YML, DOT_CONFIG_YML

DOIT_CONFIG = {
    'default_tasks': ['deploy', 'rmimages', 'rmvolumes', 'count'],
    'verbosity': 2,
}

USER = os.getenv('USER')

MINIMUM_DOCKER_COMPOSE_VERSION = '1.6'

LOG_LEVELS = [
    'DEBUG',
    'INFO',
    'WARNING',
    'ERROR',
    'CRITICAL',
]

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

def check_hash(program):
    from subprocess import check_call, CalledProcessError
    try:
        check_call('hash {program}'.format(**locals()), shell=True)
        return True
    except CalledProcessError as cpe:
        return False

def get_pkgmgr():
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    raise UnknownPkgmgrError

def task_count():
    '''
    use the cloc utility to count lines of code
    '''
    excludes = [
        'dist',
        'venv',
        '__pycache__',
        'auto_cert_cli.egg-info',
    ]
    excludes = '--exclude-dir=' + ','.join(excludes)
    scandir = os.path.dirname(__file__)
    return {
        'actions': [
            'cloc {excludes} {scandir}'.format(**locals()),
        ],
        'uptodate': [
            lambda: not check_hash('cloc'),
        ],
    }

def task_checkreqs():
    '''
    check for required software
    '''
    DEBS = [
        'docker-engine',
        #'libpq-dev',
    ]
    RPMS = [
        'docker-engine',
        #'postgresql-devel',
    ]
    return {
        'deb': {
            'actions': ['dpkg -s ' + deb for deb in DEBS],
        },
        'rpm': {
            'actions': ['rpm -q ' + rpm for rpm in RPMS],
        }
    }[get_pkgmgr()]

def task_dockercompose():
    '''
    assert docker-compose version ({0}) or higher
    '''
    from utils.function import format_docstr
    format_docstr(task_dockercompose, MINIMUM_DOCKER_COMPOSE_VERSION)
    def check_docker_compose():
        import re
        from subprocess import check_output
        from packaging.version import parse as version_parse
        pattern = '(docker-compose version) ([0-9.]+)(, build [0-9]+)'
        output = check_output('docker-compose --version', shell=True).decode('utf-8').strip()
        regex = re.compile(pattern)
        match = regex.search(output)
        version = match.groups()[1]
        assert version_parse(version) >= version_parse(MINIMUM_DOCKER_COMPOSE_VERSION)

    return {
        'actions': [
            check_docker_compose,
        ],
    }

def task_noroot():
    '''
    make sure script isn't run as root
    '''
    then = 'echo "   DO NOT RUN AS ROOT!"; echo; exit 1'
    bash = 'if [[ $(id -u) -eq 0 ]]; then {0}; fi'.format(then)
    return {
        'actions': [
            'bash -c \'{0}\''.format(bash),
        ],
    }

def task_test():
    '''
    setup venv and run pytest
    '''
    return {
        'task_dep': [
            'noroot',
            'config',
        ],
        'actions': [
            'virtualenv --python=$(which python3) venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip install -r api/requirements.txt',
            'venv/bin/pip install -r tests/requirements.txt',
            'venv/bin/pytest -v tests/',
        ],
    }

def task_version():
    '''
    write git describe to VERSION file
    '''
    return {
        'actions': [
            'git describe | xargs echo -n > api/VERSION',
        ],
    }

def task_deploy():
    '''
    deloy flask app via docker-compose
    '''
    return {
        'task_dep': [
            'noroot',
            'checkreqs',
            'version',
            'test',
            'config',
            'dockercompose'
        ],
        'actions': [
            'docker-compose build',
            'docker-compose up --remove-orphans -d',
        ],
    }

def task_rmimages():
    '''
    remove dangling docker images
    '''
    query = '`docker images -q -f dangling=true`'
    return {
        'actions': [
            'docker rmi {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_rmvolumes():
    '''
    remove dangling docker volumes
    '''
    query = '`docker volume ls -q -f dangling=true`'
    return {
        'actions': [
            'docker volume rm {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_config():
    '''
    write config.yml -> .config.yml
    '''
    log_level = 'WARNING'
    filename = '{0}/LOG_LEVEL'.format(os.path.dirname(__file__))
    if os.path.isfile(filename):
        log_level = open(filename).read().strip()
    log_level = get_var('LOG_LEVEL', log_level)
    if log_level not in LOG_LEVELS:
        raise UnknownLogLevelError(log_level)
    punch = '''
    logging:
        loggers:
            api:
                level: {log_level}
        handlers:
            console:
                level: {log_level}
    '''.format(**locals())
    return {
        'actions': [
            'echo "cp {CONFIG_YML}\n-> {DOT_CONFIG_YML}"'.format(**globals()),
            'echo "setting LOG_LEVEL={log_level}"'.format(**locals()),
            'cp {CONFIG_YML} {DOT_CONFIG_YML}'.format(**globals()),
            lambda: _update_config(DOT_CONFIG_YML, yaml.safe_load(punch)),
        ]
    }

def task_example():
    '''
    cp|strip config.yml -> config.yml.example
    '''
    apikey = '82_CHAR_APIKEY'
    punch = '''
    authorities:
        digicert:
            apikey: {apikey}
    destinations:
        zeus:
            apikey: {apikey}
    '''.format(**locals())
    return {
        'actions': [
            'cp {CONFIG_YML}.example {CONFIG_YML}.bak'.format(**globals()),
            'cp {CONFIG_YML} {CONFIG_YML}.example'.format(**globals()),
            lambda: _update_config(CONFIG_YML+'.example', yaml.safe_load(punch)),
        ],
    }

def task_tidy():
    '''
    delete cached files
    '''
    TIDY_FILES = [
        '.doit.db/',
        'venv/',
        'api/VERSION',
    ]
    return {
        'actions': [
            'rm -rf ' + ' '.join(TIDY_FILES),
            'find . | grep -E "(__pycache__|\.pyc$)" | xargs rm -rf',
        ],
    }

def task_nuke():
    '''
    git clean and reset
    '''
    return {
        'task_dep': ['tidy'],
        'actions': [
            'docker-compose kill',
            'docker-compose rm -f',
            'git clean -fd',
            'git reset --hard HEAD',
        ],
    }

def task_setup():
    '''
    setup venv
    '''
    from utils.version import version
    return {
        'actions': [
            'rm -rf auto_cert_cli.egg-info/ venv/ dist/ __pycache__/',
            'virtualenv --python=python3 venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip3 install -r cli/requirements.txt',
            'venv/bin/python3 ./setup.py install',
            'unzip -l venv/lib/python3.5/site-packages/auto_cert_cli-{0}-py3.5.egg'.format(version()),
        ],
    }

if __name__ == '__main__':
    print('should be run with doit installed')
    import doit
    doit.run(globals())
<|endoftext|>


<commit_before>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, mkdir,
                                    cmd, ucmd, ucompile, ip,
                                    s_nightly, s_sbranch, s_force, s_trigger,
                                    hash_upload_to_master)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

codebase = make_cb(['polli-sb'])
force_codebase = make_new_cb(['polli-sb'])

P = util.Property
BuildFactory = util.BuildFactory
accepted_builders = slaves.get_hostlist(slaves.infosun)

# yapf: disable
def configure(c):
    steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        # FIXME Testing:
        define("POLYJIT_DEFAULT_BRANCH", "WIP-merge-next"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        mkdir("src"),
        ucmd('cmake', P("UCHROOT_SUPERBUILD_ROOT"),
             '-DCMAKE_BUILD_TYPE=Release',
             '-DCMAKE_INSTALL_PREFIX=./_install',
             ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             '-G', 'Ninja',
             workdir='src',
             env={
                 "PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin"
             },
             name="cmake",
             description="[uchroot] cmake: release build",
             descriptionDone="[uchroot] configured."),
        ucompile("ninja", "install",
                 haltOnFailure=True, name="build jit",
                 description="[uchroot] building PolyJIT",
                 descriptionDone="[uchroot] built PolyJIT",
                 timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    steps.extend(upload_pj)

    c['builders'].append(
        builder("polyjit-superbuild", None,
                accepted_builders,
                tags=['polyjit'], factory=BuildFactory(steps)))
# yapf: enable

def schedule(c):
    c['schedulers'].extend([
        s_sbranch("branch-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"],
                  change_filter=filter.ChangeFilter(branch_re='master')),
        s_force("force-sched-polyjit-superbuild", force_codebase,
                ["polyjit-superbuild"]),
        s_trigger("trigger-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"]),
        s_nightly("nightly-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"],
                  hour=20, minute=0)
    ])

register(sys.modules[__name__])
<commit_msg>Remove fixme<commit_after>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, mkdir,
                                    cmd, ucmd, ucompile, ip,
                                    s_nightly, s_sbranch, s_force, s_trigger,
                                    hash_upload_to_master)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

codebase = make_cb(['polli-sb'])
force_codebase = make_new_cb(['polli-sb'])

P = util.Property
BuildFactory = util.BuildFactory
accepted_builders = slaves.get_hostlist(slaves.infosun)

# yapf: disable
def configure(c):
    steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "WIP-merge-next"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        mkdir("src"),
        ucmd('cmake', P("UCHROOT_SUPERBUILD_ROOT"),
             '-DCMAKE_BUILD_TYPE=Release',
             '-DCMAKE_INSTALL_PREFIX=./_install',
             ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)'),
             '-G', 'Ninja',
             workdir='src',
             env={
                 "PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin"
             },
             name="cmake",
             description="[uchroot] cmake: release build",
             descriptionDone="[uchroot] configured."),
        ucompile("ninja", "install",
                 haltOnFailure=True, name="build jit",
                 description="[uchroot] building PolyJIT",
                 descriptionDone="[uchroot] built PolyJIT",
                 timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    steps.extend(upload_pj)

    c['builders'].append(
        builder("polyjit-superbuild", None,
                accepted_builders,
                tags=['polyjit'], factory=BuildFactory(steps)))
# yapf: enable

def schedule(c):
    c['schedulers'].extend([
        s_sbranch("branch-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"],
                  change_filter=filter.ChangeFilter(branch_re='master')),
        s_force("force-sched-polyjit-superbuild", force_codebase,
                ["polyjit-superbuild"]),
        s_trigger("trigger-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"]),
        s_nightly("nightly-sched-polyjit-superbuild", codebase,
                  ["polyjit-superbuild"],
                  hour=20, minute=0)
    ])

register(sys.modules[__name__])
<|endoftext|>


<commit_before># Copyright 2021 Google LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base Configuration."""

import ml_collections

def get_config():
  """Get the default hyperparameter configuration."""
  config = ml_collections.ConfigDict()
  config.batch_size = 32
  config.eval_frequency = 20
  config.num_train_steps = 5000
  config.num_eval_steps = 20
  config.learning_rate = 0.05
  config.weight_decay = 1e-1
  config.max_target_length = 200  # ignored
  config.max_eval_target_length = 200  # ignored
  config.sampling_temperature = 0.6
  config.sampling_top_k = 20
  config.max_predict_token_length = 50
  config.save_checkpoints = True
  config.restore_checkpoints = True
  config.checkpoint_freq = 10000
  config.random_seed = 0
  config.prompt = ""
  config.factors = "constant * linear_warmup * rsqrt_decay"
  config.warmup = 1000
  config.max_length = 2000
  config.tied_weights = True

  config.pooling_mode = "CLS"
  # config.interaction = "NLI"

  config.emb_dim = 512
  config.num_heads = 8
  config.num_layers = 6
  config.qkv_dim = 512
  config.mlp_dim = 2048

  config.trial = 0  # dummy for repeated runs.
  return config
<commit_msg>Internal change<commit_after># Copyright 2021 Google LLC

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     https://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base Configuration."""

import ml_collections

def get_config():
  """Get the default hyperparameter configuration."""
  config = ml_collections.ConfigDict()
  config.batch_size = 32
  config.eval_frequency = 50
  config.num_train_steps = 5000
  config.num_eval_steps = 99999
  config.learning_rate = 0.05
  config.weight_decay = 1e-1
  config.max_target_length = 200  # ignored
  config.max_eval_target_length = 200  # ignored
  config.sampling_temperature = 0.6
  config.sampling_top_k = 20
  config.max_predict_token_length = 50
  config.save_checkpoints = True
  config.restore_checkpoints = True
  config.checkpoint_freq = 10000
  config.random_seed = 0
  config.prompt = ""
  config.factors = "constant * linear_warmup * rsqrt_decay"
  config.warmup = 1000
  config.max_length = 2000
  config.tied_weights = False

  config.pooling_mode = "CLS"

  config.emb_dim = 512
  config.num_heads = 8
  config.num_layers = 4
  config.qkv_dim = 512
  config.mlp_dim = 1024

  config.trial = 0  # dummy for repeated runs.
  return config
<|endoftext|>


<commit_before>import sys
import os.path
from tempfile import TemporaryDirectory
import tarfile
import json
import subprocess
import shutil
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent / 'src'))

from elm_doc import elm_platform, elm_project, tasks
from tests import conftest

def task_create_elm_core_fixture():
    elm_core_fixture_path = conftest.elm_core_fixture_path()
    elm_versions = ['0.19.0']
    for elm_version in elm_versions:
        tarball = str(elm_core_fixture_path(elm_version))
        yield {
            'basename': 'create_elm_core_fixutre:' + elm_version,
            'actions': [(_create_elm_core_fixture, (elm_version, tarball))],
            'targets': [tarball],
            'uptodate': [True],
        }

def _create_elm_core_fixture(elm_version: str, tarball: str):
    os.makedirs(os.path.dirname(tarball), exist_ok=True)
    workspace_path = Path(__file__).parent / 'workspace'
    # clear elm-stuff because something in there causes CORRUPT BINARY
    # if you run 'elm make' with a different ELM_HOME
    shutil.rmtree(workspace_path / elm_project.STUFF_DIRECTORY)
    with TemporaryDirectory() as tmpdir:
        root_path = Path(tmpdir)
        elm_platform.install(root_path, elm_version)
        elm_path = root_path / 'node_modules' / '.bin' / 'elm'
        elm_home_path = root_path / '.elm'
        try:
            subprocess.run(
                [elm_path, 'make', workspace_path / 'src' / 'Main.elm'],
                env=dict(os.environ, **{'ELM_HOME': str(elm_home_path)}),
                cwd=workspace_path,
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print('\nSTDOUT:\n' + e.stdout.decode('utf8'))
            print('\nSTDERR:\n' + e.stderr.decode('utf8'))
            raise e

        with tarfile.open(tarball, "w:gz") as tar:
            tar.add(str(elm_home_path), arcname=elm_home_path.name)

def task_create_package_elm_lang_org_artifact_tarball():
    return {
        'file_dep': [
            'build/package.elm-lang.org/artifacts/elm.js',
            'vendor/package.elm-lang.org/assets/favicon.ico',
            'vendor/package.elm-lang.org/assets/highlight/highlight.pack.js',
            'vendor/package.elm-lang.org/assets/highlight/LICENSE',
            'vendor/package.elm-lang.org/assets/highlight/styles/default.css',
            'vendor/package.elm-lang.org/assets/style.css',
            'vendor/package.elm-lang.org/assets/help/documentation-format.md',
            'vendor/package.elm-lang.org/assets/help/design-guidelines.md',
        ],
        'targets': [
            'build/assets.tar.gz',
        ],
        'actions': [
            'tar cf %(targets)s -C build/package.elm-lang.org/ artifacts',
            'tar rf %(targets)s -C vendor/package.elm-lang.org/ assets',
        ]
    }

def task_package_elm_lang_org_elm_js():
    root_path = Path(__file__).parent.joinpath('vendor', 'package.elm-lang.org')
    output_path = Path(__file__).parent.joinpath('build', 'package.elm-lang.org', 'artifacts', 'elm.js')
    return {
        'file_dep': list(root_path.glob('src/frontend/**/*.elm')) + [root_path / 'elm.json'],
        'targets': [
            'build/package.elm-lang.org/artifacts/elm.js',
        ],
        'actions': [(_create_package_elm_lang_org_elm_js, (output_path,))]
    }

def _read_elm_version(elm_json_path: Path) -> str:
    with open(elm_json_path) as f:
        elm_json = json.load(f)
    return elm_json['elm-version']

def _create_package_elm_lang_org_elm_js(output_path: Path):
    repo_path = Path(__file__).parent / 'vendor' / 'package.elm-lang.org'
    elm_version = _read_elm_version(repo_path / 'elm.json')
    with TemporaryDirectory() as tmpdir:
        root_path = Path(tmpdir)
        elm_platform.install(root_path, elm_version)
        elm_path = root_path / 'node_modules' / '.bin' / 'elm'
        try:
            subprocess.run(
                [
                    elm_path,
                    'make',
                    repo_path / 'src' / 'frontend' / 'Main.elm',
                    '--output',
                    str(output_path),
                ],
                cwd=repo_path,
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print('\nSTDOUT:\n' + e.stdout.decode('utf8'))
            print('\nSTDERR:\n' + e.stderr.decode('utf8'))
            raise e

def task_build_workspace_docs():
    workspace_path = Path(__file__).parent / 'workspace'
    output_path = workspace_path / 'build' / 'docs'
    elm_path = workspace_path / 'node_modules' / '.bin' / 'elm'
    config = elm_project.ProjectConfig()
    yield {
        'name': 'install_elm',
        'file_dep': [workspace_path / 'elm.json'],
        'targets': [elm_path],
        'actions': [(_install_elm, (workspace_path,))]
    }
    for task in tasks.create_tasks(
            workspace_path,
            config,
            output_path,
            elm_path=elm_path,
            mount_point='/docs',
    ):
        yield task

def _install_elm(project_path: Path):
    elm_version = _read_elm_version(project_path / 'elm.json')
    elm_platform.install(project_path, elm_version)
<commit_msg>Use double-star globbing as documented in the docs<commit_after>import sys
import os.path
from tempfile import TemporaryDirectory
import tarfile
import json
import subprocess
import shutil
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent / 'src'))

from elm_doc import elm_platform, elm_project, tasks
from tests import conftest

def task_create_elm_core_fixture():
    elm_core_fixture_path = conftest.elm_core_fixture_path()
    elm_versions = ['0.19.0']
    for elm_version in elm_versions:
        tarball = str(elm_core_fixture_path(elm_version))
        yield {
            'basename': 'create_elm_core_fixutre:' + elm_version,
            'actions': [(_create_elm_core_fixture, (elm_version, tarball))],
            'targets': [tarball],
            'uptodate': [True],
        }

def _create_elm_core_fixture(elm_version: str, tarball: str):
    os.makedirs(os.path.dirname(tarball), exist_ok=True)
    workspace_path = Path(__file__).parent / 'workspace'
    # clear elm-stuff because something in there causes CORRUPT BINARY
    # if you run 'elm make' with a different ELM_HOME
    shutil.rmtree(workspace_path / elm_project.STUFF_DIRECTORY)
    with TemporaryDirectory() as tmpdir:
        root_path = Path(tmpdir)
        elm_platform.install(root_path, elm_version)
        elm_path = root_path / 'node_modules' / '.bin' / 'elm'
        elm_home_path = root_path / '.elm'
        try:
            subprocess.run(
                [elm_path, 'make', workspace_path / 'src' / 'Main.elm'],
                env=dict(os.environ, **{'ELM_HOME': str(elm_home_path)}),
                cwd=workspace_path,
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print('\nSTDOUT:\n' + e.stdout.decode('utf8'))
            print('\nSTDERR:\n' + e.stderr.decode('utf8'))
            raise e

        with tarfile.open(tarball, "w:gz") as tar:
            tar.add(str(elm_home_path), arcname=elm_home_path.name)

def task_create_package_elm_lang_org_artifact_tarball():
    return {
        'file_dep': [
            'build/package.elm-lang.org/artifacts/elm.js',
            'vendor/package.elm-lang.org/assets/favicon.ico',
            'vendor/package.elm-lang.org/assets/highlight/highlight.pack.js',
            'vendor/package.elm-lang.org/assets/highlight/LICENSE',
            'vendor/package.elm-lang.org/assets/highlight/styles/default.css',
            'vendor/package.elm-lang.org/assets/style.css',
            'vendor/package.elm-lang.org/assets/help/documentation-format.md',
            'vendor/package.elm-lang.org/assets/help/design-guidelines.md',
        ],
        'targets': [
            'build/assets.tar.gz',
        ],
        'actions': [
            'tar cf %(targets)s -C build/package.elm-lang.org/ artifacts',
            'tar rf %(targets)s -C vendor/package.elm-lang.org/ assets',
        ]
    }

def task_package_elm_lang_org_elm_js():
    root_path = Path(__file__).parent.joinpath('vendor', 'package.elm-lang.org')
    output_path = Path(__file__).parent.joinpath('build', 'package.elm-lang.org', 'artifacts', 'elm.js')
    return {
        'file_dep': list(root_path.joinpath('src/frontend').glob('**/*.elm')) + [root_path / 'elm.json'],
        'targets': [
            'build/package.elm-lang.org/artifacts/elm.js',
        ],
        'actions': [(_create_package_elm_lang_org_elm_js, (output_path,))]
    }

def _read_elm_version(elm_json_path: Path) -> str:
    with open(elm_json_path) as f:
        elm_json = json.load(f)
    return elm_json['elm-version']

def _create_package_elm_lang_org_elm_js(output_path: Path):
    repo_path = Path(__file__).parent / 'vendor' / 'package.elm-lang.org'
    elm_version = _read_elm_version(repo_path / 'elm.json')
    with TemporaryDirectory() as tmpdir:
        root_path = Path(tmpdir)
        elm_platform.install(root_path, elm_version)
        elm_path = root_path / 'node_modules' / '.bin' / 'elm'
        try:
            subprocess.run(
                [
                    elm_path,
                    'make',
                    repo_path / 'src' / 'frontend' / 'Main.elm',
                    '--output',
                    str(output_path),
                ],
                cwd=repo_path,
                check=True,
                capture_output=True,
            )
        except subprocess.CalledProcessError as e:
            print('\nSTDOUT:\n' + e.stdout.decode('utf8'))
            print('\nSTDERR:\n' + e.stderr.decode('utf8'))
            raise e

def task_build_workspace_docs():
    workspace_path = Path(__file__).parent / 'workspace'
    output_path = workspace_path / 'build' / 'docs'
    elm_path = workspace_path / 'node_modules' / '.bin' / 'elm'
    config = elm_project.ProjectConfig()
    yield {
        'name': 'install_elm',
        'file_dep': [workspace_path / 'elm.json'],
        'targets': [elm_path],
        'actions': [(_install_elm, (workspace_path,))]
    }
    for task in tasks.create_tasks(
            workspace_path,
            config,
            output_path,
            elm_path=elm_path,
            mount_point='/docs',
    ):
        yield task

def _install_elm(project_path: Path):
    elm_version = _read_elm_version(project_path / 'elm.json')
    elm_platform.install(project_path, elm_version)
<|endoftext|>