<commit_before># Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""Define some helpers methods to create microvms from artifacts."""

import json
import os
import shutil
import tempfile
from pathlib import Path
from conftest import init_microvm, _test_images_s3_bucket
from framework.defs import DEFAULT_TEST_SESSION_ROOT_PATH
from framework.artifacts import (
    ArtifactCollection, Artifact, DiskArtifact, Snapshot,
    SnapshotType, NetIfaceConfig
)
from framework import utils
import host_tools.logging as log_tools
import host_tools.network as net_tools

class VmInstance:
    """A class that describes a microvm instance resources."""

    def __init__(self, config, kernel, disks, ssh_key, vm):
        """Initialize a Vm configuration based on artifacts."""
        self._config = config
        self._kernel = kernel
        self._disks = disks
        self._ssh_key = ssh_key
        self._vm = vm

    @property
    def config(self):
        """Return machine config artifact."""
        return self._config

    @property
    def kernel(self):
        """Return the kernel artifact."""
        return self._kernel

    @property
    def disks(self):
        """Return an array of block file paths."""
        return self._disks

    @property
    def ssh_key(self):
        """Return ssh key artifact linked to the root block device."""
        return self._ssh_key

    @property
    def vm(self):
        """Return the Microvm object instance."""
        return self._vm

class MicrovmBuilder:
    """Build fresh microvms or restore from snapshot."""

    ROOT_PREFIX = "fctest-"

    _root_path = None

    def __init__(self, bin_cloner_path):
        """Initialize microvm root and cloning binary."""
        self.bin_cloner_path = bin_cloner_path
        self.init_root_path()

    @property
    def root_path(self):
        """Return the root path of the microvm."""
        return self._root_path

    def init_root_path(self):
        """Initialize microvm root path."""
        self._root_path = tempfile.mkdtemp(
            prefix=MicrovmBuilder.ROOT_PREFIX,
            dir=f"{DEFAULT_TEST_SESSION_ROOT_PATH}")

    def build(self,
              kernel: Artifact,
              disks: [DiskArtifact],
              ssh_key: Artifact,
              config: Artifact,
              net_ifaces=None,
              diff_snapshots=False,
              cpu_template=None,
              fc_binary=None,
              jailer_binary=None,
              use_ramdisk=False,
              smt=None):
        """Build a fresh microvm."""
        vm = init_microvm(self.root_path, self.bin_cloner_path,
                          fc_binary, jailer_binary)

        # Start firecracker.
        vm.spawn(use_ramdisk=use_ramdisk)

        # Link the microvm to kernel, rootfs, ssh_key artifacts.
        vm.kernel_file = kernel.local_path()
        vm.rootfs_file = disks[0].local_path()
        # copy rootfs to ramdisk if needed
        jailed_rootfs_path = vm.copy_to_jail_ramfs(vm.rootfs_file) if \
            use_ramdisk else vm.create_jailed_resource(vm.rootfs_file)

        # Download ssh key into the microvm root.
        ssh_key.download(vm.path)
        vm.ssh_config['ssh_key_path'] = ssh_key.local_path()
        os.chmod(vm.ssh_config['ssh_key_path'], 0o400)

        # Provide a default network configuration.
        if net_ifaces is None or len(net_ifaces) == 0:
            ifaces = [NetIfaceConfig()]
        else:
            ifaces = net_ifaces

        # Configure network interfaces using artifacts.
        for iface in ifaces:
            vm.create_tap_and_ssh_config(host_ip=iface.host_ip,
                                         guest_ip=iface.guest_ip,
                                         netmask_len=iface.netmask,
                                         tapname=iface.tap_name)
            guest_mac = net_tools.mac_from_ip(iface.guest_ip)
            response = vm.network.put(
                iface_id=iface.dev_name,
                host_dev_name=iface.tap_name,
                guest_mac=guest_mac
            )
            assert vm.api_session.is_status_no_content(response.status_code)

        with open(config.local_path(), encoding='utf-8') as \
                microvm_config_file:
            microvm_config = json.load(microvm_config_file)

        response = vm.basic_config(
            add_root_device=False,
            boot_args='console=ttyS0 reboot=k panic=1'
        )

        # Add the root file system with rw permissions.
        response = vm.drive.put(
            drive_id='rootfs',
            path_on_host=jailed_rootfs_path,
            is_root_device=True,
            is_read_only=False
        )
        assert vm.api_session \
            .is_status_no_content(response.status_code), \
            response.text

        # Apply the microvm artifact configuration and template.
        response = vm.machine_cfg.put(
            vcpu_count=int(microvm_config['vcpu_count']),
            mem_size_mib=int(microvm_config['mem_size_mib']),
            smt=smt,
            track_dirty_pages=diff_snapshots,
            cpu_template=cpu_template,
        )
        assert vm.api_session.is_status_no_content(response.status_code)

        vm.vcpus_count = int(microvm_config['vcpu_count'])

        return VmInstance(config, kernel, disks, ssh_key, vm)

    # This function currently returns the vm and a metrics_fifo which
    # is needed by the performance integration tests.
    # TODO: Move all metrics functionality to microvm (encapsulating the fifo)
    # so we do not need to move it around polluting the code.
    def build_from_snapshot(self,
                            snapshot: Snapshot,
                            resume=False,
                            # Enable incremental snapshot capability.
                            diff_snapshots=False,
                            use_ramdisk=False,
                            fc_binary=None, jailer_binary=None):
        """Build a microvm from a snapshot artifact."""
        vm = init_microvm(self.root_path, self.bin_cloner_path,
                          fc_binary, jailer_binary,)
        vm.spawn(log_level='Error', use_ramdisk=use_ramdisk)
        vm.api_session.untime()

        metrics_file_path = os.path.join(vm.path, 'metrics.log')
        metrics_fifo = log_tools.Fifo(metrics_file_path)
        response = vm.metrics.put(
            metrics_path=vm.create_jailed_resource(metrics_fifo.path)
        )
        assert vm.api_session.is_status_no_content(response.status_code)

        # Hardlink all the snapshot files into the microvm jail.
        jailed_mem = vm.copy_to_jail_ramfs(snapshot.mem) if use_ramdisk else \
            vm.create_jailed_resource(snapshot.mem)
        jailed_vmstate = vm.copy_to_jail_ramfs(snapshot.vmstate) \
            if use_ramdisk else vm.create_jailed_resource(snapshot.vmstate)

        assert len(snapshot.disks) > 0, "Snapshot requires at least one disk."
        _jailed_disks = []
        for disk in snapshot.disks:
            _jailed_disks.append(vm.copy_to_jail_ramfs(disk) if use_ramdisk
                                 else vm.create_jailed_resource(disk))

        vm.ssh_config['ssh_key_path'] = snapshot.ssh_key.local_path()

        # Create network interfaces.
        for iface in snapshot.net_ifaces:
            vm.create_tap_and_ssh_config(host_ip=iface.host_ip,
                                         guest_ip=iface.guest_ip,
                                         netmask_len=iface.netmask,
                                         tapname=iface.tap_name)
        response = vm.snapshot.load(mem_file_path=jailed_mem,
                                    snapshot_path=jailed_vmstate,
                                    diff=diff_snapshots,
                                    resume=resume)
        status_ok = vm.api_session.is_status_no_content(response.status_code)

        # Verify response status and cleanup if needed before assert.
        if not status_ok:
            # Destroy VM here before we assert.
            vm.kill()
            del vm

        assert status_ok, response.text

        # Return a resumed microvm.
        return vm, metrics_fifo

    def build_from_artifacts(self, config,
                             kernel, disks, cpu_template,
                             net_ifaces=None, diff_snapshots=False,
                             fc_binary=None, jailer_binary=None):
        """Spawns a new Firecracker and applies specified config."""
        artifacts = ArtifactCollection(_test_images_s3_bucket())
        # Pick the first artifact in the set.
        config = artifacts.microvms(keyword=config)[0]
        kernel = artifacts.kernels(keyword=kernel)[0]
        disks = artifacts.disks(keyword=disks)
        config.download()
        kernel.download()
        attached_disks = []
        for disk in disks:
            disk.download()
            attached_disks.append(disk.copy())

        # SSH key is attached to root disk artifact.
        # Builder will download ssh key in the VM root.
        ssh_key = disks[0].ssh_key()
        # Create a fresh microvm from artifacts.
        return self.build(kernel=kernel,
                          disks=attached_disks,
                          ssh_key=ssh_key,
                          config=config,
                          net_ifaces=net_ifaces,
                          diff_snapshots=diff_snapshots,
                          cpu_template=cpu_template,
                          fc_binary=fc_binary,
                          jailer_binary=jailer_binary)

    def build_vm_nano(self, fc_binary=None, jailer_binary=None,
                      net_ifaces=None, diff_snapshots=False):
        """Create a clean VM in an initial state."""
        return self.build_from_artifacts("2vcpu_256mb",
                                         "vmlinux-4.14",
                                         "ubuntu-18.04",
                                         None,
                                         net_ifaces=net_ifaces,
                                         diff_snapshots=diff_snapshots,
                                         fc_binary=fc_binary,
                                         jailer_binary=jailer_binary)

    def build_vm_micro(self, fc_binary=None, jailer_binary=None,
                       net_ifaces=None, diff_snapshots=False):
        """Create a clean VM in an initial state."""
        return self.build_from_artifacts("2vcpu_512mb",
                                         "vmlinux-4.14",
                                         "ubuntu-18.04",
                                         None,
                                         net_ifaces=net_ifaces,
                                         diff_snapshots=diff_snapshots,
                                         fc_binary=fc_binary,
                                         jailer_binary=jailer_binary)

    def cleanup(self):
        """Clean up this builder context."""
        if self._root_path:
            shutil.rmtree(self._root_path, ignore_errors=True)

    def __del__(self):
        """Teardown the object."""
        self.cleanup()

class SnapshotBuilder:  # pylint: disable=too-few-public-methods
    """Create a snapshot from a running microvm."""

    def __init__(self, microvm):
        """Initialize the snapshot builder."""
        self._microvm = microvm

    def create_snapshot_dir(self):
        """Create dir and files for saving snapshot state and memory."""
        chroot_path = self._microvm.jailer.chroot_path()
        snapshot_dir = os.path.join(chroot_path, "snapshot")
        Path(snapshot_dir).mkdir(parents=True, exist_ok=True)
        cmd = 'chown {}:{} {}'.format(self._microvm.jailer.uid,
                                      self._microvm.jailer.gid,
                                      snapshot_dir)
        utils.run_cmd(cmd)
        return snapshot_dir

    def create(self,
               disks,
               ssh_key: Artifact,
               snapshot_type: SnapshotType = SnapshotType.FULL,
               target_version: str = None,
               mem_file_name: str = "vm.mem",
               snapshot_name: str = "vm.vmstate",
               net_ifaces=None,
               use_ramdisk=False):
        """Create a Snapshot object from a microvm and artifacts."""
        if use_ramdisk:
            snaps_dir = self._microvm.jailer.chroot_ramfs_path()
            mem_full_path = os.path.join(snaps_dir, mem_file_name)
            vmstate_full_path = os.path.join(snaps_dir, snapshot_name)

            memsize = self._microvm.machine_cfg.configuration['mem_size_mib']
            # Pre-allocate ram for memfile to eliminate allocation variability.
            utils.run_cmd('dd if=/dev/zero of={} bs=1M count={}'.format(
                mem_full_path, memsize
            ))
            cmd = 'chown {}:{} {}'.format(
                self._microvm.jailer.uid,
                self._microvm.jailer.gid,
                mem_full_path
            )
            utils.run_cmd(cmd)
        else:
            snaps_dir = self.create_snapshot_dir()
            mem_full_path = os.path.join(snaps_dir, mem_file_name)
            vmstate_full_path = os.path.join(snaps_dir, snapshot_name)

        snaps_dir_name = os.path.basename(snaps_dir)
        self._microvm.pause_to_snapshot(
            mem_file_path=os.path.join('/', snaps_dir_name, mem_file_name),
            snapshot_path=os.path.join('/', snaps_dir_name, snapshot_name),
            diff=snapshot_type == SnapshotType.DIFF,
            version=target_version)

        # Create a copy of the ssh_key artifact.
        ssh_key_copy = ssh_key.copy()
        return Snapshot(mem=mem_full_path,
                        vmstate=vmstate_full_path,
                        # TODO: To support more disks we need to figure out a
                        # simple and flexible way to store snapshot artifacts
                        # in S3. This should be done in a PR where we add tests
                        # that resume from S3 snapshot artifacts.
                        disks=disks,
                        net_ifaces=net_ifaces or [NetIfaceConfig()],
                        ssh_key=ssh_key_copy)
<commit_msg>[CI]: Added option to start vm undemonized<commit_after># Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""Define some helpers methods to create microvms from artifacts."""

import json
import os
import shutil
import tempfile
from pathlib import Path
from conftest import init_microvm, _test_images_s3_bucket
from framework.defs import DEFAULT_TEST_SESSION_ROOT_PATH
from framework.artifacts import (
    ArtifactCollection, Artifact, DiskArtifact, Snapshot,
    SnapshotType, NetIfaceConfig
)
from framework import utils
import host_tools.logging as log_tools
import host_tools.network as net_tools

class VmInstance:
    """A class that describes a microvm instance resources."""

    def __init__(self, config, kernel, disks, ssh_key, vm):
        """Initialize a Vm configuration based on artifacts."""
        self._config = config
        self._kernel = kernel
        self._disks = disks
        self._ssh_key = ssh_key
        self._vm = vm

    @property
    def config(self):
        """Return machine config artifact."""
        return self._config

    @property
    def kernel(self):
        """Return the kernel artifact."""
        return self._kernel

    @property
    def disks(self):
        """Return an array of block file paths."""
        return self._disks

    @property
    def ssh_key(self):
        """Return ssh key artifact linked to the root block device."""
        return self._ssh_key

    @property
    def vm(self):
        """Return the Microvm object instance."""
        return self._vm

class MicrovmBuilder:
    """Build fresh microvms or restore from snapshot."""

    ROOT_PREFIX = "fctest-"

    _root_path = None

    def __init__(self, bin_cloner_path):
        """Initialize microvm root and cloning binary."""
        self.bin_cloner_path = bin_cloner_path
        self.init_root_path()

    @property
    def root_path(self):
        """Return the root path of the microvm."""
        return self._root_path

    def init_root_path(self):
        """Initialize microvm root path."""
        self._root_path = tempfile.mkdtemp(
            prefix=MicrovmBuilder.ROOT_PREFIX,
            dir=f"{DEFAULT_TEST_SESSION_ROOT_PATH}")

    def build(self,
              kernel: Artifact,
              disks: [DiskArtifact],
              ssh_key: Artifact,
              config: Artifact,
              net_ifaces=None,
              diff_snapshots=False,
              cpu_template=None,
              fc_binary=None,
              jailer_binary=None,
              use_ramdisk=False,
              smt=None, daemonize=True):
        """Build a fresh microvm."""
        vm = init_microvm(self.root_path, self.bin_cloner_path,
                          fc_binary, jailer_binary)
        vm.jailer.daemonize = daemonize
        # Start firecracker.
        vm.spawn(use_ramdisk=use_ramdisk)

        # Link the microvm to kernel, rootfs, ssh_key artifacts.
        vm.kernel_file = kernel.local_path()
        vm.rootfs_file = disks[0].local_path()
        # copy rootfs to ramdisk if needed
        jailed_rootfs_path = vm.copy_to_jail_ramfs(vm.rootfs_file) if \
            use_ramdisk else vm.create_jailed_resource(vm.rootfs_file)

        # Download ssh key into the microvm root.
        ssh_key.download(vm.path)
        vm.ssh_config['ssh_key_path'] = ssh_key.local_path()
        os.chmod(vm.ssh_config['ssh_key_path'], 0o400)

        # Provide a default network configuration.
        if net_ifaces is None or len(net_ifaces) == 0:
            ifaces = [NetIfaceConfig()]
        else:
            ifaces = net_ifaces

        # Configure network interfaces using artifacts.
        for iface in ifaces:
            vm.create_tap_and_ssh_config(host_ip=iface.host_ip,
                                         guest_ip=iface.guest_ip,
                                         netmask_len=iface.netmask,
                                         tapname=iface.tap_name)
            guest_mac = net_tools.mac_from_ip(iface.guest_ip)
            response = vm.network.put(
                iface_id=iface.dev_name,
                host_dev_name=iface.tap_name,
                guest_mac=guest_mac
            )
            assert vm.api_session.is_status_no_content(response.status_code)

        with open(config.local_path(), encoding='utf-8') as \
                microvm_config_file:
            microvm_config = json.load(microvm_config_file)

        response = vm.basic_config(
            add_root_device=False,
            boot_args='console=ttyS0 reboot=k panic=1'
        )

        # Add the root file system with rw permissions.
        response = vm.drive.put(
            drive_id='rootfs',
            path_on_host=jailed_rootfs_path,
            is_root_device=True,
            is_read_only=False
        )
        assert vm.api_session \
            .is_status_no_content(response.status_code), \
            response.text

        # Apply the microvm artifact configuration and template.
        response = vm.machine_cfg.put(
            vcpu_count=int(microvm_config['vcpu_count']),
            mem_size_mib=int(microvm_config['mem_size_mib']),
            smt=smt,
            track_dirty_pages=diff_snapshots,
            cpu_template=cpu_template,
        )
        assert vm.api_session.is_status_no_content(response.status_code)

        vm.vcpus_count = int(microvm_config['vcpu_count'])

        return VmInstance(config, kernel, disks, ssh_key, vm)

    # This function currently returns the vm and a metrics_fifo which
    # is needed by the performance integration tests.
    # TODO: Move all metrics functionality to microvm (encapsulating the fifo)
    # so we do not need to move it around polluting the code.
    def build_from_snapshot(self,
                            snapshot: Snapshot,
                            resume=False,
                            # Enable incremental snapshot capability.
                            diff_snapshots=False,
                            use_ramdisk=False,
                            fc_binary=None, jailer_binary=None,
                            daemonize=True):
        """Build a microvm from a snapshot artifact."""
        vm = init_microvm(self.root_path, self.bin_cloner_path,
                          fc_binary, jailer_binary,)
        vm.jailer.daemonize = daemonize
        vm.spawn(log_level='Error', use_ramdisk=use_ramdisk)
        vm.api_session.untime()

        metrics_file_path = os.path.join(vm.path, 'metrics.log')
        metrics_fifo = log_tools.Fifo(metrics_file_path)
        response = vm.metrics.put(
            metrics_path=vm.create_jailed_resource(metrics_fifo.path)
        )
        assert vm.api_session.is_status_no_content(response.status_code)

        # Hardlink all the snapshot files into the microvm jail.
        jailed_mem = vm.copy_to_jail_ramfs(snapshot.mem) if use_ramdisk else \
            vm.create_jailed_resource(snapshot.mem)
        jailed_vmstate = vm.copy_to_jail_ramfs(snapshot.vmstate) \
            if use_ramdisk else vm.create_jailed_resource(snapshot.vmstate)

        assert len(snapshot.disks) > 0, "Snapshot requires at least one disk."
        _jailed_disks = []
        for disk in snapshot.disks:
            _jailed_disks.append(vm.copy_to_jail_ramfs(disk) if use_ramdisk
                                 else vm.create_jailed_resource(disk))

        vm.ssh_config['ssh_key_path'] = snapshot.ssh_key.local_path()

        # Create network interfaces.
        for iface in snapshot.net_ifaces:
            vm.create_tap_and_ssh_config(host_ip=iface.host_ip,
                                         guest_ip=iface.guest_ip,
                                         netmask_len=iface.netmask,
                                         tapname=iface.tap_name)
        response = vm.snapshot.load(mem_file_path=jailed_mem,
                                    snapshot_path=jailed_vmstate,
                                    diff=diff_snapshots,
                                    resume=resume)
        status_ok = vm.api_session.is_status_no_content(response.status_code)

        # Verify response status and cleanup if needed before assert.
        if not status_ok:
            # Destroy VM here before we assert.
            vm.kill()
            del vm

        assert status_ok, response.text

        # Return a resumed microvm.
        return vm, metrics_fifo

    def build_from_artifacts(self, config,
                             kernel, disks, cpu_template,
                             net_ifaces=None, diff_snapshots=False,
                             fc_binary=None, jailer_binary=None,
                             daemonize=True):
        """Spawns a new Firecracker and applies specified config."""
        artifacts = ArtifactCollection(_test_images_s3_bucket())
        # Pick the first artifact in the set.
        config = artifacts.microvms(keyword=config)[0]
        kernel = artifacts.kernels(keyword=kernel)[0]
        disks = artifacts.disks(keyword=disks)
        config.download()
        kernel.download()
        attached_disks = []
        for disk in disks:
            disk.download()
            attached_disks.append(disk.copy())

        # SSH key is attached to root disk artifact.
        # Builder will download ssh key in the VM root.
        ssh_key = disks[0].ssh_key()
        # Create a fresh microvm from artifacts.
        return self.build(kernel=kernel,
                          disks=attached_disks,
                          ssh_key=ssh_key,
                          config=config,
                          net_ifaces=net_ifaces,
                          diff_snapshots=diff_snapshots,
                          cpu_template=cpu_template,
                          fc_binary=fc_binary,
                          jailer_binary=jailer_binary,
                          daemonize=daemonize)

    def build_vm_nano(self, fc_binary=None, jailer_binary=None,
                      net_ifaces=None, diff_snapshots=False, daemonize=True):
        """Create a clean VM in an initial state."""
        return self.build_from_artifacts("2vcpu_256mb",
                                         "vmlinux-4.14",
                                         "ubuntu-18.04",
                                         None,
                                         net_ifaces=net_ifaces,
                                         diff_snapshots=diff_snapshots,
                                         fc_binary=fc_binary,
                                         jailer_binary=jailer_binary,
                                         daemonize=daemonize)

    def build_vm_micro(self, fc_binary=None, jailer_binary=None,
                       net_ifaces=None, diff_snapshots=False):
        """Create a clean VM in an initial state."""
        return self.build_from_artifacts("2vcpu_512mb",
                                         "vmlinux-4.14",
                                         "ubuntu-18.04",
                                         None,
                                         net_ifaces=net_ifaces,
                                         diff_snapshots=diff_snapshots,
                                         fc_binary=fc_binary,
                                         jailer_binary=jailer_binary)

    def cleanup(self):
        """Clean up this builder context."""
        if self._root_path:
            shutil.rmtree(self._root_path, ignore_errors=True)

    def __del__(self):
        """Teardown the object."""
        self.cleanup()

class SnapshotBuilder:  # pylint: disable=too-few-public-methods
    """Create a snapshot from a running microvm."""

    def __init__(self, microvm):
        """Initialize the snapshot builder."""
        self._microvm = microvm

    def create_snapshot_dir(self):
        """Create dir and files for saving snapshot state and memory."""
        chroot_path = self._microvm.jailer.chroot_path()
        snapshot_dir = os.path.join(chroot_path, "snapshot")
        Path(snapshot_dir).mkdir(parents=True, exist_ok=True)
        cmd = 'chown {}:{} {}'.format(self._microvm.jailer.uid,
                                      self._microvm.jailer.gid,
                                      snapshot_dir)
        utils.run_cmd(cmd)
        return snapshot_dir

    def create(self,
               disks,
               ssh_key: Artifact,
               snapshot_type: SnapshotType = SnapshotType.FULL,
               target_version: str = None,
               mem_file_name: str = "vm.mem",
               snapshot_name: str = "vm.vmstate",
               net_ifaces=None,
               use_ramdisk=False):
        """Create a Snapshot object from a microvm and artifacts."""
        if use_ramdisk:
            snaps_dir = self._microvm.jailer.chroot_ramfs_path()
            mem_full_path = os.path.join(snaps_dir, mem_file_name)
            vmstate_full_path = os.path.join(snaps_dir, snapshot_name)

            memsize = self._microvm.machine_cfg.configuration['mem_size_mib']
            # Pre-allocate ram for memfile to eliminate allocation variability.
            utils.run_cmd('dd if=/dev/zero of={} bs=1M count={}'.format(
                mem_full_path, memsize
            ))
            cmd = 'chown {}:{} {}'.format(
                self._microvm.jailer.uid,
                self._microvm.jailer.gid,
                mem_full_path
            )
            utils.run_cmd(cmd)
        else:
            snaps_dir = self.create_snapshot_dir()
            mem_full_path = os.path.join(snaps_dir, mem_file_name)
            vmstate_full_path = os.path.join(snaps_dir, snapshot_name)

        snaps_dir_name = os.path.basename(snaps_dir)
        self._microvm.pause_to_snapshot(
            mem_file_path=os.path.join('/', snaps_dir_name, mem_file_name),
            snapshot_path=os.path.join('/', snaps_dir_name, snapshot_name),
            diff=snapshot_type == SnapshotType.DIFF,
            version=target_version)

        # Create a copy of the ssh_key artifact.
        ssh_key_copy = ssh_key.copy()
        return Snapshot(mem=mem_full_path,
                        vmstate=vmstate_full_path,
                        # TODO: To support more disks we need to figure out a
                        # simple and flexible way to store snapshot artifacts
                        # in S3. This should be done in a PR where we add tests
                        # that resume from S3 snapshot artifacts.
                        disks=disks,
                        net_ifaces=net_ifaces or [NetIfaceConfig()],
                        ssh_key=ssh_key_copy)
<|endoftext|>


<commit_before>import time
import datetime
import json
import pickle
import base64
import collections

import sys,os
sys.path.append( os.getcwd() )

import downloader
from sites import ksml as ksml_parser

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#from pyvirtualdisplay import Display

def login(driver, username, password):
    driver.get('https://media.portal.worldoftulo.com/Login?continue=https%3A%2F%2Fbackend.worldoftulo.com%2Foauth2%2Fauth%3Fclient_id%3D56b9cb80a672017f61000001%26redirect_uri%3Dhttp%253A%252F%252Fwww.ksml.fi%252Ftulo_sso_redirect.jsp%26state%3Dhttp%253A%252F%252Fwww.ksml.fi%252F%2523%26response_type%3Dcode%26oid%3Dmedia%26accountOrigin%3DKE')

    username_elem = driver.find_element_by_id( 'Username' )
    password_elem = driver.find_element_by_id( 'Password' )
    submit_btn = driver.find_element_by_xpath( '//input[@type="submit"]' )

    username_elem.send_keys( username )
    password_elem.send_keys( password )
    submit_btn.click()

    time.sleep(10)

def collect_urls(driver, start_date, end_date, error):
    print "Collecting urls: " + start_date + '...' + end_date

    urls = []

    try:
        driver.get( 'http://www.ksml.fi/arkisto/?tem=archive_lsearch5&dayfrom=19960101&dayto=19960102' )

    except Exception, e:
        print e
        print "Error in collecting urls: " + start_date + '...' + end_date

        error.write( "Error in collecting urls: " + start_date + '...' + end_date + '\n' )

    while True:
        try:
            element = WebDriverWait(driver, 30).until(
                EC.visibility_of_element_located((By.ID, 'neocontent'))
                )

        except Exception, e:
            print e
            print "Error in collecting urls: " + start_date + '...' + end_date

            error.write( "Error in collecting urls: " + start_date + '...' + end_date + '\n' )
            break

        finally:
            remove_ad(driver, 'ESM_Tarranurkka')
            remove_ad(driver, 'ESM_Tikkeri')

            content = driver.find_element_by_id('neocontent')

            tags = content.find_elements_by_tag_name('a')

            if not tags:
                error.write( "No urls found: " + start_date + '...' + end_date + 'page' + str(page) + '\n')

            for tag in tags:

                url = get_url_from_element( driver, tag )

                if not url:
                    error.write( "Error in getting url: " + tag + '\n' )

                if 'search' in url:
                    continue

                urls.append(url)

            print str(len(urls)) + ' urls collected...'

            paginator = content.find_element_by_class_name('paginatorArchive')

            if 'Seuraava' not in paginator.find_elements_by_tag_name('a')[-1].get_attribute('innerHTML'):
                break
            else:
                paginator.find_elements_by_tag_name('a')[-1].click()

    save_urls( urls, start_date, end_date )

    return urls

def get_url_from_element( driver, tag ):
    attempt = 1
    while True:
        try:
            url = tag.get_attribute('href')
        except Exception, e:
            if attempt > 5:
                return ''
            attempt += 1
            continue

        break

    return url

def save_urls(urls, start_date, end_date):
    print "Saving urls: " + start_date + '...' + end_date

    url_dir = 'saved_urls/ksml'

    if not os.path.exists( url_dir ):
        os.makedirs( url_dir )

    try:
        url_log = open( url_dir + start_date + '_' + end_date + '.log', 'w' )

        for url in urls:
            url_log.write( url.replace('neo', 'arkisto/') + '\n' )

    except Exception, e:
        print e
        print "Error in saving urls " + start_date + '...' + end_date

def download(driver, url, raw_dir, error):
    url = url.replace('neo', 'arkisto/')

    try:
        driver.get(url)

        element = WebDriverWait(driver, 30).until(
            EC.visibility_of_element_located((By.ID, 'neocontent'))
            )

    except Exception, e:
        print e
        print "Failed " + url

        error.write( "Error in downloading content: " + url + '\n')

    finally:

        try:
            content = driver.find_element_by_id('neocontent')

            story = ksml_parser.parse_from_archive( url, content.get_attribute('innerHTML') )

            f = base64.encodestring( url )

            pickle.dump( story, open( raw_dir + f + '.pickle', 'w' ) )

            return story['http']

        except Exception, e:
            print e
            print "Failed " + url

            error.write( "Error in downloading content: " + url + '\n')

def remove_ad(driver, ad_id):
    try:
        ad = driver.find_element_by_id( ad_id )
        ad_images = ad.find_elements_by_tag_name('img')
        ad_images[0].click()

    except Exception, e:
        pass

def split_to_months(year):

    leap_years = ['1996', '2000', '2004', '2008', '2012', '2016']

    feb_end = '0228'
    if year in leap_years:
        feb_end = '0229'

    if year == '2017':
        return {'0101' : '0131', '0201' : feb_end, '0301' : '0302'}

    periods = {'0101' : '0131',
               '0201' : feb_end,
               '0301' : '0331',
               '0401' : '0430',
               '0501' : '0531',
               '0601' : '0630',
               '0701' : '0731',
               '0801' : '0831',
               '0901' : '0930',
               '1001' : '1031',
               '1101' : '1130',
               '1201' : '1231'
               }

    return periods

if __name__ == '__main__':

    stamp = datetime.datetime.now().isoformat().split('.')[0]

    raw_dir = 'data-raw/' + stamp + '/' ## where pickles are stored
    data_dir = 'data/' + stamp + '/' ## where json outputs are stored
    error_dir = 'error-logs/' + stamp + '/' ## save error logs here

    for f in [raw_dir, data_dir, error_dir]:
        if not os.path.exists( f ):
            os.makedirs( f )

    #display = Display(visible=0, size=(800, 600))
    #display.start()

    driver = webdriver.Firefox()

    login( driver, sys.argv[1], sys.argv[2] )

    http_status = collections.defaultdict( int )

    for year in sys.argv[3:]:
        months = split_to_months( year )

        for start, end in months.items():
            start_date = year + start
            end_date = year + end

            error = open( error_dir + 'error_' + start_date + '_' + end_date + '.log', 'w' )

            urls = collect_urls( driver, start_date, end_date, error )

            print str( len(urls) ) + " urls collected: " + start_date + '...' + end_date
            print "Downloading content: " + start_date + '...' + end_date

            downloaded = 0

            for url in urls:
                s = download( driver, url, raw_dir, error )

                downloaded += 1

                print str(downloaded) + " stories downloaded from " + start_date + '...' + end_date

                http_status[ s ] += 1

    driver.quit()

    print 'Final status'
    for s, c in http_status.items():
        print s, '\t', c

    # regroup files to nicer folders
    store = downloader.resort_pickles( raw_dir )

    # store files as json
    for filename, data in store.items():
        json.dump( data , open(  data_dir + filename + '.json', 'w' ) )
<commit_msg>Fix date issue<commit_after>import time
import datetime
import json
import pickle
import base64
import collections

import sys,os
sys.path.append( os.getcwd() )

import downloader
from sites import ksml as ksml_parser

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#from pyvirtualdisplay import Display

def login(driver, username, password):
    driver.get('https://media.portal.worldoftulo.com/Login?continue=https%3A%2F%2Fbackend.worldoftulo.com%2Foauth2%2Fauth%3Fclient_id%3D56b9cb80a672017f61000001%26redirect_uri%3Dhttp%253A%252F%252Fwww.ksml.fi%252Ftulo_sso_redirect.jsp%26state%3Dhttp%253A%252F%252Fwww.ksml.fi%252F%2523%26response_type%3Dcode%26oid%3Dmedia%26accountOrigin%3DKE')

    username_elem = driver.find_element_by_id( 'Username' )
    password_elem = driver.find_element_by_id( 'Password' )
    submit_btn = driver.find_element_by_xpath( '//input[@type="submit"]' )

    username_elem.send_keys( username )
    password_elem.send_keys( password )
    submit_btn.click()

    time.sleep(10)

def collect_urls(driver, start_date, end_date, error):
    print "Collecting urls: " + start_date + '...' + end_date

    urls = []

    try:
        driver.get( 'http://www.ksml.fi/arkisto/?tem=archive_lsearch5&dayfrom=' + start_date +'&dayto=' + end_date )

    except Exception, e:
        print e
        print "Error in collecting urls: " + start_date + '...' + end_date

        error.write( "Error in collecting urls: " + start_date + '...' + end_date + '\n' )

    while True:
        try:
            element = WebDriverWait(driver, 30).until(
                EC.visibility_of_element_located((By.ID, 'neocontent'))
                )

        except Exception, e:
            print e
            print "Error in collecting urls: " + start_date + '...' + end_date

            error.write( "Error in collecting urls: " + start_date + '...' + end_date + '\n' )
            break

        finally:
            remove_ad(driver, 'ESM_Tarranurkka')
            remove_ad(driver, 'ESM_Tikkeri')

            content = driver.find_element_by_id('neocontent')

            tags = content.find_elements_by_tag_name('a')

            if not tags:
                error.write( "No urls found: " + start_date + '...' + end_date + 'page' + str(page) + '\n')

            for tag in tags:

                url = get_url_from_element( driver, tag )

                if not url:
                    error.write( "Error in getting url: " + tag + '\n' )

                if 'search' in url:
                    continue

                urls.append(url)

            print str(len(urls)) + ' urls collected...'

            paginator = content.find_element_by_class_name('paginatorArchive')

            if 'Seuraava' not in paginator.find_elements_by_tag_name('a')[-1].get_attribute('innerHTML'):
                break
            else:
                paginator.find_elements_by_tag_name('a')[-1].click()

    save_urls( urls, start_date, end_date )

    return urls

def get_url_from_element( driver, tag ):
    attempt = 1
    while True:
        try:
            url = tag.get_attribute('href')
        except Exception, e:
            if attempt > 5:
                return ''
            attempt += 1
            continue

        break

    return url

def save_urls(urls, start_date, end_date):
    print "Saving urls: " + start_date + '...' + end_date

    url_dir = 'saved_urls/ksml'

    if not os.path.exists( url_dir ):
        os.makedirs( url_dir )

    try:
        url_log = open( url_dir + start_date + '_' + end_date + '.log', 'w' )

        for url in urls:
            url_log.write( url.replace('neo', 'arkisto/') + '\n' )

    except Exception, e:
        print e
        print "Error in saving urls " + start_date + '...' + end_date

def download(driver, url, raw_dir, error):
    url = url.replace('neo', 'arkisto/')

    try:
        driver.get(url)

        element = WebDriverWait(driver, 30).until(
            EC.visibility_of_element_located((By.ID, 'neocontent'))
            )

    except Exception, e:
        print e
        print "Failed " + url

        error.write( "Error in downloading content: " + url + '\n')

    finally:

        try:
            content = driver.find_element_by_id('neocontent')

            story = ksml_parser.parse_from_archive( url, content.get_attribute('innerHTML') )

            f = base64.encodestring( url )

            pickle.dump( story, open( raw_dir + f + '.pickle', 'w' ) )

            return story['http']

        except Exception, e:
            print e
            print "Failed " + url

            error.write( "Error in downloading content: " + url + '\n')

def remove_ad(driver, ad_id):
    try:
        ad = driver.find_element_by_id( ad_id )
        ad_images = ad.find_elements_by_tag_name('img')
        ad_images[0].click()

    except Exception, e:
        pass

def split_to_months(year):

    leap_years = ['1996', '2000', '2004', '2008', '2012', '2016']

    feb_end = '0228'
    if year in leap_years:
        feb_end = '0229'

    if year == '2017':
        return {'0101' : '0131', '0201' : feb_end, '0301' : '0302'}

    periods = {'0101' : '0131',
               '0201' : feb_end,
               '0301' : '0331',
               '0401' : '0430',
               '0501' : '0531',
               '0601' : '0630',
               '0701' : '0731',
               '0801' : '0831',
               '0901' : '0930',
               '1001' : '1031',
               '1101' : '1130',
               '1201' : '1231'
               }

    return periods

if __name__ == '__main__':

    stamp = datetime.datetime.now().isoformat().split('.')[0]

    raw_dir = 'data-raw/' + stamp + '/' ## where pickles are stored
    data_dir = 'data/' + stamp + '/' ## where json outputs are stored
    error_dir = 'error-logs/' + stamp + '/' ## save error logs here

    for f in [raw_dir, data_dir, error_dir]:
        if not os.path.exists( f ):
            os.makedirs( f )

    #display = Display(visible=0, size=(800, 600))
    #display.start()

    driver = webdriver.Firefox()

    login( driver, sys.argv[1], sys.argv[2] )

    http_status = collections.defaultdict( int )

    for year in sys.argv[3:]:
        months = split_to_months( year )

        for start, end in months.items():
            start_date = year + start
            end_date = year + end

            error = open( error_dir + 'error_' + start_date + '_' + end_date + '.log', 'w' )

            urls = collect_urls( driver, start_date, end_date, error )

            print str( len(urls) ) + " urls collected: " + start_date + '...' + end_date
            print "Downloading content: " + start_date + '...' + end_date

            downloaded = 0

            for url in urls:
                s = download( driver, url, raw_dir, error )

                downloaded += 1

                print str(downloaded) + " stories downloaded from " + start_date + '...' + end_date

                http_status[ s ] += 1

    driver.quit()

    print 'Final status'
    for s, c in http_status.items():
        print s, '\t', c

    # regroup files to nicer folders
    store = downloader.resort_pickles( raw_dir )

    # store files as json
    for filename, data in store.items():
        json.dump( data , open(  data_dir + filename + '.json', 'w' ) )
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
from spack import *

class RBiocgenerics(RPackage):
    """S4 generic functions needed by many Bioconductor packages."""
    homepage = "https://www.bioconductor.org/packages/BiocGenerics/"
    url      = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.22.0.tar.gz"
    list_url = homepage
    version('0.22.0', 'ef910f2011c0652e1f5fdf3b14219490')
<commit_msg>r-biocgenerics: added version 0.22.1 and added versioned dependency on r (#5683)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
from spack import *

class RBiocgenerics(RPackage):
    """S4 generic functions needed by many Bioconductor packages."""
    homepage = "https://www.bioconductor.org/packages/BiocGenerics/"
    url      = "https://www.bioconductor.org/packages/release/bioc/src/contrib/BiocGenerics_0.22.0.tar.gz"
    list_url = homepage

    version('0.22.1', '19759052960991f065e6542851d56efa')
    version('0.22.0', 'ef910f2011c0652e1f5fdf3b14219490')

    depends_on('r@3.4.0:3.4.9', when='@0.22.0:0.22.1')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""The option dialog use to adjust the setting of algorithm."""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2021"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"

from typing import cast, List, Tuple, Dict, Union, Optional
from qtpy.QtCore import Slot, Qt
from qtpy.QtWidgets import (
    QDialog,
    QTableWidgetItem,
    QDoubleSpinBox,
    QSpinBox,
    QWidget,
)
from pyslvs.metaheuristics import default, AlgorithmType
from pyslvs_ui.info import html
from .options_ui import Ui_Dialog

_Value = Union[int, float]

class AlgorithmOptionDialog(QDialog, Ui_Dialog):
    """Option dialog.

    Only edit the settings after closed.
    """

    def __init__(
        self,
        opt: AlgorithmType,
        settings: Dict[str, _Value],
        parent: QWidget
    ):
        """Load the settings to user interface."""
        super(AlgorithmOptionDialog, self).__init__(parent)
        self.setupUi(self)
        self.setWindowFlags(self.windowFlags()
                            & ~Qt.WindowContextHelpButtonHint)
        self.setWindowTitle(f"{opt.value} Options")
        self.opt = opt
        self.__init_alg_table()
        self.alg_table.setColumnWidth(0, 200)
        self.alg_table.setColumnWidth(1, 90)
        self.__set_args(settings)

    def __init_alg_table(self) -> None:
        """Initialize the algorithm table widgets."""
        def write_table(
            integers: Optional[List[Tuple[str, str, str]]] = None,
            floats: Optional[List[Tuple[str, str, str]]] = None
        ):
            """Use to write table data."""
            if integers is None:
                integers = []
            if floats is None:
                floats = []
            i = 0
            for options, box, max_value in (
                (integers, QSpinBox, 9),
                (floats, QDoubleSpinBox, 10.)
            ):
                for name, tooltip, tooltip in options:
                    self.alg_table.insertRow(i)
                    name_cell = QTableWidgetItem(name)
                    name_cell.setToolTip(tooltip)
                    self.alg_table.setItem(i, 0, name_cell)
                    spinbox = box()
                    spinbox.setMaximum(max_value)
                    spinbox.setToolTip(tooltip)
                    self.alg_table.setCellWidget(i, 1, spinbox)
                    i += 1

        if self.opt == AlgorithmType.RGA:
            write_table(
                floats=[
                    ("Crossover Rate", 'pCross',
                        html("The chance of crossover.")),
                    ("Mutation Rate", 'pMute',
                        html("The chance of mutation.")),
                    ("Winning Rate", 'pWin',
                        html("The chance of winning.")),
                    ("Delta value", 'bDelta',
                        html("The power value when matching chromosome."))
                ]
            )
        elif self.opt == AlgorithmType.FA:
            write_table(
                floats=[
                    ("Alpha value", 'alpha', html(
                        "Alpha value is the step size of the firefly.")),
                    ("Minimum Beta value", 'beta_min', html(
                        "The minimal attraction, must not less than this.")),
                    ("Gamma value", 'gamma', html(
                        "Beta will multiplied by exponential power value "
                        "with this weight factor.")),
                    ("Beta0 value", 'beta0', html(
                        "The attraction of two firefly in 0 distance."))
                ]
            )
        elif self.opt == AlgorithmType.DE:
            write_table(
                integers=[
                    ("Evolutionary strategy (0-9)", 'strategy',
                        html("There are 10 way to evolution."))
                ],
                floats=[
                    ("Weight factor", 'F', html(
                        "Weight factor is usually between 0.5 and 1"
                        "(in rare cases > 1).")),
                    ("Recombination factor", 'CR',
                        html("The chance of crossover possible."))
                ]
            )

    def __set_args(self, settings: Dict[str, _Value]) -> None:
        """Set arguments by settings dict."""
        if 'max_gen' in settings:
            self.max_gen.setValue(settings['max_gen'])
        elif 'min_fit' in settings:
            self.min_fit_option.setChecked(True)
            self.min_fit.setValue(settings['min_fit'])
        elif 'max_time' in settings:
            self.max_time_option.setChecked(True)
            # In second (int)
            max_time = cast(int, settings['max_time'])
            self.max_time_h.setValue(max_time // 3600)
            self.max_time_m.setValue((max_time % 3600) // 60)
            self.max_time_s.setValue(max_time % 3600 % 60)
        self.report.setValue(settings['report'])
        if self.opt == AlgorithmType.RGA:
            self.pop_size.setValue(settings['nPop'])
            for i, tag in enumerate(('pCross', 'pMute', 'pWin', 'bDelta')):
                self.alg_table.cellWidget(i, 1).setValue(settings[tag])
        elif self.opt == AlgorithmType.FA:
            self.pop_size.setValue(settings['n'])
            for i, tag in enumerate(('alpha', 'beta_min', 'gamma', 'beta0')):
                self.alg_table.cellWidget(i, 1).setValue(settings[tag])
        elif self.opt == AlgorithmType.DE:
            self.pop_size.setValue(settings['NP'])
            for i, tag in enumerate(('strategy', 'F', 'CR')):
                self.alg_table.cellWidget(i, 1).setValue(settings[tag])
        elif self.opt == AlgorithmType.TLBO:
            self.pop_size.setValue(settings['class_size'])

    @Slot(name='on_reset_button_clicked')
    def __reset(self) -> None:
        """Reset the settings to default."""
        self.__set_args(default(self.opt))
<commit_msg>Update optimization arguments dynamically.<commit_after># -*- coding: utf-8 -*-

"""The option dialog use to adjust the setting of algorithm."""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2021"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"

from typing import cast, Mapping, Union
from qtpy.QtCore import Slot, Qt
from qtpy.QtWidgets import (
    QDialog,
    QTableWidgetItem,
    QDoubleSpinBox,
    QSpinBox,
    QWidget,
)
from pyslvs.metaheuristics import default, AlgorithmType
from .options_ui import Ui_Dialog

_Value = Union[int, float]

class AlgorithmOptionDialog(QDialog, Ui_Dialog):
    """Option dialog.

    Only edit the settings after closed.
    """

    def __init__(
        self,
        opt: AlgorithmType,
        settings: Mapping[str, _Value],
        parent: QWidget
    ):
        """Load the settings to user interface."""
        super(AlgorithmOptionDialog, self).__init__(parent)
        self.setupUi(self)
        self.setWindowFlags(self.windowFlags()
                            & ~Qt.WindowContextHelpButtonHint)
        self.setWindowTitle(f"{opt.value} Options")
        self.opt = opt
        self.dft = {tag: value for tag, value in default(self.opt).items()
                    if tag not in {'pop_num', 'report', 'max_gen', 'min_fit',
                                   'max_time', 'slow_down'}}
        self.__init_alg_table()
        self.alg_table.setColumnWidth(0, 200)
        self.alg_table.setColumnWidth(1, 90)
        self.__set_args(settings)

    def __init_alg_table(self) -> None:
        """Initialize the algorithm table widgets."""
        self.alg_table.setRowCount(len(self.dft))
        for i, (tag, value) in enumerate(self.dft.items()):
            self.alg_table.setItem(i, 0, QTableWidgetItem(tag))
            if isinstance(value, int):
                w = QSpinBox()
                w.setMaximum(1e5)
            else:
                w = QDoubleSpinBox()
            w.setValue(value)
            self.alg_table.setCellWidget(i, 1, w)

    def __set_args(self, settings: Mapping[str, _Value]) -> None:
        """Set arguments by settings dict."""
        if 'max_gen' in settings:
            self.max_gen.setValue(settings['max_gen'])
        elif 'min_fit' in settings:
            self.min_fit_option.setChecked(True)
            self.min_fit.setValue(settings['min_fit'])
        elif 'max_time' in settings:
            self.max_time_option.setChecked(True)
            # In second (int)
            max_time = cast(int, settings['max_time'])
            self.max_time_h.setValue(max_time // 3600)
            self.max_time_m.setValue((max_time % 3600) // 60)
            self.max_time_s.setValue(max_time % 3600 % 60)
        self.pop_size.setValue(settings['pop_num'])
        self.report.setValue(settings['report'])
        for i, tag in enumerate(self.dft):
            self.alg_table.cellWidget(i, 1).setValue(settings.get(tag, self.dft[tag]))

    @Slot(name='on_reset_button_clicked')
    def __reset(self) -> None:
        """Reset the settings to default."""
        self.__set_args(default(self.opt))
<|endoftext|>


<commit_before>from flask import Blueprint

bp_v1 = Blueprint('accounts_v1', __name__)<commit_msg>added newline to fileend<commit_after>from flask import Blueprint

bp_v1 = Blueprint('accounts_v1', __name__)
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

# TODO: try switching to CMakePackage for more control over build
class PyTorch(PythonPackage):
    """Tensors and Dynamic neural networks in Python
    with strong GPU acceleration."""

    homepage = "https://pytorch.org/"
    git      = "https://github.com/pytorch/pytorch.git"
    install_time_test_callbacks = ['install_test', 'import_module_test']

    maintainers = ['adamjstewart']
    import_modules = [
        'tools', 'caffe2', 'torch', 'tools.cwrap', 'tools.autograd',
        'tools.setup_helpers', 'tools.shared', 'tools.jit', 'tools.pyi',
        'tools.nnwrap', 'tools.cwrap.plugins', 'caffe2.core', 'caffe2.proto',
        'caffe2.python', 'caffe2.distributed', 'caffe2.perfkernels',
        'caffe2.experiments', 'caffe2.contrib', 'caffe2.quantization',
        'caffe2.core.nomnigraph', 'caffe2.python.ideep', 'caffe2.python.mint',
        'caffe2.python.layers', 'caffe2.python.onnx', 'caffe2.python.trt',
        'caffe2.python.models', 'caffe2.python.docs', 'caffe2.python.modeling',
        'caffe2.python.mkl', 'caffe2.python.examples',
        'caffe2.python.predictor', 'caffe2.python.helpers',
        'caffe2.python.rnn', 'caffe2.python.onnx.bin',
        'caffe2.python.models.seq2seq', 'caffe2.experiments.python',
        'caffe2.contrib.nnpack', 'caffe2.contrib.warpctc',
        'caffe2.contrib.nccl', 'caffe2.contrib.playground',
        'caffe2.contrib.gloo', 'caffe2.contrib.script', 'caffe2.contrib.prof',
        'caffe2.contrib.tensorboard', 'caffe2.contrib.aten',
        'caffe2.contrib.playground.resnetdemo',
        'caffe2.contrib.script.examples', 'caffe2.contrib.aten.docs',
        'caffe2.quantization.server', 'torch.nn', 'torch.onnx',
        'torch.distributed', 'torch.autograd', 'torch.multiprocessing',
        'torch.cuda', 'torch.backends', 'torch.optim', 'torch.utils',
        'torch.contrib', 'torch.jit', 'torch.sparse',
        'torch.for_onnx', 'torch._thnn', 'torch.distributions',
        'torch.nn.parallel', 'torch.nn._functions', 'torch.nn.backends',
        'torch.nn.utils', 'torch.nn.modules', 'torch.nn.parallel.deprecated',
        'torch.nn._functions.thnn', 'torch.distributed.deprecated',
        'torch.autograd._functions', 'torch.backends.cuda',
        'torch.backends.mkl', 'torch.backends.mkldnn', 'torch.backends.openmp',
        'torch.backends.cudnn', 'torch.utils.backcompat',
        'torch.utils.bottleneck', 'torch.utils.ffi', 'torch.utils.tensorboard',
        'torch.utils.data', 'torch.utils.data._utils'
    ]

    version('master', branch='master', submodules=True)
    version('1.3.0', tag='v1.3.0', submodules=True)
    version('1.2.0', tag='v1.2.0', submodules=True)
    version('1.1.0', tag='v1.1.0', submodules=True)
    version('1.0.1', tag='v1.0.1', submodules=True)
    version('1.0.0', tag='v1.0.0', submodules=True)
    version('0.4.1', tag='v0.4.1', submodules=True)
    version('0.4.0', tag='v0.4.0', submodules=True)
    version('0.3.1', tag='v0.3.1', submodules=True)

    variant('cuda', default=True, description='Enables CUDA build')
    variant('cudnn', default=False, description='Enables the cuDNN build')
    variant('magma', default=False, description='Enables the MAGMA build')
    variant('fbgemm', default=False, description='Enables the FBGEMM build')
    variant('test', default=False, description='Enables the test build')
    variant('miopen', default=False, description='Enables the MIOpen build')
    variant('mkldnn', default=False, description='Enables use of MKLDNN')
    variant('nnpack', default=False, description='Enables NNPACK build')
    variant('qnnpack', default=False, description='Enables QNNPACK build (quantized 8-bit operators)')
    variant('distributed', default=False, description='Enables distributed (c10d, gloo, mpi, etc.) build')
    variant('nccl', default=False, description='Use Spack-installed NCCL')
    variant('caffe2', default=False, description='Enables Caffe2 operators build')
    variant('gloo', default=False, description='Enables features related to distributed support')
    variant('opencv', default=False, description='Enables use of OpenCV for additional operators')
    variant('openmp', default=True, description='Enables use of OpenMP for parallelization')
    variant('ffmpeg', default=False, description='Enables use of ffmpeg for additional operators')
    variant('leveldb', default=False, description='Enables use of LevelDB for storage')
    variant('lmdb', default=False, description='Enables use of LMDB for storage')
    variant('binary', default=False, description='Enables the additional binaries/ build')
    variant('redis', default=False, description='Use Redis for distributed workflows')
    variant('zstd', default=False, description='Enables use of ZSTD')
    variant('tbb', default=False, description='Enables TBB support')

    conflicts('+cudnn', when='~cuda')
    conflicts('+magma', when='~cuda')
    conflicts('+fbgemm', when='@:0.4')
    conflicts('+miopen', when='@:0.4')
    conflicts('+mkldnn', when='@:0.3')
    conflicts('+qnnpack', when='@:0.4')
    conflicts('+nccl', when='~cuda')
    conflicts('+opencv', when='@:0.4')
    conflicts('+ffmpeg', when='@:0.4')
    conflicts('+leveldb', when='@:0.4')
    conflicts('+lmdb', when='@:0.4')
    conflicts('+binary', when='@:0.4')
    conflicts('+redis', when='@:1.0')
    conflicts('+zstd', when='@:1.0')
    conflicts('+tbb', when='@:1.1')

    # Required dependencies
    depends_on('cmake@3.5:', type='build')
    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-future', when='@1.1: ^python@:2', type='build')
    depends_on('py-pyyaml', type=('build', 'run'))
    depends_on('py-typing', when='@0.4: ^python@:3.4', type=('build', 'run'))
    depends_on('py-pybind11', when='@0.4:', type=('build', 'run'))
    depends_on('blas')
    depends_on('lapack')
    depends_on('protobuf', when='@0.4:')
    depends_on('eigen', when='@0.4:')
    # TODO: replace all third_party packages with Spack packages

    # Optional dependencies
    depends_on('cuda@7.5:', when='+cuda', type=('build', 'link', 'run'))
    depends_on('cuda@9:', when='@1.1:+cuda', type=('build', 'link', 'run'))
    depends_on('cudnn@6:', when='+cudnn')
    depends_on('cudnn@7:', when='@1.1:+cudnn')
    depends_on('magma', when='+magma')
    # TODO: add dependency: https://github.com/pytorch/FBGEMM
    depends_on('fbgemm', when='+fbgemm')
    # TODO: add dependency: https://github.com/ROCmSoftwarePlatform/MIOpen
    depends_on('miopen', when='+miopen')
    depends_on('mkl', when='+mkldnn')
    # TODO: add dependency: https://github.com/Maratyszcza/NNPACK
    depends_on('nnpack', when='+nnpack')
    depends_on('qnnpack', when='+qnnpack')
    depends_on('mpi', when='+distributed')
    depends_on('nccl', when='+nccl')
    depends_on('gloo', when='+gloo')
    depends_on('opencv', when='+opencv')
    depends_on('llvm-openmp', when='%clang platform=darwin +openmp')
    depends_on('ffmpeg', when='+ffmpeg')
    depends_on('leveldb', when='+leveldb')
    depends_on('lmdb', when='+lmdb')
    depends_on('redis', when='+redis')
    depends_on('zstd', when='+zstd')
    depends_on('tbb', when='+tbb')

    # Test dependencies
    depends_on('ninja', type='test')
    depends_on('py-hypothesis', type='test')
    depends_on('py-six', type='test')
    depends_on('py-psutil', type='test')

    def setup_build_environment(self, env):
        def enable_or_disable(variant, keyword='USE', var=None, newer=False):
            """Set environment variable to enable or disable support for a
            particular variant.

            Parameters:
                variant (str): the variant to check
                keyword (str): the prefix to use for enabling/disabling
                var (str): CMake variable to set. Defaults to variant.upper()
                newer (bool): newer variants that never used NO_*
            """
            if var is None:
                var = variant.upper()

            # Version 1.1.0 switched from NO_* to USE_* or BUILD_*
            # But some newer variants have always used USE_* or BUILD_*
            if self.spec.satisfies('@1.1:') or newer:
                if '+' + variant in self.spec:
                    env.set(keyword + '_' + var, 'ON')
                else:
                    env.set(keyword + '_' + var, 'OFF')
            else:
                if '+' + variant in self.spec:
                    env.unset('NO_' + var)
                else:
                    env.set('NO_' + var, 'ON')

        # Build system has problems locating MKL libraries
        # See https://github.com/pytorch/pytorch/issues/24334
        if 'mkl' in self.spec:
            env.prepend_path('CMAKE_PREFIX_PATH', self.spec['mkl'].prefix.mkl)

        env.set('MAX_JOBS', make_jobs)

        # Don't use vendored third-party libraries
        env.set('BUILD_CUSTOM_PROTOBUF', 'OFF')
        env.set('USE_PYTORCH_QNNPACK', 'OFF')
        env.set('USE_SYSTEM_EIGEN_INSTALL', 'ON')
        env.set('pybind11_DIR', self.spec['py-pybind11'].prefix)
        env.set('pybind11_INCLUDE_DIR',
                self.spec['py-pybind11'].prefix.include)

        enable_or_disable('cuda')
        if '+cuda' in self.spec:
            env.set('CUDA_HOME', self.spec['cuda'].prefix)

        enable_or_disable('cudnn')
        if '+cudnn' in self.spec:
            env.set('CUDNN_LIB_DIR', self.spec['cudnn'].libs.directories[0])
            env.set('CUDNN_INCLUDE_DIR', self.spec['cudnn'].prefix.include)
            env.set('CUDNN_LIBRARY', self.spec['cudnn'].libs[0])

        enable_or_disable('fbgemm')
        enable_or_disable('test', keyword='BUILD')

        enable_or_disable('miopen')
        if '+miopen' in self.spec:
            env.set('MIOPEN_LIB_DIR', self.spec['miopen'].libs.directories[0])
            env.set('MIOPEN_INCLUDE_DIR', self.spec['miopen'].prefix.include)
            env.set('MIOPEN_LIBRARY', self.spec['miopen'].libs[0])

        enable_or_disable('mkldnn')
        if '+mkldnn' in self.spec:
            env.set('MKLDNN_HOME', self.spec['intel-mkl'].prefix)

        enable_or_disable('nnpack')
        enable_or_disable('qnnpack')
        enable_or_disable('distributed')

        enable_or_disable('nccl')
        enable_or_disable('nccl', var='SYSTEM_NCCL')
        if '+nccl' in self.spec:
            env.set('NCCL_ROOT', self.spec['nccl'].prefix)
            env.set('NCCL_LIB_DIR', self.spec['nccl'].libs.directories[0])
            env.set('NCCL_INCLUDE_DIR', self.spec['nccl'].prefix.include)

        enable_or_disable('caffe2', keyword='BUILD', var='CAFFE2_OPS')
        enable_or_disable('gloo', newer=True)
        enable_or_disable('gloo', var='IBVERBS', newer=True)
        enable_or_disable('opencv', newer=True)
        enable_or_disable('openmp', newer=True)
        enable_or_disable('ffmpeg', newer=True)
        enable_or_disable('leveldb', newer=True)
        enable_or_disable('lmdb', newer=True)
        enable_or_disable('binary', keyword='BUILD', newer=True)

        env.set('PYTORCH_BUILD_VERSION', self.version)
        env.set('PYTORCH_BUILD_NUMBER', 0)

        # BLAS to be used by Caffe2. Can be MKL, Eigen, ATLAS, or OpenBLAS.
        if '^mkl' in self.spec:
            env.set('BLAS', 'MKL')
        elif '^eigen' in self.spec:
            env.set('BLAS', 'Eigen')
        elif '^atlas' in self.spec:
            env.set('BLAS', 'ATLAS')
        elif '^openblas' in self.spec:
            env.set('BLAS', 'OpenBLAS')

        enable_or_disable('redis', newer=True)
        enable_or_disable('zstd', newer=True)
        enable_or_disable('tbb', newer=True)

    def test(self):
        pass

    def install_test(self):
        with working_dir('test'):
            python('run_test.py')
<commit_msg>Speed up PyTorch build (#13621)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

# TODO: try switching to CMakePackage for more control over build
class PyTorch(PythonPackage):
    """Tensors and Dynamic neural networks in Python
    with strong GPU acceleration."""

    homepage = "https://pytorch.org/"
    git      = "https://github.com/pytorch/pytorch.git"
    install_time_test_callbacks = ['install_test', 'import_module_test']

    maintainers = ['adamjstewart']
    import_modules = [
        'tools', 'caffe2', 'torch', 'tools.cwrap', 'tools.autograd',
        'tools.setup_helpers', 'tools.shared', 'tools.jit', 'tools.pyi',
        'tools.nnwrap', 'tools.cwrap.plugins', 'caffe2.core', 'caffe2.proto',
        'caffe2.python', 'caffe2.distributed', 'caffe2.perfkernels',
        'caffe2.experiments', 'caffe2.contrib', 'caffe2.quantization',
        'caffe2.core.nomnigraph', 'caffe2.python.ideep', 'caffe2.python.mint',
        'caffe2.python.layers', 'caffe2.python.onnx', 'caffe2.python.trt',
        'caffe2.python.models', 'caffe2.python.docs', 'caffe2.python.modeling',
        'caffe2.python.mkl', 'caffe2.python.examples',
        'caffe2.python.predictor', 'caffe2.python.helpers',
        'caffe2.python.rnn', 'caffe2.python.onnx.bin',
        'caffe2.python.models.seq2seq', 'caffe2.experiments.python',
        'caffe2.contrib.nnpack', 'caffe2.contrib.warpctc',
        'caffe2.contrib.nccl', 'caffe2.contrib.playground',
        'caffe2.contrib.gloo', 'caffe2.contrib.script', 'caffe2.contrib.prof',
        'caffe2.contrib.tensorboard', 'caffe2.contrib.aten',
        'caffe2.contrib.playground.resnetdemo',
        'caffe2.contrib.script.examples', 'caffe2.contrib.aten.docs',
        'caffe2.quantization.server', 'torch.nn', 'torch.onnx',
        'torch.distributed', 'torch.autograd', 'torch.multiprocessing',
        'torch.cuda', 'torch.backends', 'torch.optim', 'torch.utils',
        'torch.contrib', 'torch.jit', 'torch.sparse',
        'torch.for_onnx', 'torch._thnn', 'torch.distributions',
        'torch.nn.parallel', 'torch.nn._functions', 'torch.nn.backends',
        'torch.nn.utils', 'torch.nn.modules', 'torch.nn.parallel.deprecated',
        'torch.nn._functions.thnn', 'torch.distributed.deprecated',
        'torch.autograd._functions', 'torch.backends.cuda',
        'torch.backends.mkl', 'torch.backends.mkldnn', 'torch.backends.openmp',
        'torch.backends.cudnn', 'torch.utils.backcompat',
        'torch.utils.bottleneck', 'torch.utils.ffi', 'torch.utils.tensorboard',
        'torch.utils.data', 'torch.utils.data._utils'
    ]

    version('master', branch='master', submodules=True)
    version('1.3.0', tag='v1.3.0', submodules=True)
    version('1.2.0', tag='v1.2.0', submodules=True)
    version('1.1.0', tag='v1.1.0', submodules=True)
    version('1.0.1', tag='v1.0.1', submodules=True)
    version('1.0.0', tag='v1.0.0', submodules=True)
    version('0.4.1', tag='v0.4.1', submodules=True)
    version('0.4.0', tag='v0.4.0', submodules=True)
    version('0.3.1', tag='v0.3.1', submodules=True)

    variant('cuda', default=True, description='Enables CUDA build')
    variant('cudnn', default=False, description='Enables the cuDNN build')
    variant('magma', default=False, description='Enables the MAGMA build')
    variant('fbgemm', default=False, description='Enables the FBGEMM build')
    variant('test', default=False, description='Enables the test build')
    variant('miopen', default=False, description='Enables the MIOpen build')
    variant('mkldnn', default=False, description='Enables use of MKLDNN')
    variant('nnpack', default=False, description='Enables NNPACK build')
    variant('qnnpack', default=False, description='Enables QNNPACK build (quantized 8-bit operators)')
    variant('distributed', default=False, description='Enables distributed (c10d, gloo, mpi, etc.) build')
    variant('nccl', default=False, description='Use Spack-installed NCCL')
    variant('caffe2', default=False, description='Enables Caffe2 operators build')
    variant('gloo', default=False, description='Enables features related to distributed support')
    variant('opencv', default=False, description='Enables use of OpenCV for additional operators')
    variant('openmp', default=True, description='Enables use of OpenMP for parallelization')
    variant('ffmpeg', default=False, description='Enables use of ffmpeg for additional operators')
    variant('leveldb', default=False, description='Enables use of LevelDB for storage')
    variant('lmdb', default=False, description='Enables use of LMDB for storage')
    variant('binary', default=False, description='Enables the additional binaries/ build')
    variant('redis', default=False, description='Use Redis for distributed workflows')
    variant('zstd', default=False, description='Enables use of ZSTD')
    variant('tbb', default=False, description='Enables TBB support')

    conflicts('+cudnn', when='~cuda')
    conflicts('+magma', when='~cuda')
    conflicts('+fbgemm', when='@:0.4')
    conflicts('+miopen', when='@:0.4')
    conflicts('+mkldnn', when='@:0.3')
    conflicts('+qnnpack', when='@:0.4')
    conflicts('+nccl', when='~cuda')
    conflicts('+opencv', when='@:0.4')
    conflicts('+ffmpeg', when='@:0.4')
    conflicts('+leveldb', when='@:0.4')
    conflicts('+lmdb', when='@:0.4')
    conflicts('+binary', when='@:0.4')
    conflicts('+redis', when='@:1.0')
    conflicts('+zstd', when='@:1.0')
    conflicts('+tbb', when='@:1.1')

    # Required dependencies
    depends_on('cmake@3.5:', type='build')
    # Use Ninja generator to speed up build times
    # Automatically used if found
    depends_on('ninja@1.5:', type='build')
    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-future', when='@1.1: ^python@:2', type='build')
    depends_on('py-pyyaml', type=('build', 'run'))
    depends_on('py-typing', when='@0.4: ^python@:3.4', type=('build', 'run'))
    depends_on('py-pybind11', when='@0.4:', type=('build', 'run'))
    depends_on('blas')
    depends_on('lapack')
    depends_on('protobuf', when='@0.4:')
    depends_on('eigen', when='@0.4:')
    # TODO: replace all third_party packages with Spack packages

    # Optional dependencies
    depends_on('cuda@7.5:', when='+cuda', type=('build', 'link', 'run'))
    depends_on('cuda@9:', when='@1.1:+cuda', type=('build', 'link', 'run'))
    depends_on('cudnn@6:', when='+cudnn')
    depends_on('cudnn@7:', when='@1.1:+cudnn')
    depends_on('magma', when='+magma')
    # TODO: add dependency: https://github.com/pytorch/FBGEMM
    depends_on('fbgemm', when='+fbgemm')
    # TODO: add dependency: https://github.com/ROCmSoftwarePlatform/MIOpen
    depends_on('miopen', when='+miopen')
    depends_on('mkl', when='+mkldnn')
    # TODO: add dependency: https://github.com/Maratyszcza/NNPACK
    depends_on('nnpack', when='+nnpack')
    depends_on('qnnpack', when='+qnnpack')
    depends_on('mpi', when='+distributed')
    depends_on('nccl', when='+nccl')
    depends_on('gloo', when='+gloo')
    depends_on('opencv', when='+opencv')
    depends_on('llvm-openmp', when='%clang platform=darwin +openmp')
    depends_on('ffmpeg', when='+ffmpeg')
    depends_on('leveldb', when='+leveldb')
    depends_on('lmdb', when='+lmdb')
    depends_on('redis', when='+redis')
    depends_on('zstd', when='+zstd')
    depends_on('tbb', when='+tbb')

    # Test dependencies
    depends_on('py-hypothesis', type='test')
    depends_on('py-six', type='test')
    depends_on('py-psutil', type='test')

    # Both build and install run cmake/make/make install
    # Only run once to speed up build times
    phases = ['install']

    def setup_build_environment(self, env):
        def enable_or_disable(variant, keyword='USE', var=None, newer=False):
            """Set environment variable to enable or disable support for a
            particular variant.

            Parameters:
                variant (str): the variant to check
                keyword (str): the prefix to use for enabling/disabling
                var (str): CMake variable to set. Defaults to variant.upper()
                newer (bool): newer variants that never used NO_*
            """
            if var is None:
                var = variant.upper()

            # Version 1.1.0 switched from NO_* to USE_* or BUILD_*
            # But some newer variants have always used USE_* or BUILD_*
            if self.spec.satisfies('@1.1:') or newer:
                if '+' + variant in self.spec:
                    env.set(keyword + '_' + var, 'ON')
                else:
                    env.set(keyword + '_' + var, 'OFF')
            else:
                if '+' + variant in self.spec:
                    env.unset('NO_' + var)
                else:
                    env.set('NO_' + var, 'ON')

        # Build system has problems locating MKL libraries
        # See https://github.com/pytorch/pytorch/issues/24334
        if 'mkl' in self.spec:
            env.prepend_path('CMAKE_PREFIX_PATH', self.spec['mkl'].prefix.mkl)

        # Build in parallel to speed up build times
        env.set('MAX_JOBS', make_jobs)

        # Spack logs have trouble handling colored output
        env.set('COLORIZE_OUTPUT', 'OFF')

        # Don't use vendored third-party libraries
        env.set('BUILD_CUSTOM_PROTOBUF', 'OFF')
        env.set('USE_PYTORCH_QNNPACK', 'OFF')
        env.set('USE_SYSTEM_EIGEN_INSTALL', 'ON')
        env.set('pybind11_DIR', self.spec['py-pybind11'].prefix)
        env.set('pybind11_INCLUDE_DIR',
                self.spec['py-pybind11'].prefix.include)

        enable_or_disable('cuda')
        if '+cuda' in self.spec:
            env.set('CUDA_HOME', self.spec['cuda'].prefix)

        enable_or_disable('cudnn')
        if '+cudnn' in self.spec:
            env.set('CUDNN_LIB_DIR', self.spec['cudnn'].libs.directories[0])
            env.set('CUDNN_INCLUDE_DIR', self.spec['cudnn'].prefix.include)
            env.set('CUDNN_LIBRARY', self.spec['cudnn'].libs[0])

        enable_or_disable('fbgemm')
        enable_or_disable('test', keyword='BUILD')

        if '+miopen' in self.spec:
            env.set('MIOPEN_LIB_DIR', self.spec['miopen'].libs.directories[0])
            env.set('MIOPEN_INCLUDE_DIR', self.spec['miopen'].prefix.include)
            env.set('MIOPEN_LIBRARY', self.spec['miopen'].libs[0])

        enable_or_disable('mkldnn')
        if '+mkldnn' in self.spec:
            env.set('MKLDNN_HOME', self.spec['intel-mkl'].prefix)

        enable_or_disable('nnpack')
        enable_or_disable('qnnpack')
        enable_or_disable('distributed')

        enable_or_disable('nccl')
        enable_or_disable('nccl', var='SYSTEM_NCCL')
        if '+nccl' in self.spec:
            env.set('NCCL_ROOT', self.spec['nccl'].prefix)
            env.set('NCCL_LIB_DIR', self.spec['nccl'].libs.directories[0])
            env.set('NCCL_INCLUDE_DIR', self.spec['nccl'].prefix.include)

        enable_or_disable('caffe2', keyword='BUILD', var='CAFFE2_OPS')
        enable_or_disable('gloo', newer=True)
        enable_or_disable('opencv', newer=True)
        enable_or_disable('openmp', newer=True)
        enable_or_disable('ffmpeg', newer=True)
        enable_or_disable('leveldb', newer=True)
        enable_or_disable('lmdb', newer=True)
        enable_or_disable('binary', keyword='BUILD', newer=True)

        env.set('PYTORCH_BUILD_VERSION', self.version)
        env.set('PYTORCH_BUILD_NUMBER', 0)

        # BLAS to be used by Caffe2
        if '^mkl' in self.spec:
            env.set('BLAS', 'MKL')
        elif '^atlas' in self.spec:
            env.set('BLAS', 'ATLAS')
        elif '^openblas' in self.spec:
            env.set('BLAS', 'OpenBLAS')
        elif '^veclibfort' in self.spec:
            env.set('BLAS', 'vecLib')
        elif '^libflame' in self.spec:
            env.set('BLAS', 'FLAME')
        elif '^eigen' in self.spec:
            env.set('BLAS', 'Eigen')

        enable_or_disable('redis', newer=True)
        enable_or_disable('zstd', newer=True)
        enable_or_disable('tbb', newer=True)

    def test(self):
        pass

    def install_test(self):
        with working_dir('test'):
            python('run_test.py')
<|endoftext|>


<commit_before>#! /usr/bin/python

# Copyright (C) GRyCAP - I3M - UPV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import sys
import base64
from mock import MagicMock
from mock import patch

sys.path.append("..")
sys.path.append(".")
sys.path.append("../..")

from scar.providers.aws.batchfunction import Batch

class TestBatch(unittest.TestCase):

    def __init__(self, *args):
        unittest.TestCase.__init__(self, *args)

    def test_init(self):
        batch = Batch({'lambda': {'name': 'fname'}})
        self.assertEqual(type(batch.client.client).__name__, "Batch")

    @patch('boto3.Session')
    @patch('scar.providers.aws.launchtemplates.SupervisorUtils.get_supervisor_binary_url')
    @patch('scar.providers.aws.controller.FileUtils.load_tmp_config_file')
    def test_create_batch_environment(self, load_tmp_config_file, get_supervisor_binary_url,
                                      boto_session):
        session = MagicMock(['client'])
        client = MagicMock(['register_job_definition', 'describe_launch_templates',
                            'create_launch_template', 'create_compute_environment',
                            'describe_compute_environments', 'create_job_queue'])
        session.client.return_value = client
        boto_session.return_value = session
        load_tmp_config_file.return_value = {}
        get_supervisor_binary_url.return_value = "https://some.es"

        batch = Batch({'lambda': {'name': 'fname',
                                  'supervisor': {'version': '1.4.2'},
                                  'container': {'image': 'some/image:tag',
                                                'environment': {'Variables': {}}}},
                       'batch': {'service_role': 'srole',
                                 'memory': 1024,
                                 'vcpus': 1,
                                 'type': 'MANAGED',
                                 'state': 'ENABLED',
                                 'enable_gpu': True,
                                 'compute_resources': {'instance_role': 'irole',
                                                       'type': 'EC2',
                                                       'min_v_cpus': 0,
                                                       'max_v_cpus': 2,
                                                       'desired_v_cpus': 1,
                                                       'instance_types': ['m1.small'],
                                                       'subnets': [],
                                                       'security_group_ids': [],
                                                       'launch_template_name': 'temp_name'},
                                 'environment': {'Variables': {}},
                                 'multi_node_parallel': {'enabled': False}},
                       'iam': {'account_id': 'id', 'role': 'role'}})

        batch.client.client.register_job_definition.return_value = {}
        batch.client.client.describe_launch_templates.return_value = {}
        batch.client.client.create_launch_template.return_value = {
            'LaunchTemplate': {'LatestVersionNumber': '1'}
        }
        batch.client.client.create_compute_environment.return_value = {}
        batch.client.client.describe_compute_environments.return_value = {
            'computeEnvironments': [{'state': 'ENABLED', 'status': 'VALID'}]
        }
        batch.client.client.create_job_queue.return_value = {}
        batch.create_batch_environment()

        func_config = b"container:\n"
        func_config += b"  environment:\n"
        func_config += b"    Variables: {}\n"
        func_config += b"  image: some/image:tag\n"
        func_config += b"name: fname\n"
        func_config += b"storage_providers: {}\n"
        func_config += b"supervisor:\n"
        func_config += b"  version: 1.4.2\n"
        func_config_64 = base64.b64encode(func_config).decode()
        res = {'jobDefinitionName': 'fname',
               'containerProperties': {
                   'image': 'some/image:tag',
                   'memory': 1024,
                   'vcpus': 1,
                   'command': ['/bin/sh', '-c',
                               'echo $EVENT | /opt/faas-supervisor/bin/supervisor'],
                   'volumes': [{'host': {'sourcePath': '/opt/faas-supervisor/bin'},
                               'name': 'supervisor-bin'}],
                   'environment': [{'name': 'AWS_LAMBDA_FUNCTION_NAME', 'value': 'fname'},
                                   {'name': 'SCRIPT', 'value': ''},
                                   {'name': 'FUNCTION_CONFIG', 'value': func_config_64}],
                   'mountPoints': [{'containerPath': '/opt/faas-supervisor/bin',
                                   'sourceVolume': 'supervisor-bin'}],
                   'resourceRequirements': [{'value': '1',
                                             'type': 'GPU'}]},
               'type': 'container'}
        self.assertEqual(batch.client.client.register_job_definition.call_args_list[0][1], res)
        self.assertEqual(batch.client.client.create_launch_template.call_args_list[0][1]['LaunchTemplateName'], 'temp_name')
        self.assertEqual(batch.client.client.create_launch_template.call_args_list[0][1]['VersionDescription'], '1.4.2')
        res = {'computeEnvironmentName': 'fname',
               'serviceRole': 'srole',
               'type': 'MANAGED',
               'state': 'ENABLED',
               'computeResources': {'type': 'EC2',
                                    'minvCpus': 0,
                                    'maxvCpus': 2,
                                    'desiredvCpus': 1,
                                    'instanceTypes': ['m1.small'],
                                    'subnets': [],
                                    'securityGroupIds': [],
                                    'instanceRole': 'irole',
                                    'launchTemplate': {'launchTemplateName': 'temp_name',
                                                       'version': '1'}}}
        self.assertEqual(batch.client.client.create_compute_environment.call_args_list[0][1], res)

    @patch('boto3.Session')
    @patch('scar.providers.aws.controller.FileUtils.load_tmp_config_file')
    def test_delete_compute_environment(self, load_tmp_config_file, boto_session):
        session = MagicMock(['client'])
        client = MagicMock(['describe_job_definitions', 'deregister_job_definition',
                            'describe_job_queues', 'update_job_queue', 'delete_job_queue',
                            'describe_compute_environments', 'update_compute_environment',
                            'delete_compute_environment'])
        session.client.return_value = client
        boto_session.return_value = session

        batch = Batch({'lambda': {'name': 'fname'}})

        batch.client.client.describe_job_definitions.return_value = {}
        batch.client.client.describe_job_queues.side_effect = [{'jobQueues': [{'state': 'ENABLED',
                                                                               'status': 'VALID'}]},
                                                               {'jobQueues': [{'state': 'DISABLED',
                                                                               'status': 'VALID'}]},
                                                               {'jobQueues': []}]
        batch.client.client.describe_compute_environments.side_effect = [
            {'computeEnvironments': [{'state': 'ENABLED', 'status': 'VALID'}]},
            {'computeEnvironments': [{'state': 'DISABLED', 'status': 'VALID'}]},
            {'computeEnvironments': []}
        ]

        batch.delete_compute_environment()

        res = {'jobQueue': 'fname', 'state': 'DISABLED'}
        self.assertEqual(batch.client.client.update_job_queue.call_args_list[0][1], res)
        res = {'jobQueue': 'fname'}
        self.assertEqual(batch.client.client.delete_job_queue.call_args_list[0][1], res)

        res = {'computeEnvironment': 'fname', 'state': 'DISABLED'}
        self.assertEqual(batch.client.client.update_compute_environment.call_args_list[0][1], res)
        res = {'computeEnvironment': 'fname'}
        self.assertEqual(batch.client.client.delete_compute_environment.call_args_list[0][1], res)
<commit_msg>Add tests<commit_after>#! /usr/bin/python

# Copyright (C) GRyCAP - I3M - UPV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import sys
import base64
from mock import MagicMock
from mock import patch

sys.path.append("..")
sys.path.append(".")
sys.path.append("../..")

from scar.providers.aws.batchfunction import Batch

class TestBatch(unittest.TestCase):

    def __init__(self, *args):
        unittest.TestCase.__init__(self, *args)

    def test_init(self):
        batch = Batch({'lambda': {'name': 'fname'}})
        self.assertEqual(type(batch.client.client).__name__, "Batch")

    @patch('boto3.Session')
    @patch('scar.providers.aws.launchtemplates.SupervisorUtils.get_supervisor_binary_url')
    @patch('scar.providers.aws.controller.FileUtils.load_tmp_config_file')
    def test_create_batch_environment(self, load_tmp_config_file, get_supervisor_binary_url,
                                      boto_session):
        session = MagicMock(['client'])
        client = MagicMock(['register_job_definition', 'describe_launch_templates',
                            'create_launch_template', 'create_compute_environment',
                            'describe_compute_environments', 'create_job_queue'])
        session.client.return_value = client
        boto_session.return_value = session
        load_tmp_config_file.return_value = {}
        get_supervisor_binary_url.return_value = "https://some.es"

        batch = Batch({'lambda': {'name': 'fname',
                                  'supervisor': {'version': '1.4.2'},
                                  'container': {'image': 'some/image:tag',
                                                'environment': {'Variables': {}}}},
                       'batch': {'service_role': 'srole',
                                 'memory': 1024,
                                 'vcpus': 1,
                                 'type': 'MANAGED',
                                 'state': 'ENABLED',
                                 'enable_gpu': True,
                                 'compute_resources': {'instance_role': 'irole',
                                                       'type': 'EC2',
                                                       'min_v_cpus': 0,
                                                       'max_v_cpus': 2,
                                                       'desired_v_cpus': 1,
                                                       'instance_types': ['m1.small'],
                                                       'subnets': [],
                                                       'security_group_ids': [],
                                                       'launch_template_name': 'temp_name'},
                                 'environment': {'Variables': {}},
                                 'multi_node_parallel': {'enabled': False}},
                       'iam': {'account_id': 'id', 'role': 'role'}})

        batch.client.client.register_job_definition.return_value = {}
        batch.client.client.describe_launch_templates.return_value = {}
        batch.client.client.create_launch_template.return_value = {
            'LaunchTemplate': {'LatestVersionNumber': '1'}
        }
        batch.client.client.create_compute_environment.return_value = {}
        batch.client.client.describe_compute_environments.return_value = {
            'computeEnvironments': [{'state': 'ENABLED', 'status': 'VALID'}]
        }
        batch.client.client.create_job_queue.return_value = {}
        batch.create_batch_environment()

        func_config = b"container:\n"
        func_config += b"  environment:\n"
        func_config += b"    Variables: {}\n"
        func_config += b"  image: some/image:tag\n"
        func_config += b"name: fname\n"
        func_config += b"storage_providers: {}\n"
        func_config += b"supervisor:\n"
        func_config += b"  version: 1.4.2\n"
        func_config_64 = base64.b64encode(func_config).decode()
        res = {'jobDefinitionName': 'fname',
               'containerProperties': {
                   'image': 'some/image:tag',
                   'memory': 1024,
                   'vcpus': 1,
                   'command': ['/bin/sh', '-c',
                               'echo $EVENT | /opt/faas-supervisor/bin/supervisor'],
                   'volumes': [{'host': {'sourcePath': '/opt/faas-supervisor/bin'},
                               'name': 'supervisor-bin'}],
                   'environment': [{'name': 'AWS_LAMBDA_FUNCTION_NAME', 'value': 'fname'},
                                   {'name': 'SCRIPT', 'value': ''},
                                   {'name': 'FUNCTION_CONFIG', 'value': func_config_64}],
                   'mountPoints': [{'containerPath': '/opt/faas-supervisor/bin',
                                   'sourceVolume': 'supervisor-bin'}],
                   'resourceRequirements': [{'value': '1',
                                             'type': 'GPU'}]},
               'type': 'container'}
        self.assertEqual(batch.client.client.register_job_definition.call_args_list[0][1], res)
        self.assertEqual(batch.client.client.create_launch_template.call_args_list[0][1]['LaunchTemplateName'], 'temp_name')
        self.assertEqual(batch.client.client.create_launch_template.call_args_list[0][1]['VersionDescription'], '1.4.2')
        res = {'computeEnvironmentName': 'fname',
               'serviceRole': 'srole',
               'type': 'MANAGED',
               'state': 'ENABLED',
               'computeResources': {'type': 'EC2',
                                    'minvCpus': 0,
                                    'maxvCpus': 2,
                                    'desiredvCpus': 1,
                                    'instanceTypes': ['m1.small'],
                                    'subnets': [],
                                    'securityGroupIds': [],
                                    'instanceRole': 'irole',
                                    'launchTemplate': {'launchTemplateName': 'temp_name',
                                                       'version': '1'}}}
        self.assertEqual(batch.client.client.create_compute_environment.call_args_list[0][1], res)

    @patch('boto3.Session')
    def test_delete_compute_environment(self, boto_session):
        session = MagicMock(['client'])
        client = MagicMock(['describe_job_definitions', 'deregister_job_definition',
                            'describe_job_queues', 'update_job_queue', 'delete_job_queue',
                            'describe_compute_environments', 'update_compute_environment',
                            'delete_compute_environment'])
        session.client.return_value = client
        boto_session.return_value = session

        batch = Batch({'lambda': {'name': 'fname'}})

        batch.client.client.describe_job_definitions.return_value = {}
        batch.client.client.describe_job_queues.side_effect = [{'jobQueues': [{'state': 'ENABLED',
                                                                               'status': 'VALID'}]},
                                                               {'jobQueues': [{'state': 'DISABLED',
                                                                               'status': 'VALID'}]},
                                                               {'jobQueues': []}]
        batch.client.client.describe_compute_environments.side_effect = [
            {'computeEnvironments': [{'state': 'ENABLED', 'status': 'VALID'}]},
            {'computeEnvironments': [{'state': 'DISABLED', 'status': 'VALID'}]},
            {'computeEnvironments': []}
        ]

        batch.delete_compute_environment()

        res = {'jobQueue': 'fname', 'state': 'DISABLED'}
        self.assertEqual(batch.client.client.update_job_queue.call_args_list[0][1], res)
        res = {'jobQueue': 'fname'}
        self.assertEqual(batch.client.client.delete_job_queue.call_args_list[0][1], res)

        res = {'computeEnvironment': 'fname', 'state': 'DISABLED'}
        self.assertEqual(batch.client.client.update_compute_environment.call_args_list[0][1], res)
        res = {'computeEnvironment': 'fname'}
        self.assertEqual(batch.client.client.delete_compute_environment.call_args_list[0][1], res)
<|endoftext|>