<commit_before>//! Command buffers.
//!
//! A command buffer collects a list of commands to be submitted to the device.
//! Each command buffer has specific capabilities for graphics, compute or transfer operations,
//! and can be either a "primary" command buffer or a "secondary" command buffer.  Operations
//! always start from a primary command buffer, but a primary command buffer can contain calls
//! to secondary command buffers that contain snippets of commands that do specific things, similar
//! to function calls.
//!
//! All the possible commands are implemented in the `RawCommandBuffer` trait, and then the `CommandBuffer`
//! and related types make a generic, strongly-typed wrapper around it that only expose the methods that
//! are valid for the capabilities it provides.

// TODO: Document pipelines and subpasses better.

use queue::capability::{Capability, Supports};
use Backend;

use std::borrow::Borrow;
use std::marker::PhantomData;

mod compute;
mod graphics;
mod raw;
mod render_pass;
mod transfer;

pub use self::graphics::*;
pub use self::raw::{
    ClearColorRaw, ClearDepthStencilRaw, ClearValueRaw, CommandBufferFlags,
    CommandBufferInheritanceInfo, DescriptorSetOffset, IntoRawCommandBuffer, Level as RawLevel,
    RawCommandBuffer,
};
pub use self::render_pass::*;
pub use self::transfer::*;

/// Trait indicating how many times a Submit object can be submitted to a command buffer.
pub trait Shot {}
/// Indicates a Submit that can only be submitted once.
pub enum OneShot {}
impl Shot for OneShot {}

/// Indicates a Submit that can be submitted multiple times.
pub enum MultiShot {}
impl Shot for MultiShot {}

/// A trait indicating the level of a command buffer.
pub trait Level {}

/// Indicates a primary command buffer.

/// Vulkan describes a primary command buffer as one which can be directly submitted
/// to a queue, and can execute `Secondary` command buffers.
pub enum Primary {}
impl Level for Primary {}

/// Indicates a secondary command buffer.
///
/// Vulkan describes a secondary command buffer as one which cannot be directly submitted
/// to a queue, but can be executed by a primary command buffer. This allows
/// multiple secondary command buffers to be constructed which do specific
/// things and can then be composed together into primary command buffers.
pub enum Secondary {}
impl Level for Secondary {}

/// A property of a command buffer to be submitted to a queue with specific capability.
pub trait Submittable<B: Backend, C: Capability, L: Level>: Borrow<B::CommandBuffer> {}

/// A convenience alias for not typing out the full signature of a secondary command buffer.
pub type SecondaryCommandBuffer<B, C, S = OneShot> = CommandBuffer<B, C, S, Secondary>;

/// A strongly-typed command buffer that will only implement methods that are valid for the operations
/// it supports.
pub struct CommandBuffer<B: Backend, C, S = OneShot, L = Primary, R = <B as Backend>::CommandBuffer>
{
    pub(crate) raw: R,
    pub(crate) _marker: PhantomData<(B, C, S, L)>,
}

impl<B, C, S, L, R> Borrow<R> for CommandBuffer<B, C, S, L, R>
where
    R: RawCommandBuffer<B>,
    B: Backend<CommandBuffer = R>,
{
    fn borrow(&self) -> &B::CommandBuffer {
        &self.raw
    }
}

impl<B: Backend, C, K: Capability + Supports<C>, S, L: Level> Submittable<B, K, L>
    for CommandBuffer<B, C, S, L>
{
}

impl<B: Backend, C, S, L> IntoRawCommandBuffer<B, C> for CommandBuffer<B, C, S, L> {
    fn into_raw(self) -> B::CommandBuffer {
        self.raw
    }
}

impl<B: Backend, C> CommandBuffer<B, C, OneShot, Primary> {
    /// Begin recording a one-shot primary command buffer.
    pub unsafe fn begin(&mut self) {
        let flags = CommandBufferFlags::ONE_TIME_SUBMIT;
        self.raw
            .begin(flags, CommandBufferInheritanceInfo::default());
    }
}

impl<B: Backend, C> CommandBuffer<B, C, MultiShot, Primary> {
    /// Begin recording a multi-shot primary command buffer.
    pub unsafe fn begin(&mut self, allow_pending_resubmit: bool) {
        let flags = if allow_pending_resubmit {
            CommandBufferFlags::SIMULTANEOUS_USE
        } else {
            CommandBufferFlags::empty()
        };
        self.raw
            .begin(flags, CommandBufferInheritanceInfo::default());
    }
}

impl<B: Backend, C> CommandBuffer<B, C, OneShot, Secondary> {
    /// Begin recording a one-shot secondary command buffer.
    pub unsafe fn begin(&mut self, inheritance: CommandBufferInheritanceInfo<B>) {
        let flags = CommandBufferFlags::ONE_TIME_SUBMIT;
        self.raw.begin(flags, inheritance);
    }
}

impl<B: Backend, C> CommandBuffer<B, C, MultiShot, Secondary> {
    /// Begin recording a multi-shot secondary command buffer.
    pub unsafe fn begin(
        &mut self,
        allow_pending_resubmit: bool,
        inheritance: CommandBufferInheritanceInfo<B>,
    ) {
        let flags = if allow_pending_resubmit {
            CommandBufferFlags::SIMULTANEOUS_USE
        } else {
            CommandBufferFlags::empty()
        };
        self.raw.begin(flags, inheritance);
    }
}

impl<B: Backend, C, S: Shot, L: Level> CommandBuffer<B, C, S, L> {
    /// Create a new typed command buffer from a raw command pool.
    pub unsafe fn new(raw: B::CommandBuffer) -> Self {
        CommandBuffer {
            raw,
            _marker: PhantomData,
        }
    }

    /// Finish recording commands to the command buffers.
    ///
    /// The command pool must be reset to able to re-record commands.
    pub unsafe fn finish(&mut self) {
        self.raw.finish();
    }

    /*
    /// Get a reference to the raw command buffer
    pub fn as_raw(&self) -> &B::CommandBuffer {
        &self.raw
    }

    /// Get a mutable reference to the raw command buffer
    pub fn as_raw_mut(&mut self) -> &mut B::CommandBuffer {
        &mut self.raw
    }*/

    /// Downgrade a command buffer to a lesser capability type.
    pub unsafe fn downgrade<D>(&mut self) -> &mut CommandBuffer<B, D, S>
    where
        C: Supports<D>,
    {
        ::std::mem::transmute(self)
    }
}

impl<B: Backend, C, S: Shot> CommandBuffer<B, C, S, Primary> {
    /// Identical to the `RawCommandBuffer` method of the same name.
    pub unsafe fn execute_commands<'a, I, T, K>(&mut self, cmd_buffers: I)
    where
        K: Capability,
        T: 'a + Submittable<B, K, Secondary>,
        I: IntoIterator<Item = &'a T>,
        C: Supports<K>,
    {
        self.raw
            .execute_commands(cmd_buffers.into_iter().map(|cmb| cmb.borrow()));
    }
}
<commit_msg>Expose `RawCommandBuffer::reset()` in `CommandBuffer`<commit_after>//! Command buffers.
//!
//! A command buffer collects a list of commands to be submitted to the device.
//! Each command buffer has specific capabilities for graphics, compute or transfer operations,
//! and can be either a "primary" command buffer or a "secondary" command buffer.  Operations
//! always start from a primary command buffer, but a primary command buffer can contain calls
//! to secondary command buffers that contain snippets of commands that do specific things, similar
//! to function calls.
//!
//! All the possible commands are implemented in the `RawCommandBuffer` trait, and then the `CommandBuffer`
//! and related types make a generic, strongly-typed wrapper around it that only expose the methods that
//! are valid for the capabilities it provides.

// TODO: Document pipelines and subpasses better.

use queue::capability::{Capability, Supports};
use Backend;

use std::borrow::Borrow;
use std::marker::PhantomData;

mod compute;
mod graphics;
mod raw;
mod render_pass;
mod transfer;

pub use self::graphics::*;
pub use self::raw::{
    ClearColorRaw, ClearDepthStencilRaw, ClearValueRaw, CommandBufferFlags,
    CommandBufferInheritanceInfo, DescriptorSetOffset, IntoRawCommandBuffer, Level as RawLevel,
    RawCommandBuffer,
};
pub use self::render_pass::*;
pub use self::transfer::*;

/// Trait indicating how many times a Submit object can be submitted to a command buffer.
pub trait Shot {}
/// Indicates a Submit that can only be submitted once.
pub enum OneShot {}
impl Shot for OneShot {}

/// Indicates a Submit that can be submitted multiple times.
pub enum MultiShot {}
impl Shot for MultiShot {}

/// A trait indicating the level of a command buffer.
pub trait Level {}

/// Indicates a primary command buffer.

/// Vulkan describes a primary command buffer as one which can be directly submitted
/// to a queue, and can execute `Secondary` command buffers.
pub enum Primary {}
impl Level for Primary {}

/// Indicates a secondary command buffer.
///
/// Vulkan describes a secondary command buffer as one which cannot be directly submitted
/// to a queue, but can be executed by a primary command buffer. This allows
/// multiple secondary command buffers to be constructed which do specific
/// things and can then be composed together into primary command buffers.
pub enum Secondary {}
impl Level for Secondary {}

/// A property of a command buffer to be submitted to a queue with specific capability.
pub trait Submittable<B: Backend, C: Capability, L: Level>: Borrow<B::CommandBuffer> {}

/// A convenience alias for not typing out the full signature of a secondary command buffer.
pub type SecondaryCommandBuffer<B, C, S = OneShot> = CommandBuffer<B, C, S, Secondary>;

/// A strongly-typed command buffer that will only implement methods that are valid for the operations
/// it supports.
pub struct CommandBuffer<B: Backend, C, S = OneShot, L = Primary, R = <B as Backend>::CommandBuffer>
{
    pub(crate) raw: R,
    pub(crate) _marker: PhantomData<(B, C, S, L)>,
}

impl<B, C, S, L, R> Borrow<R> for CommandBuffer<B, C, S, L, R>
where
    R: RawCommandBuffer<B>,
    B: Backend<CommandBuffer = R>,
{
    fn borrow(&self) -> &B::CommandBuffer {
        &self.raw
    }
}

impl<B: Backend, C, K: Capability + Supports<C>, S, L: Level> Submittable<B, K, L>
    for CommandBuffer<B, C, S, L>
{
}

impl<B: Backend, C, S, L> IntoRawCommandBuffer<B, C> for CommandBuffer<B, C, S, L> {
    fn into_raw(self) -> B::CommandBuffer {
        self.raw
    }
}

impl<B: Backend, C> CommandBuffer<B, C, OneShot, Primary> {
    /// Begin recording a one-shot primary command buffer.
    pub unsafe fn begin(&mut self) {
        let flags = CommandBufferFlags::ONE_TIME_SUBMIT;
        self.raw
            .begin(flags, CommandBufferInheritanceInfo::default());
    }
}

impl<B: Backend, C> CommandBuffer<B, C, MultiShot, Primary> {
    /// Begin recording a multi-shot primary command buffer.
    pub unsafe fn begin(&mut self, allow_pending_resubmit: bool) {
        let flags = if allow_pending_resubmit {
            CommandBufferFlags::SIMULTANEOUS_USE
        } else {
            CommandBufferFlags::empty()
        };
        self.raw
            .begin(flags, CommandBufferInheritanceInfo::default());
    }
}

impl<B: Backend, C> CommandBuffer<B, C, OneShot, Secondary> {
    /// Begin recording a one-shot secondary command buffer.
    pub unsafe fn begin(&mut self, inheritance: CommandBufferInheritanceInfo<B>) {
        let flags = CommandBufferFlags::ONE_TIME_SUBMIT;
        self.raw.begin(flags, inheritance);
    }
}

impl<B: Backend, C> CommandBuffer<B, C, MultiShot, Secondary> {
    /// Begin recording a multi-shot secondary command buffer.
    pub unsafe fn begin(
        &mut self,
        allow_pending_resubmit: bool,
        inheritance: CommandBufferInheritanceInfo<B>,
    ) {
        let flags = if allow_pending_resubmit {
            CommandBufferFlags::SIMULTANEOUS_USE
        } else {
            CommandBufferFlags::empty()
        };
        self.raw.begin(flags, inheritance);
    }
}

impl<B: Backend, C, S: Shot, L: Level> CommandBuffer<B, C, S, L> {
    /// Create a new typed command buffer from a raw command pool.
    pub unsafe fn new(raw: B::CommandBuffer) -> Self {
        CommandBuffer {
            raw,
            _marker: PhantomData,
        }
    }

    /// Finish recording commands to the command buffers.
    ///
    /// The command buffer must be reset to able to re-record commands.
    pub unsafe fn finish(&mut self) {
        self.raw.finish();
    }

    /// Empties the command buffer, optionally releasing all resources from the
    /// commands that have been submitted. The command buffer is moved back to
    /// the "initial" state.
    ///
    /// The command buffer must not be in the "pending" state. Additionally, the
    /// command pool must have been created with the RESET_INDIVIDUAL flag to be
    /// able to reset individual buffers.
    pub unsafe fn reset(&mut self, release_resources: bool) {
        self.raw.reset(release_resources);
    }

    /*
    /// Get a reference to the raw command buffer
    pub fn as_raw(&self) -> &B::CommandBuffer {
        &self.raw
    }

    /// Get a mutable reference to the raw command buffer
    pub fn as_raw_mut(&mut self) -> &mut B::CommandBuffer {
        &mut self.raw
    }*/

    /// Downgrade a command buffer to a lesser capability type.
    pub unsafe fn downgrade<D>(&mut self) -> &mut CommandBuffer<B, D, S>
    where
        C: Supports<D>,
    {
        ::std::mem::transmute(self)
    }
}

impl<B: Backend, C, S: Shot> CommandBuffer<B, C, S, Primary> {
    /// Identical to the `RawCommandBuffer` method of the same name.
    pub unsafe fn execute_commands<'a, I, T, K>(&mut self, cmd_buffers: I)
    where
        K: Capability,
        T: 'a + Submittable<B, K, Secondary>,
        I: IntoIterator<Item = &'a T>,
        C: Supports<K>,
    {
        self.raw
            .execute_commands(cmd_buffers.into_iter().map(|cmb| cmb.borrow()));
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// min-llvm-version 4.0
// ignore-cloudabi no std::env

#![feature(cfg_target_feature)]

use std::env;

fn main() {
    match env::var("TARGET") {
        Ok(s) => {
            // Skip this tests on i586-unknown-linux-gnu where sse2 is disabled
            if s.contains("i586") {
                return
            }
        }
        Err(_) => return,
    }
    if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
        assert!(cfg!(target_feature = "sse2"),
                "SSE2 was not detected as available on an x86 platform");
    }
}
<commit_msg>Ignore run-pass/sse2 when using system LLVM<commit_after>// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// no-system-llvm -- needs MCSubtargetInfo::getFeatureTable()
// ignore-cloudabi no std::env

#![feature(cfg_target_feature)]

use std::env;

fn main() {
    match env::var("TARGET") {
        Ok(s) => {
            // Skip this tests on i586-unknown-linux-gnu where sse2 is disabled
            if s.contains("i586") {
                return
            }
        }
        Err(_) => return,
    }
    if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
        assert!(cfg!(target_feature = "sse2"),
                "SSE2 was not detected as available on an x86 platform");
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

pub mod android;
pub mod fdt;

use std::collections::BTreeMap;
use std::error::Error as StdError;
use std::fmt::{self, Display};
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom};
use std::os::unix::io::AsRawFd;
use std::sync::Arc;

use devices::virtio::VirtioDevice;
use devices::{
    Bus, BusDevice, BusError, PciDevice, PciDeviceError, PciInterruptPin, PciRoot, ProxyDevice,
    Serial, SerialParameters, DEFAULT_SERIAL_PARAMS, SERIAL_ADDR,
};
use io_jail::Minijail;
use kvm::{IoeventAddress, Kvm, Vcpu, Vm};
use resources::SystemAllocator;
use sync::Mutex;
use sys_util::{syslog, EventFd, GuestAddress, GuestMemory, GuestMemoryError};

/// Holds the pieces needed to build a VM. Passed to `build_vm` in the `LinuxArch` trait below to
/// create a `RunnableLinuxVm`.
pub struct VmComponents {
    pub memory_size: u64,
    pub vcpu_count: u32,
    pub vcpu_affinity: Vec<usize>,
    pub kernel_image: File,
    pub android_fstab: Option<File>,
    pub initrd_image: Option<File>,
    pub extra_kernel_params: Vec<String>,
    pub wayland_dmabuf: bool,
}

/// Holds the elements needed to run a Linux VM. Created by `build_vm`.
pub struct RunnableLinuxVm {
    pub vm: Vm,
    pub kvm: Kvm,
    pub resources: SystemAllocator,
    pub stdio_serial: Option<Arc<Mutex<Serial>>>,
    pub exit_evt: EventFd,
    pub vcpus: Vec<Vcpu>,
    pub vcpu_affinity: Vec<usize>,
    pub irq_chip: Option<File>,
    pub io_bus: Bus,
    pub mmio_bus: Bus,
    pub pid_debug_label_map: BTreeMap<u32, String>,
}

/// The device and optional jail.
pub struct VirtioDeviceStub {
    pub dev: Box<dyn VirtioDevice>,
    pub jail: Option<Minijail>,
}

/// Trait which is implemented for each Linux Architecture in order to
/// set up the memory, cpus, and system devices and to boot the kernel.
pub trait LinuxArch {
    type Error: StdError;

    /// Takes `VmComponents` and generates a `RunnableLinuxVm`.
    ///
    /// # Arguments
    ///
    /// * `components` - Parts to use to build the VM.
    /// * `split_irqchip` - whether to use a split IRQ chip (i.e. userspace PIT/PIC/IOAPIC)
    /// * `serial_parameters` - definitions for how the serial devices should be configured.
    /// * `create_devices` - Function to generate a list of devices.
    fn build_vm<F, E>(
        components: VmComponents,
        split_irqchip: bool,
        serial_parameters: &BTreeMap<u8, SerialParameters>,
        create_devices: F,
    ) -> Result<RunnableLinuxVm, Self::Error>
    where
        F: FnOnce(&GuestMemory, &EventFd) -> Result<Vec<(Box<dyn PciDevice>, Option<Minijail>)>, E>,
        E: StdError + 'static;
}

/// Errors for device manager.
#[derive(Debug)]
pub enum DeviceRegistrationError {
    /// Could not allocate IO space for the device.
    AllocateIoAddrs(PciDeviceError),
    /// Could not allocate device address space for the device.
    AllocateDeviceAddrs(PciDeviceError),
    /// Could not allocate an IRQ number.
    AllocateIrq,
    /// Could not create the mmio device to wrap a VirtioDevice.
    CreateMmioDevice(sys_util::Error),
    //  Unable to create serial device from serial parameters
    CreateSerialDevice(devices::SerialError),
    /// Could not create an event fd.
    EventFdCreate(sys_util::Error),
    /// Could not add a device to the mmio bus.
    MmioInsert(BusError),
    /// Failed to register ioevent with VM.
    RegisterIoevent(sys_util::Error),
    /// Failed to register irq eventfd with VM.
    RegisterIrqfd(sys_util::Error),
    /// Failed to initialize proxy device for jailed device.
    ProxyDeviceCreation(devices::ProxyError),
    /// Appending to kernel command line failed.
    Cmdline(kernel_cmdline::Error),
    /// No more IRQs are available.
    IrqsExhausted,
    /// No more MMIO space available.
    AddrsExhausted,
    /// Could not register PCI device capabilities.
    RegisterDeviceCapabilities(PciDeviceError),
}

impl Display for DeviceRegistrationError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::DeviceRegistrationError::*;

        match self {
            AllocateIoAddrs(e) => write!(f, "Allocating IO addresses: {}", e),
            AllocateDeviceAddrs(e) => write!(f, "Allocating device addresses: {}", e),
            AllocateIrq => write!(f, "Allocating IRQ number"),
            CreateMmioDevice(e) => write!(f, "failed to create mmio device: {}", e),
            CreateSerialDevice(e) => write!(f, "failed to create serial device: {}", e),
            Cmdline(e) => write!(f, "unable to add device to kernel command line: {}", e),
            EventFdCreate(e) => write!(f, "failed to create eventfd: {}", e),
            MmioInsert(e) => write!(f, "failed to add to mmio bus: {}", e),
            RegisterIoevent(e) => write!(f, "failed to register ioevent to VM: {}", e),
            RegisterIrqfd(e) => write!(f, "failed to register irq eventfd to VM: {}", e),
            ProxyDeviceCreation(e) => write!(f, "failed to create proxy device: {}", e),
            IrqsExhausted => write!(f, "no more IRQs are available"),
            AddrsExhausted => write!(f, "no more addresses are available"),
            RegisterDeviceCapabilities(e) => {
                write!(f, "could not register PCI device capabilities: {}", e)
            }
        }
    }
}

/// Creates a root PCI device for use by this Vm.
pub fn generate_pci_root(
    devices: Vec<(Box<dyn PciDevice>, Option<Minijail>)>,
    mmio_bus: &mut Bus,
    resources: &mut SystemAllocator,
    vm: &mut Vm,
) -> Result<(PciRoot, Vec<(u32, PciInterruptPin)>, BTreeMap<u32, String>), DeviceRegistrationError>
{
    let mut root = PciRoot::new();
    let mut pci_irqs = Vec::new();
    let mut pid_labels = BTreeMap::new();
    for (dev_idx, (mut device, jail)) in devices.into_iter().enumerate() {
        // Only support one bus.
        device.assign_bus_dev(0, dev_idx as u8);

        let mut keep_fds = device.keep_fds();
        syslog::push_fds(&mut keep_fds);

        let irqfd = EventFd::new().map_err(DeviceRegistrationError::EventFdCreate)?;
        let irq_resample_fd = EventFd::new().map_err(DeviceRegistrationError::EventFdCreate)?;
        let irq_num = resources
            .allocate_irq()
            .ok_or(DeviceRegistrationError::AllocateIrq)? as u32;
        let pci_irq_pin = match dev_idx % 4 {
            0 => PciInterruptPin::IntA,
            1 => PciInterruptPin::IntB,
            2 => PciInterruptPin::IntC,
            3 => PciInterruptPin::IntD,
            _ => panic!(""), // Obviously not possible, but the compiler is not smart enough.
        };
        vm.register_irqfd_resample(&irqfd, &irq_resample_fd, irq_num)
            .map_err(DeviceRegistrationError::RegisterIrqfd)?;
        keep_fds.push(irqfd.as_raw_fd());
        keep_fds.push(irq_resample_fd.as_raw_fd());
        device.assign_irq(irqfd, irq_resample_fd, irq_num, pci_irq_pin);
        pci_irqs.push((dev_idx as u32, pci_irq_pin));

        let ranges = device
            .allocate_io_bars(resources)
            .map_err(DeviceRegistrationError::AllocateIoAddrs)?;
        let device_ranges = device
            .allocate_device_bars(resources)
            .map_err(DeviceRegistrationError::AllocateDeviceAddrs)?;
        device
            .register_device_capabilities()
            .map_err(DeviceRegistrationError::RegisterDeviceCapabilities)?;
        for (event, addr, datamatch) in device.ioeventfds() {
            let io_addr = IoeventAddress::Mmio(addr);
            vm.register_ioevent(&event, io_addr, datamatch)
                .map_err(DeviceRegistrationError::RegisterIoevent)?;
            keep_fds.push(event.as_raw_fd());
        }
        let arced_dev: Arc<Mutex<dyn BusDevice>> = if let Some(jail) = jail {
            let proxy = ProxyDevice::new(device, &jail, keep_fds)
                .map_err(DeviceRegistrationError::ProxyDeviceCreation)?;
            pid_labels.insert(proxy.pid() as u32, proxy.debug_label());
            Arc::new(Mutex::new(proxy))
        } else {
            device.on_sandboxed();
            Arc::new(Mutex::new(device))
        };
        root.add_device(arced_dev.clone());
        for range in &ranges {
            mmio_bus
                .insert(arced_dev.clone(), range.0, range.1, true)
                .map_err(DeviceRegistrationError::MmioInsert)?;
        }

        for range in &device_ranges {
            mmio_bus
                .insert(arced_dev.clone(), range.0, range.1, true)
                .map_err(DeviceRegistrationError::MmioInsert)?;
        }
    }
    Ok((root, pci_irqs, pid_labels))
}

/// Adds serial devices to the provided bus based on the serial parameters given. Returns the serial
///  port number and serial device to be used for stdout if defined.
///
/// # Arguments
///
/// * `io_bus` - Bus to add the devices to
/// * `com_evt_1_3` - eventfd for com1 and com3
/// * `com_evt_1_4` - eventfd for com2 and com4
/// * `io_bus` - Bus to add the devices to
/// * `serial_parameters` - definitions of serial parameter configuationis. If a setting is not
///     provided for a port, then it will use the default configuation.
pub fn add_serial_devices(
    io_bus: &mut Bus,
    com_evt_1_3: &EventFd,
    com_evt_2_4: &EventFd,
    serial_parameters: &BTreeMap<u8, SerialParameters>,
) -> Result<(Option<u8>, Option<Arc<Mutex<Serial>>>), DeviceRegistrationError> {
    let mut stdio_serial_num = None;
    let mut stdio_serial = None;

    for x in 0..3 {
        let com_evt = match x {
            0 => com_evt_1_3,
            1 => com_evt_2_4,
            2 => com_evt_1_3,
            3 => com_evt_2_4,
            _ => com_evt_1_3,
        };

        let param = serial_parameters
            .get(&(x + 1))
            .unwrap_or(&DEFAULT_SERIAL_PARAMS[x as usize]);

        let com = Arc::new(Mutex::new(
            param
                .create_serial_device(&com_evt)
                .map_err(DeviceRegistrationError::CreateSerialDevice)?,
        ));
        io_bus
            .insert(com.clone(), SERIAL_ADDR[x as usize], 0x8, false)
            .unwrap();

        if param.console {
            stdio_serial_num = Some(x + 1);
            stdio_serial = Some(com.clone());
        }
    }

    Ok((stdio_serial_num, stdio_serial))
}

/// Errors for image loading.
#[derive(Debug)]
pub enum LoadImageError {
    BadAlignment(u64),
    Seek(io::Error),
    ImageSizeTooLarge(u64),
    ReadToMemory(GuestMemoryError),
}

impl Display for LoadImageError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::LoadImageError::*;

        match self {
            BadAlignment(a) => write!(f, "Alignment not a power of two: {}", a),
            Seek(e) => write!(f, "Seek failed: {}", e),
            ImageSizeTooLarge(size) => write!(f, "Image size too large: {}", size),
            ReadToMemory(e) => write!(f, "Reading image into memory failed: {}", e),
        }
    }
}

/// Load an image from a file into guest memory.
///
/// # Arguments
///
/// * `guest_mem` - The memory to be used by the guest.
/// * `guest_addr` - The starting address to load the image in the guest memory.
/// * `max_size` - The amount of space in bytes available in the guest memory for the image.
/// * `image` - The file containing the image to be loaded.
///
/// The size in bytes of the loaded image is returned.
pub fn load_image<F>(
    guest_mem: &GuestMemory,
    image: &mut F,
    guest_addr: GuestAddress,
    max_size: u64,
) -> Result<usize, LoadImageError>
where
    F: Read + Seek,
{
    let size = image.seek(SeekFrom::End(0)).map_err(LoadImageError::Seek)?;

    if size > usize::max_value() as u64 || size > max_size {
        return Err(LoadImageError::ImageSizeTooLarge(size));
    }

    // This is safe due to the bounds check above.
    let size = size as usize;

    image
        .seek(SeekFrom::Start(0))
        .map_err(LoadImageError::Seek)?;

    guest_mem
        .read_to_memory(guest_addr, image, size)
        .map_err(LoadImageError::ReadToMemory)?;

    Ok(size)
}

/// Load an image from a file into guest memory at the highest possible address.
///
/// # Arguments
///
/// * `guest_mem` - The memory to be used by the guest.
/// * `image` - The file containing the image to be loaded.
/// * `min_guest_addr` - The minimum address of the start of the image.
/// * `max_guest_addr` - The address to load the last byte of the image.
/// * `align` - The minimum alignment of the start address of the image in bytes
///   (must be a power of two).
///
/// The guest address and size in bytes of the loaded image are returned.
pub fn load_image_high<F>(
    guest_mem: &GuestMemory,
    image: &mut F,
    min_guest_addr: GuestAddress,
    max_guest_addr: GuestAddress,
    align: u64,
) -> Result<(GuestAddress, usize), LoadImageError>
where
    F: Read + Seek,
{
    if !align.is_power_of_two() {
        return Err(LoadImageError::BadAlignment(align));
    }

    let max_size = max_guest_addr.offset_from(min_guest_addr) & !(align - 1);
    let size = image.seek(SeekFrom::End(0)).map_err(LoadImageError::Seek)?;

    if size > usize::max_value() as u64 || size > max_size {
        return Err(LoadImageError::ImageSizeTooLarge(size));
    }

    image
        .seek(SeekFrom::Start(0))
        .map_err(LoadImageError::Seek)?;

    // Load image at the maximum aligned address allowed.
    // The subtraction cannot underflow because of the size checks above.
    let guest_addr = GuestAddress((max_guest_addr.offset() - size) & !(align - 1));

    // This is safe due to the bounds check above.
    let size = size as usize;

    guest_mem
        .read_to_memory(guest_addr, image, size)
        .map_err(LoadImageError::ReadToMemory)?;

    Ok((guest_addr, size))
}
<commit_msg>arch: initialize 4th serial port<commit_after>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

pub mod android;
pub mod fdt;

use std::collections::BTreeMap;
use std::error::Error as StdError;
use std::fmt::{self, Display};
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom};
use std::os::unix::io::AsRawFd;
use std::sync::Arc;

use devices::virtio::VirtioDevice;
use devices::{
    Bus, BusDevice, BusError, PciDevice, PciDeviceError, PciInterruptPin, PciRoot, ProxyDevice,
    Serial, SerialParameters, DEFAULT_SERIAL_PARAMS, SERIAL_ADDR,
};
use io_jail::Minijail;
use kvm::{IoeventAddress, Kvm, Vcpu, Vm};
use resources::SystemAllocator;
use sync::Mutex;
use sys_util::{syslog, EventFd, GuestAddress, GuestMemory, GuestMemoryError};

/// Holds the pieces needed to build a VM. Passed to `build_vm` in the `LinuxArch` trait below to
/// create a `RunnableLinuxVm`.
pub struct VmComponents {
    pub memory_size: u64,
    pub vcpu_count: u32,
    pub vcpu_affinity: Vec<usize>,
    pub kernel_image: File,
    pub android_fstab: Option<File>,
    pub initrd_image: Option<File>,
    pub extra_kernel_params: Vec<String>,
    pub wayland_dmabuf: bool,
}

/// Holds the elements needed to run a Linux VM. Created by `build_vm`.
pub struct RunnableLinuxVm {
    pub vm: Vm,
    pub kvm: Kvm,
    pub resources: SystemAllocator,
    pub stdio_serial: Option<Arc<Mutex<Serial>>>,
    pub exit_evt: EventFd,
    pub vcpus: Vec<Vcpu>,
    pub vcpu_affinity: Vec<usize>,
    pub irq_chip: Option<File>,
    pub io_bus: Bus,
    pub mmio_bus: Bus,
    pub pid_debug_label_map: BTreeMap<u32, String>,
}

/// The device and optional jail.
pub struct VirtioDeviceStub {
    pub dev: Box<dyn VirtioDevice>,
    pub jail: Option<Minijail>,
}

/// Trait which is implemented for each Linux Architecture in order to
/// set up the memory, cpus, and system devices and to boot the kernel.
pub trait LinuxArch {
    type Error: StdError;

    /// Takes `VmComponents` and generates a `RunnableLinuxVm`.
    ///
    /// # Arguments
    ///
    /// * `components` - Parts to use to build the VM.
    /// * `split_irqchip` - whether to use a split IRQ chip (i.e. userspace PIT/PIC/IOAPIC)
    /// * `serial_parameters` - definitions for how the serial devices should be configured.
    /// * `create_devices` - Function to generate a list of devices.
    fn build_vm<F, E>(
        components: VmComponents,
        split_irqchip: bool,
        serial_parameters: &BTreeMap<u8, SerialParameters>,
        create_devices: F,
    ) -> Result<RunnableLinuxVm, Self::Error>
    where
        F: FnOnce(&GuestMemory, &EventFd) -> Result<Vec<(Box<dyn PciDevice>, Option<Minijail>)>, E>,
        E: StdError + 'static;
}

/// Errors for device manager.
#[derive(Debug)]
pub enum DeviceRegistrationError {
    /// Could not allocate IO space for the device.
    AllocateIoAddrs(PciDeviceError),
    /// Could not allocate device address space for the device.
    AllocateDeviceAddrs(PciDeviceError),
    /// Could not allocate an IRQ number.
    AllocateIrq,
    /// Could not create the mmio device to wrap a VirtioDevice.
    CreateMmioDevice(sys_util::Error),
    //  Unable to create serial device from serial parameters
    CreateSerialDevice(devices::SerialError),
    /// Could not create an event fd.
    EventFdCreate(sys_util::Error),
    /// Could not add a device to the mmio bus.
    MmioInsert(BusError),
    /// Failed to register ioevent with VM.
    RegisterIoevent(sys_util::Error),
    /// Failed to register irq eventfd with VM.
    RegisterIrqfd(sys_util::Error),
    /// Failed to initialize proxy device for jailed device.
    ProxyDeviceCreation(devices::ProxyError),
    /// Appending to kernel command line failed.
    Cmdline(kernel_cmdline::Error),
    /// No more IRQs are available.
    IrqsExhausted,
    /// No more MMIO space available.
    AddrsExhausted,
    /// Could not register PCI device capabilities.
    RegisterDeviceCapabilities(PciDeviceError),
}

impl Display for DeviceRegistrationError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::DeviceRegistrationError::*;

        match self {
            AllocateIoAddrs(e) => write!(f, "Allocating IO addresses: {}", e),
            AllocateDeviceAddrs(e) => write!(f, "Allocating device addresses: {}", e),
            AllocateIrq => write!(f, "Allocating IRQ number"),
            CreateMmioDevice(e) => write!(f, "failed to create mmio device: {}", e),
            CreateSerialDevice(e) => write!(f, "failed to create serial device: {}", e),
            Cmdline(e) => write!(f, "unable to add device to kernel command line: {}", e),
            EventFdCreate(e) => write!(f, "failed to create eventfd: {}", e),
            MmioInsert(e) => write!(f, "failed to add to mmio bus: {}", e),
            RegisterIoevent(e) => write!(f, "failed to register ioevent to VM: {}", e),
            RegisterIrqfd(e) => write!(f, "failed to register irq eventfd to VM: {}", e),
            ProxyDeviceCreation(e) => write!(f, "failed to create proxy device: {}", e),
            IrqsExhausted => write!(f, "no more IRQs are available"),
            AddrsExhausted => write!(f, "no more addresses are available"),
            RegisterDeviceCapabilities(e) => {
                write!(f, "could not register PCI device capabilities: {}", e)
            }
        }
    }
}

/// Creates a root PCI device for use by this Vm.
pub fn generate_pci_root(
    devices: Vec<(Box<dyn PciDevice>, Option<Minijail>)>,
    mmio_bus: &mut Bus,
    resources: &mut SystemAllocator,
    vm: &mut Vm,
) -> Result<(PciRoot, Vec<(u32, PciInterruptPin)>, BTreeMap<u32, String>), DeviceRegistrationError>
{
    let mut root = PciRoot::new();
    let mut pci_irqs = Vec::new();
    let mut pid_labels = BTreeMap::new();
    for (dev_idx, (mut device, jail)) in devices.into_iter().enumerate() {
        // Only support one bus.
        device.assign_bus_dev(0, dev_idx as u8);

        let mut keep_fds = device.keep_fds();
        syslog::push_fds(&mut keep_fds);

        let irqfd = EventFd::new().map_err(DeviceRegistrationError::EventFdCreate)?;
        let irq_resample_fd = EventFd::new().map_err(DeviceRegistrationError::EventFdCreate)?;
        let irq_num = resources
            .allocate_irq()
            .ok_or(DeviceRegistrationError::AllocateIrq)? as u32;
        let pci_irq_pin = match dev_idx % 4 {
            0 => PciInterruptPin::IntA,
            1 => PciInterruptPin::IntB,
            2 => PciInterruptPin::IntC,
            3 => PciInterruptPin::IntD,
            _ => panic!(""), // Obviously not possible, but the compiler is not smart enough.
        };
        vm.register_irqfd_resample(&irqfd, &irq_resample_fd, irq_num)
            .map_err(DeviceRegistrationError::RegisterIrqfd)?;
        keep_fds.push(irqfd.as_raw_fd());
        keep_fds.push(irq_resample_fd.as_raw_fd());
        device.assign_irq(irqfd, irq_resample_fd, irq_num, pci_irq_pin);
        pci_irqs.push((dev_idx as u32, pci_irq_pin));

        let ranges = device
            .allocate_io_bars(resources)
            .map_err(DeviceRegistrationError::AllocateIoAddrs)?;
        let device_ranges = device
            .allocate_device_bars(resources)
            .map_err(DeviceRegistrationError::AllocateDeviceAddrs)?;
        device
            .register_device_capabilities()
            .map_err(DeviceRegistrationError::RegisterDeviceCapabilities)?;
        for (event, addr, datamatch) in device.ioeventfds() {
            let io_addr = IoeventAddress::Mmio(addr);
            vm.register_ioevent(&event, io_addr, datamatch)
                .map_err(DeviceRegistrationError::RegisterIoevent)?;
            keep_fds.push(event.as_raw_fd());
        }
        let arced_dev: Arc<Mutex<dyn BusDevice>> = if let Some(jail) = jail {
            let proxy = ProxyDevice::new(device, &jail, keep_fds)
                .map_err(DeviceRegistrationError::ProxyDeviceCreation)?;
            pid_labels.insert(proxy.pid() as u32, proxy.debug_label());
            Arc::new(Mutex::new(proxy))
        } else {
            device.on_sandboxed();
            Arc::new(Mutex::new(device))
        };
        root.add_device(arced_dev.clone());
        for range in &ranges {
            mmio_bus
                .insert(arced_dev.clone(), range.0, range.1, true)
                .map_err(DeviceRegistrationError::MmioInsert)?;
        }

        for range in &device_ranges {
            mmio_bus
                .insert(arced_dev.clone(), range.0, range.1, true)
                .map_err(DeviceRegistrationError::MmioInsert)?;
        }
    }
    Ok((root, pci_irqs, pid_labels))
}

/// Adds serial devices to the provided bus based on the serial parameters given. Returns the serial
///  port number and serial device to be used for stdout if defined.
///
/// # Arguments
///
/// * `io_bus` - Bus to add the devices to
/// * `com_evt_1_3` - eventfd for com1 and com3
/// * `com_evt_1_4` - eventfd for com2 and com4
/// * `io_bus` - Bus to add the devices to
/// * `serial_parameters` - definitions of serial parameter configuationis. If a setting is not
///     provided for a port, then it will use the default configuation.
pub fn add_serial_devices(
    io_bus: &mut Bus,
    com_evt_1_3: &EventFd,
    com_evt_2_4: &EventFd,
    serial_parameters: &BTreeMap<u8, SerialParameters>,
) -> Result<(Option<u8>, Option<Arc<Mutex<Serial>>>), DeviceRegistrationError> {
    let mut stdio_serial_num = None;
    let mut stdio_serial = None;

    for x in 0..=3 {
        let com_evt = match x {
            0 => com_evt_1_3,
            1 => com_evt_2_4,
            2 => com_evt_1_3,
            3 => com_evt_2_4,
            _ => com_evt_1_3,
        };

        let param = serial_parameters
            .get(&(x + 1))
            .unwrap_or(&DEFAULT_SERIAL_PARAMS[x as usize]);

        let com = Arc::new(Mutex::new(
            param
                .create_serial_device(&com_evt)
                .map_err(DeviceRegistrationError::CreateSerialDevice)?,
        ));
        io_bus
            .insert(com.clone(), SERIAL_ADDR[x as usize], 0x8, false)
            .unwrap();

        if param.console {
            stdio_serial_num = Some(x + 1);
            stdio_serial = Some(com.clone());
        }
    }

    Ok((stdio_serial_num, stdio_serial))
}

/// Errors for image loading.
#[derive(Debug)]
pub enum LoadImageError {
    BadAlignment(u64),
    Seek(io::Error),
    ImageSizeTooLarge(u64),
    ReadToMemory(GuestMemoryError),
}

impl Display for LoadImageError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::LoadImageError::*;

        match self {
            BadAlignment(a) => write!(f, "Alignment not a power of two: {}", a),
            Seek(e) => write!(f, "Seek failed: {}", e),
            ImageSizeTooLarge(size) => write!(f, "Image size too large: {}", size),
            ReadToMemory(e) => write!(f, "Reading image into memory failed: {}", e),
        }
    }
}

/// Load an image from a file into guest memory.
///
/// # Arguments
///
/// * `guest_mem` - The memory to be used by the guest.
/// * `guest_addr` - The starting address to load the image in the guest memory.
/// * `max_size` - The amount of space in bytes available in the guest memory for the image.
/// * `image` - The file containing the image to be loaded.
///
/// The size in bytes of the loaded image is returned.
pub fn load_image<F>(
    guest_mem: &GuestMemory,
    image: &mut F,
    guest_addr: GuestAddress,
    max_size: u64,
) -> Result<usize, LoadImageError>
where
    F: Read + Seek,
{
    let size = image.seek(SeekFrom::End(0)).map_err(LoadImageError::Seek)?;

    if size > usize::max_value() as u64 || size > max_size {
        return Err(LoadImageError::ImageSizeTooLarge(size));
    }

    // This is safe due to the bounds check above.
    let size = size as usize;

    image
        .seek(SeekFrom::Start(0))
        .map_err(LoadImageError::Seek)?;

    guest_mem
        .read_to_memory(guest_addr, image, size)
        .map_err(LoadImageError::ReadToMemory)?;

    Ok(size)
}

/// Load an image from a file into guest memory at the highest possible address.
///
/// # Arguments
///
/// * `guest_mem` - The memory to be used by the guest.
/// * `image` - The file containing the image to be loaded.
/// * `min_guest_addr` - The minimum address of the start of the image.
/// * `max_guest_addr` - The address to load the last byte of the image.
/// * `align` - The minimum alignment of the start address of the image in bytes
///   (must be a power of two).
///
/// The guest address and size in bytes of the loaded image are returned.
pub fn load_image_high<F>(
    guest_mem: &GuestMemory,
    image: &mut F,
    min_guest_addr: GuestAddress,
    max_guest_addr: GuestAddress,
    align: u64,
) -> Result<(GuestAddress, usize), LoadImageError>
where
    F: Read + Seek,
{
    if !align.is_power_of_two() {
        return Err(LoadImageError::BadAlignment(align));
    }

    let max_size = max_guest_addr.offset_from(min_guest_addr) & !(align - 1);
    let size = image.seek(SeekFrom::End(0)).map_err(LoadImageError::Seek)?;

    if size > usize::max_value() as u64 || size > max_size {
        return Err(LoadImageError::ImageSizeTooLarge(size));
    }

    image
        .seek(SeekFrom::Start(0))
        .map_err(LoadImageError::Seek)?;

    // Load image at the maximum aligned address allowed.
    // The subtraction cannot underflow because of the size checks above.
    let guest_addr = GuestAddress((max_guest_addr.offset() - size) & !(align - 1));

    // This is safe due to the bounds check above.
    let size = size as usize;

    guest_mem
        .read_to_memory(guest_addr, image, size)
        .map_err(LoadImageError::ReadToMemory)?;

    Ok((guest_addr, size))
}
<|endoftext|>


<commit_before>use std::any::Any;
use std::error::Error;

use {Response, Scope, EventSet};

/// A trait that every state machine in the loop must implement
pub trait Machine: Sized {
    /// Context type for the state machine
    ///
    /// This is a container of the global state for the application
    type Context;
    /// Seed is piece of data that is needed to initialize the machine
    ///
    /// It needs Any because it's put into Box<Error> object when state machine
    /// is failed to create. Hopefully this is not huge limitation.
    ///
    /// Note: this is only used to create machines returned by this machine.
    /// So unless this machine processses accepting socket this should
    /// probably be Void.
    type Seed: Any+Sized;

    /// Create a machine from some data
    ///
    /// The error should be rare enough so that Box<Error> overhead
    /// is negligible. Most errors here should be resource exhaustion, like
    /// there are no slots in Slab or system limit on epoll watches exceeded.
    ///
    /// Note: this method is used internally (by event loop) to create a
    /// socket from a Seed returned by this machine. This method should
    /// **not** be used to create machine by external code. Create a
    /// machine-specific `Type::new` method for the purpose.
    fn create(seed: Self::Seed, scope: &mut Scope<Self::Context>)
        -> Result<Self, Box<Error>>;

    /// Socket readiness notification
    fn ready(self, events: EventSet, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Called after spawn event
    ///
    /// This is mostly a continuation event. I.e. when you accept a socket
    /// and return a new state machine from `ready()`. You may wish to accept
    /// another socket right now. This is what `spawned` event is for.
    fn spawned(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Called instead of spawned, if there is no slab space
    ///
    /// For example, in `accept` handler you might want to put the thing
    /// into temporary storage, stop accepting and wait until slot is empty
    /// again.
    ///
    /// Note: it's useless to spawn from here, so we expect Option<Self> here.
    fn spawn_error(self, _scope: &mut Scope<Self::Context>, error: Box<Error>)
        -> Option<Self>
    {
        panic!("Error spawning state machine: {}", error);
    }

    /// Timeout happened
    fn timeout(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Message received
    fn wakeup(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;
}
<commit_msg>Add more documentation to `wakeup` method<commit_after>use std::any::Any;
use std::error::Error;

use {Response, Scope, EventSet};

/// A trait that every state machine in the loop must implement
pub trait Machine: Sized {
    /// Context type for the state machine
    ///
    /// This is a container of the global state for the application
    type Context;
    /// Seed is piece of data that is needed to initialize the machine
    ///
    /// It needs Any because it's put into Box<Error> object when state machine
    /// is failed to create. Hopefully this is not huge limitation.
    ///
    /// Note: this is only used to create machines returned by this machine.
    /// So unless this machine processses accepting socket this should
    /// probably be Void.
    type Seed: Any+Sized;

    /// Create a machine from some data
    ///
    /// The error should be rare enough so that Box<Error> overhead
    /// is negligible. Most errors here should be resource exhaustion, like
    /// there are no slots in Slab or system limit on epoll watches exceeded.
    ///
    /// Note: this method is used internally (by event loop) to create a
    /// socket from a Seed returned by this machine. This method should
    /// **not** be used to create machine by external code. Create a
    /// machine-specific `Type::new` method for the purpose.
    fn create(seed: Self::Seed, scope: &mut Scope<Self::Context>)
        -> Result<Self, Box<Error>>;

    /// Socket readiness notification
    fn ready(self, events: EventSet, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Called after spawn event
    ///
    /// This is mostly a continuation event. I.e. when you accept a socket
    /// and return a new state machine from `ready()`. You may wish to accept
    /// another socket right now. This is what `spawned` event is for.
    fn spawned(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Called instead of spawned, if there is no slab space
    ///
    /// For example, in `accept` handler you might want to put the thing
    /// into temporary storage, stop accepting and wait until slot is empty
    /// again.
    ///
    /// Note: it's useless to spawn from here, so we expect Option<Self> here.
    fn spawn_error(self, _scope: &mut Scope<Self::Context>, error: Box<Error>)
        -> Option<Self>
    {
        panic!("Error spawning state machine: {}", error);
    }

    /// Timeout happened
    fn timeout(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;

    /// Message received
    ///
    /// Note the spurious wakeups are possible, because messages are
    /// asynchronous, and state machine is identified by token.
    /// Tokens are reused quickly.
    ///
    /// So never make this `unreachable!()` or `unimplemented!()`
    fn wakeup(self, scope: &mut Scope<Self::Context>)
        -> Response<Self, Self::Seed>;
}
<|endoftext|>


<commit_before>// Copyright 2015 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.
extern crate zmq;
extern crate tempfile;

use zmq::{Error, SNDMORE};
use backend::{Backend, DEFAULT_DB_ADDRESS};
use data::{Task, TaskReport, TaskStatus, Service};

use std::thread;
use std::sync::Arc;
use std::sync::Mutex;

use std::ops::Deref;
use std::collections::HashMap;

use std::path::Path;
use std::fs::File;
// use tempfile::TempFile;
use std::io::{Write};
use std::io::Read;

pub struct TaskManager {
  pub source_port : usize,
  pub result_port : usize,
  pub queue_size : usize,
  pub message_size : usize,
  pub backend_address : String
}
pub struct Server {
  pub port : usize,
  pub queue_size : usize,
  pub message_size : usize,
  pub backend : Backend,
  pub backend_address : String
}

impl Default for TaskManager {
  fn default() -> TaskManager {
    TaskManager {
        source_port : 5555,
        result_port : 5555,
        queue_size : 100,
        message_size : 100000,
        backend_address : DEFAULT_DB_ADDRESS.clone().to_string()
    } } }

impl TaskManager {
  pub fn start<'manager>(&'manager self) -> Result<(), Error> {
    // We'll use some local memoization shared between source and sink:
    let services: HashMap<String, Option<Service>> = HashMap::new();
    let progress_queue: HashMap<i64, Task> = HashMap::new();
    let done_queue: Vec<TaskReport> = Vec::new();

    let services_arc = Arc::new(Mutex::new(services));
    let progress_queue_arc = Arc::new(Mutex::new(progress_queue));
    let done_queue_arc = Arc::new(Mutex::new(done_queue));

    // First prepare the source ventilator
    let source_port = self.source_port.clone();
    let source_queue_size = self.queue_size.clone();
    let source_message_size = self.message_size.clone();
    let source_backend_address = self.backend_address.clone();

    let vent_services_arc = services_arc.clone();
    let vent_progress_queue_arc = progress_queue_arc.clone();
    let vent_thread = thread::spawn(move || {
      let sources = Server {
        port : source_port,
        queue_size : source_queue_size,
        message_size : source_message_size,
        backend : Backend::from_address(&source_backend_address),
        backend_address : source_backend_address.clone()
      };
      sources.start_ventilator(vent_services_arc, vent_progress_queue_arc).unwrap();
    });

    // Next prepare the finalize thread which will persist finished jobs to the DB
    let finalize_backend_address = self.backend_address.clone();
    let finalize_done_queue_arc = done_queue_arc.clone();
    let finalize_thread = thread::spawn(move || {
      let finalize_backend = Backend::from_address(&finalize_backend_address);
      // Persist every 1 second, if there is something to record
      loop {
        match Server::mark_done_arc(&finalize_backend, &finalize_done_queue_arc) {
          true => {
            true;
          } // we did some work, on to the next iteration
          false => { // If we have no reports to process, sleep for a second and recheck
            thread::sleep_ms(1000);
          }
        }
      };
    });

    // Now prepare the results sink
    let result_port = self.result_port.clone();
    let result_queue_size = self.queue_size.clone();
    let result_message_size = self.message_size.clone();
    let result_backend_address = self.backend_address.clone();

    let sink_services_arc = services_arc.clone();
    let sink_progress_queue_arc = progress_queue_arc.clone();

    let sink_done_queue_arc = done_queue_arc.clone();
    let sink_thread = thread::spawn(move || {
      let results = Server {
        port : result_port,
        queue_size : result_queue_size,
        message_size : result_message_size,
        backend : Backend::from_address(&result_backend_address),
        backend_address: result_backend_address.clone()
      };
      results.start_sink(sink_services_arc, sink_progress_queue_arc, sink_done_queue_arc).unwrap();
    });

    vent_thread.join().unwrap();
    sink_thread.join().unwrap();
    finalize_thread.join().unwrap();
    Ok(())
  }
}

impl Server {
  pub fn start_ventilator(&self,
      services_arc : Arc<Mutex<HashMap<String, Option<Service>>>>,
      progress_queue_arc : Arc<Mutex<HashMap<i64, Task>>>)
      -> Result <(),Error> {
    // We have a Ventilator-exclusive "queues" stack for tasks to be dispatched
    let mut queues : HashMap<String, Vec<Task>> = HashMap::new();
    // Assuming this is the only And tidy up the postgres tasks:
    self.backend.clear_limbo_tasks().unwrap();
    // Ok, let's bind to a port and start broadcasting
    let mut context = zmq::Context::new();
    let mut ventilator = context.socket(zmq::ROUTER).unwrap();
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(ventilator.bind(&address).is_ok());
    let mut source_job_count = 0;

    loop {
      let mut msg = zmq::Message::new().unwrap();
      let mut identity = zmq::Message::new().unwrap();
      ventilator.recv(&mut identity, 0).unwrap();
      ventilator.recv(&mut msg, 0).unwrap();
      let service_name = msg.as_str().unwrap().to_string();
      println!("Task requested for service: {}", service_name.clone());
      source_job_count += 1;

      let mut dispatched_task : Option<Task> = None;
      match self.get_sync_service_record(&services_arc, service_name.clone()) {
        None => {},
        Some(service) => {
          if !queues.contains_key(&service_name) {
            queues.insert(service_name.clone(), Vec::new());
          }
          let mut task_queue : &mut Vec<Task> = queues.get_mut(&service_name).unwrap();
          if task_queue.is_empty() {
            task_queue.extend(self.backend.fetch_tasks(&service, self.queue_size).unwrap()); }
          match task_queue.pop() {
            Some(current_task) => {
              let taskid = current_task.id.unwrap();
              let serviceid = current_task.serviceid;

              dispatched_task = Some(current_task.clone());

              ventilator.send_msg(identity, SNDMORE).unwrap();
              ventilator.send_str(&taskid.to_string(), SNDMORE).unwrap();
              if serviceid == 1 { // No payload needed for init
                ventilator.send(&[],0).unwrap(); }
              else {
                // Regular services fetch the task payload and transfer it to the worker
                let file_opt = service.prepare_input_stream(current_task.clone());
                if file_opt.is_ok() {
                  let mut file = file_opt.unwrap();
                  let mut total_outgoing = 0;
                  loop {
                    // Stream input data via zmq
                    let mut data = vec![0; self.message_size];
                    let size = file.read(&mut data).unwrap();
                    total_outgoing += size;
                    data.truncate(size);

                    if size < self.message_size {
                      // If exhausted, send the last frame
                      ventilator.send(&data,0).unwrap();
                      // And terminate
                      break;
                    } else {
                      // If more to go, send the frame and indicate there's more to come
                      ventilator.send(&data,SNDMORE).unwrap();
                    }
                  }
                  println!("Source job {}, message size: {}", source_job_count, total_outgoing);
                } else {
                  // TODO: smart handling of failures
                  ventilator.send(&[],0).unwrap();
                }
              }
            },
            None => {}
          };
        }
      };
      // Record that a task has been dispatched in the progress queue
      if dispatched_task.is_some() {
        Server::push_progress_task(&progress_queue_arc, dispatched_task.unwrap());
      }
    }
  }

  pub fn start_sink(&self,
      services_arc : Arc<Mutex<HashMap<String, Option<Service>>>>,
      progress_queue_arc : Arc<Mutex<HashMap<i64, Task>>>,
      done_queue_arc : Arc<Mutex<Vec<TaskReport>>>)
      -> Result <(),Error> {

    // Ok, let's bind to a port and start broadcasting
    let mut context = zmq::Context::new();
    let mut sink = context.socket(zmq::PULL).unwrap();
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(sink.bind(&address).is_ok());

    let mut sink_job_count = 0;

    loop {
      let mut recv_msg = zmq::Message::new().unwrap();
      let mut taskid_msg = zmq::Message::new().unwrap();
      let mut service_msg = zmq::Message::new().unwrap();

      sink.recv(&mut service_msg, 0).unwrap();
      let service_name = service_msg.as_str().unwrap();

      sink.recv(&mut taskid_msg, 0).unwrap();
      let taskid_str = taskid_msg.as_str().unwrap();
      let taskid = taskid_str.parse::<i64>().unwrap();
      // We have a job, count it
      sink_job_count += 1;
      let mut total_incoming = 0;

      match Server::pop_progress_task(&progress_queue_arc, taskid) {
        None => {} // TODO: No such task, what to do?
        Some(task) => {

          // println!("{:?}", task);
          let service_option = Server::get_service_record(&services_arc, service_name.to_string());
          match service_option.clone() {
            None => {}, // TODO: Handle errors
            Some(service) => {
              let serviceid = service.id.unwrap();
              println!("Service: {:?}", serviceid);
              if serviceid == task.serviceid {
                // println!("Task and Service match up.");
                if serviceid == 1 { // No payload needed for init
                  sink.recv(&mut recv_msg, 0).unwrap();
                  let done_report = TaskReport {
                    task : task.clone(),
                    status : TaskStatus::NoProblem,
                    messages : Vec::new()
                  };
                  Server::push_done_queue(&done_queue_arc, done_report);
                }
                else {
                  // Receive the rest of the input in the correct file
                  let recv_dir = Path::new(&task.entry).parent().unwrap();
                  // "_" + &service.version.round().to_string() +
                  let recv_pathname = recv_dir.to_str().unwrap().to_string() + "/" + &service.name + ".zip";
                  let recv_path = Path::new(&recv_pathname);
                  // println!("Will write to {:?}", recv_path);
                  let mut file = File::create(recv_path).unwrap();
                  loop {
                    sink.recv(&mut recv_msg, 0).unwrap();

                    total_incoming += file.write(recv_msg.deref()).unwrap();
                    if !sink.get_rcvmore().unwrap() {
                      break;
                    }
                  }
                  // Then mark the task done. This can be in a new thread later on
                  let done_report = task.generate_report(recv_path);
                  Server::push_done_queue(&done_queue_arc, done_report);
                }

              }
              else {
                // Otherwise just discard the rest of the message
                loop {
                  sink.recv(&mut recv_msg, 0).unwrap();
                  if !sink.get_rcvmore().unwrap() {
                    break;
                  }
                }
              }
            }
          };
        }
      }
      println!("Sink job {}, message size: {}", sink_job_count, total_incoming);

      // let mut file = File::create("/tmp/cortex_sink_".to_string() + &sink_job_count.to_string()).unwrap();
      // file.write_all(&payload).unwrap();
    }
  }

  fn get_sync_service_record(&self, services_arc : &Arc<Mutex<HashMap<String, Option<Service>>>>, service_name : String) -> Option<Service> {
    let mut services = services_arc.lock().unwrap();
    let service_record = services.entry(service_name.clone()).or_insert(
      Service::from_name(&self.backend.connection, service_name.clone()).unwrap()).clone();
    service_record
  }

  fn get_service_record(services_arc : &Arc<Mutex<HashMap<String, Option<Service>>>>, service_name : String) -> Option<Service> {
    let services = services_arc.lock().unwrap();
    let service_record = services.get(&service_name);
    match service_record {
      None => None, // TODO: Handle errors
      Some(service_option) => service_option.clone()
    }
  }

  pub fn mark_done_arc(backend : &Backend, reports_arc: &Arc<Mutex<Vec<TaskReport>>>) -> bool {
    let mut reports = reports_arc.lock().unwrap();
    if reports.len() > 0 {
      backend.mark_done(&reports).unwrap(); // TODO: error handling if DB fails
      reports.clear();
      true
    } else {
      false
    }
  }
  pub fn push_done_queue(reports_arc : &Arc<Mutex<Vec<TaskReport>>>, report : TaskReport) {
    let mut reports = reports_arc.lock().unwrap();
    reports.push(report)
  }

  fn pop_progress_task(progress_queue_arc : &Arc<Mutex<HashMap<i64, Task>>>, taskid: i64) -> Option<Task> {
    let mut progress_queue = progress_queue_arc.lock().unwrap();
    progress_queue.remove(&taskid)
  }

  fn push_progress_task(progress_queue_arc : &Arc<Mutex<HashMap<i64, Task>>>, progress_task: Task) {
    let mut progress_queue = progress_queue_arc.lock().unwrap();
    progress_queue.insert(progress_task.id.unwrap(), progress_task);
  }
}<commit_msg>slightly better log messages for sink receiver<commit_after>// Copyright 2015 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.
extern crate zmq;
extern crate tempfile;

use zmq::{Error, SNDMORE};
use backend::{Backend, DEFAULT_DB_ADDRESS};
use data::{Task, TaskReport, TaskStatus, Service};

use std::thread;
use std::sync::Arc;
use std::sync::Mutex;

use std::ops::Deref;
use std::collections::HashMap;

use std::path::Path;
use std::fs::File;
// use tempfile::TempFile;
use std::io::{Write};
use std::io::Read;

pub struct TaskManager {
  pub source_port : usize,
  pub result_port : usize,
  pub queue_size : usize,
  pub message_size : usize,
  pub backend_address : String
}
pub struct Server {
  pub port : usize,
  pub queue_size : usize,
  pub message_size : usize,
  pub backend : Backend,
  pub backend_address : String
}

impl Default for TaskManager {
  fn default() -> TaskManager {
    TaskManager {
        source_port : 5555,
        result_port : 5555,
        queue_size : 100,
        message_size : 100000,
        backend_address : DEFAULT_DB_ADDRESS.clone().to_string()
    } } }

impl TaskManager {
  pub fn start<'manager>(&'manager self) -> Result<(), Error> {
    // We'll use some local memoization shared between source and sink:
    let services: HashMap<String, Option<Service>> = HashMap::new();
    let progress_queue: HashMap<i64, Task> = HashMap::new();
    let done_queue: Vec<TaskReport> = Vec::new();

    let services_arc = Arc::new(Mutex::new(services));
    let progress_queue_arc = Arc::new(Mutex::new(progress_queue));
    let done_queue_arc = Arc::new(Mutex::new(done_queue));

    // First prepare the source ventilator
    let source_port = self.source_port.clone();
    let source_queue_size = self.queue_size.clone();
    let source_message_size = self.message_size.clone();
    let source_backend_address = self.backend_address.clone();

    let vent_services_arc = services_arc.clone();
    let vent_progress_queue_arc = progress_queue_arc.clone();
    let vent_thread = thread::spawn(move || {
      let sources = Server {
        port : source_port,
        queue_size : source_queue_size,
        message_size : source_message_size,
        backend : Backend::from_address(&source_backend_address),
        backend_address : source_backend_address.clone()
      };
      sources.start_ventilator(vent_services_arc, vent_progress_queue_arc).unwrap();
    });

    // Next prepare the finalize thread which will persist finished jobs to the DB
    let finalize_backend_address = self.backend_address.clone();
    let finalize_done_queue_arc = done_queue_arc.clone();
    let finalize_thread = thread::spawn(move || {
      let finalize_backend = Backend::from_address(&finalize_backend_address);
      // Persist every 1 second, if there is something to record
      loop {
        match Server::mark_done_arc(&finalize_backend, &finalize_done_queue_arc) {
          true => {
            true;
          } // we did some work, on to the next iteration
          false => { // If we have no reports to process, sleep for a second and recheck
            thread::sleep_ms(1000);
          }
        }
      };
    });

    // Now prepare the results sink
    let result_port = self.result_port.clone();
    let result_queue_size = self.queue_size.clone();
    let result_message_size = self.message_size.clone();
    let result_backend_address = self.backend_address.clone();

    let sink_services_arc = services_arc.clone();
    let sink_progress_queue_arc = progress_queue_arc.clone();

    let sink_done_queue_arc = done_queue_arc.clone();
    let sink_thread = thread::spawn(move || {
      let results = Server {
        port : result_port,
        queue_size : result_queue_size,
        message_size : result_message_size,
        backend : Backend::from_address(&result_backend_address),
        backend_address: result_backend_address.clone()
      };
      results.start_sink(sink_services_arc, sink_progress_queue_arc, sink_done_queue_arc).unwrap();
    });

    vent_thread.join().unwrap();
    sink_thread.join().unwrap();
    finalize_thread.join().unwrap();
    Ok(())
  }
}

impl Server {
  pub fn start_ventilator(&self,
      services_arc : Arc<Mutex<HashMap<String, Option<Service>>>>,
      progress_queue_arc : Arc<Mutex<HashMap<i64, Task>>>)
      -> Result <(),Error> {
    // We have a Ventilator-exclusive "queues" stack for tasks to be dispatched
    let mut queues : HashMap<String, Vec<Task>> = HashMap::new();
    // Assuming this is the only And tidy up the postgres tasks:
    self.backend.clear_limbo_tasks().unwrap();
    // Ok, let's bind to a port and start broadcasting
    let mut context = zmq::Context::new();
    let mut ventilator = context.socket(zmq::ROUTER).unwrap();
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(ventilator.bind(&address).is_ok());
    let mut source_job_count = 0;

    loop {
      let mut msg = zmq::Message::new().unwrap();
      let mut identity = zmq::Message::new().unwrap();
      ventilator.recv(&mut identity, 0).unwrap();
      ventilator.recv(&mut msg, 0).unwrap();
      let service_name = msg.as_str().unwrap().to_string();
      println!("Task requested for service: {}", service_name.clone());
      source_job_count += 1;

      let mut dispatched_task : Option<Task> = None;
      match self.get_sync_service_record(&services_arc, service_name.clone()) {
        None => {},
        Some(service) => {
          if !queues.contains_key(&service_name) {
            queues.insert(service_name.clone(), Vec::new());
          }
          let mut task_queue : &mut Vec<Task> = queues.get_mut(&service_name).unwrap();
          if task_queue.is_empty() {
            task_queue.extend(self.backend.fetch_tasks(&service, self.queue_size).unwrap()); }
          match task_queue.pop() {
            Some(current_task) => {
              let taskid = current_task.id.unwrap();
              let serviceid = current_task.serviceid;

              dispatched_task = Some(current_task.clone());

              ventilator.send_msg(identity, SNDMORE).unwrap();
              ventilator.send_str(&taskid.to_string(), SNDMORE).unwrap();
              if serviceid == 1 { // No payload needed for init
                ventilator.send(&[],0).unwrap(); }
              else {
                // Regular services fetch the task payload and transfer it to the worker
                let file_opt = service.prepare_input_stream(current_task.clone());
                if file_opt.is_ok() {
                  let mut file = file_opt.unwrap();
                  let mut total_outgoing = 0;
                  loop {
                    // Stream input data via zmq
                    let mut data = vec![0; self.message_size];
                    let size = file.read(&mut data).unwrap();
                    total_outgoing += size;
                    data.truncate(size);

                    if size < self.message_size {
                      // If exhausted, send the last frame
                      ventilator.send(&data,0).unwrap();
                      // And terminate
                      break;
                    } else {
                      // If more to go, send the frame and indicate there's more to come
                      ventilator.send(&data,SNDMORE).unwrap();
                    }
                  }
                  println!("Source job {}, message size: {}", source_job_count, total_outgoing);
                } else {
                  // TODO: smart handling of failures
                  ventilator.send(&[],0).unwrap();
                }
              }
            },
            None => {}
          };
        }
      };
      // Record that a task has been dispatched in the progress queue
      if dispatched_task.is_some() {
        Server::push_progress_task(&progress_queue_arc, dispatched_task.unwrap());
      }
    }
  }

  pub fn start_sink(&self,
      services_arc : Arc<Mutex<HashMap<String, Option<Service>>>>,
      progress_queue_arc : Arc<Mutex<HashMap<i64, Task>>>,
      done_queue_arc : Arc<Mutex<Vec<TaskReport>>>)
      -> Result <(),Error> {

    // Ok, let's bind to a port and start broadcasting
    let mut context = zmq::Context::new();
    let mut sink = context.socket(zmq::PULL).unwrap();
    let port_str = self.port.to_string();
    let address = "tcp://*:".to_string() + &port_str;
    assert!(sink.bind(&address).is_ok());

    let mut sink_job_count = 0;

    loop {
      let mut recv_msg = zmq::Message::new().unwrap();
      let mut taskid_msg = zmq::Message::new().unwrap();
      let mut service_msg = zmq::Message::new().unwrap();

      sink.recv(&mut service_msg, 0).unwrap();
      let service_name = service_msg.as_str().unwrap();

      sink.recv(&mut taskid_msg, 0).unwrap();
      let taskid_str = taskid_msg.as_str().unwrap();
      let taskid = taskid_str.parse::<i64>().unwrap();
      // We have a job, count it
      sink_job_count += 1;
      let mut total_incoming = 0;
      println!("Incoming sink job {:?} for Service: {:?}, taskid: {:?}", sink_job_count, service_name, taskid_str);

      match Server::pop_progress_task(&progress_queue_arc, taskid) {
        None => {} // TODO: No such task, what to do?
        Some(task) => {

          // println!("{:?}", task);
          let service_option = Server::get_service_record(&services_arc, service_name.to_string());
          match service_option.clone() {
            None => {}, // TODO: Handle errors
            Some(service) => {
              let serviceid = service.id.unwrap();
              // println!("Service: {:?}", serviceid);
              if serviceid == task.serviceid {
                // println!("Task and Service match up.");
                if serviceid == 1 { // No payload needed for init
                  sink.recv(&mut recv_msg, 0).unwrap();
                  let done_report = TaskReport {
                    task : task.clone(),
                    status : TaskStatus::NoProblem,
                    messages : Vec::new()
                  };
                  Server::push_done_queue(&done_queue_arc, done_report);
                }
                else {
                  // Receive the rest of the input in the correct file
                  let recv_dir = Path::new(&task.entry).parent().unwrap();
                  // "_" + &service.version.round().to_string() +
                  let recv_pathname = recv_dir.to_str().unwrap().to_string() + "/" + &service.name + ".zip";
                  let recv_path = Path::new(&recv_pathname);
                  // println!("Will write to {:?}", recv_path);
                  let mut file = File::create(recv_path).unwrap();
                  loop {
                    sink.recv(&mut recv_msg, 0).unwrap();

                    total_incoming += file.write(recv_msg.deref()).unwrap();
                    if !sink.get_rcvmore().unwrap() {
                      break;
                    }
                  }
                  // Then mark the task done. This can be in a new thread later on
                  let done_report = task.generate_report(recv_path);
                  Server::push_done_queue(&done_queue_arc, done_report);
                }

              }
              else {
                // Otherwise just discard the rest of the message
                loop {
                  sink.recv(&mut recv_msg, 0).unwrap();
                  if !sink.get_rcvmore().unwrap() {
                    break;
                  }
                }
              }
            }
          };
        }
      }
      println!("Sink job {}, message size: {}", sink_job_count, total_incoming);

      // let mut file = File::create("/tmp/cortex_sink_".to_string() + &sink_job_count.to_string()).unwrap();
      // file.write_all(&payload).unwrap();
    }
  }

  fn get_sync_service_record(&self, services_arc : &Arc<Mutex<HashMap<String, Option<Service>>>>, service_name : String) -> Option<Service> {
    let mut services = services_arc.lock().unwrap();
    let service_record = services.entry(service_name.clone()).or_insert(
      Service::from_name(&self.backend.connection, service_name.clone()).unwrap()).clone();
    service_record
  }

  fn get_service_record(services_arc : &Arc<Mutex<HashMap<String, Option<Service>>>>, service_name : String) -> Option<Service> {
    let services = services_arc.lock().unwrap();
    let service_record = services.get(&service_name);
    match service_record {
      None => None, // TODO: Handle errors
      Some(service_option) => service_option.clone()
    }
  }

  pub fn mark_done_arc(backend : &Backend, reports_arc: &Arc<Mutex<Vec<TaskReport>>>) -> bool {
    let mut reports = reports_arc.lock().unwrap();
    if reports.len() > 0 {
      backend.mark_done(&reports).unwrap(); // TODO: error handling if DB fails
      reports.clear();
      true
    } else {
      false
    }
  }
  pub fn push_done_queue(reports_arc : &Arc<Mutex<Vec<TaskReport>>>, report : TaskReport) {
    let mut reports = reports_arc.lock().unwrap();
    reports.push(report)
  }

  fn pop_progress_task(progress_queue_arc : &Arc<Mutex<HashMap<i64, Task>>>, taskid: i64) -> Option<Task> {
    let mut progress_queue = progress_queue_arc.lock().unwrap();
    progress_queue.remove(&taskid)
  }

  fn push_progress_task(progress_queue_arc : &Arc<Mutex<HashMap<i64, Task>>>, progress_task: Task) {
    let mut progress_queue = progress_queue_arc.lock().unwrap();
    progress_queue.insert(progress_task.id.unwrap(), progress_task);
  }
}<|endoftext|>


<commit_before>// Zinc, the bare metal stack for rust.
// Copyright 2014 Ben Gamari <bgamari@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::iter::FromIterator;

use syntax::ext::base::ExtCtxt;
use syntax::ast;
use syntax::ptr::P;
use syntax::codemap::DUMMY_SP;
use syntax::ext::build::AstBuilder;
use syntax::parse::token;

use super::super::node;

/// Generate an unsuffixed integer literal expression with a dummy span
pub fn expr_int(cx: &ExtCtxt, n: i64) -> P<ast::Expr> {
  let sign = if n < 0 {ast::Minus} else {ast::Plus};
  cx.expr_lit(DUMMY_SP, ast::LitInt(n as u64, ast::UnsuffixedIntLit(sign)))
}

/// The name of the structure representing a register
pub fn path_ident(cx: &ExtCtxt, path: &Vec<String>)
                      -> ast::Ident {
  cx.ident_of(path.clone().connect("_").as_slice())
}

/// Generate a `#[name(...)]` attribute of the given type
pub fn list_attribute(cx: &ExtCtxt, name: &'static str,
    list: Vec<&'static str>) -> ast::Attribute {
  let words =
   list.into_iter()
   .map(|word| cx.meta_word(DUMMY_SP, token::InternedString::new(word)));
  let allow = cx.meta_list(DUMMY_SP, token::InternedString::new(name),
                                FromIterator::from_iter(words));
  cx.attribute(DUMMY_SP, allow)
}

/// Generate a `#[doc="..."]` attribute of the given type
pub fn doc_attribute(cx: &ExtCtxt, docstring: token::InternedString)
                     -> ast::Attribute {
  let s: ast::Lit_ = ast::LitStr(docstring, ast::CookedStr);
  let attr =
    cx.meta_name_value(DUMMY_SP, token::InternedString::new("doc"), s);
  cx.attribute(DUMMY_SP, attr)
}

pub fn primitive_type_path(cx: &ExtCtxt, width: &node::RegWidth)
                           -> ast::Path {
  let name = match width {
    &node::RegWidth::Reg8  => "u8",
    &node::RegWidth::Reg16 => "u16",
    &node::RegWidth::Reg32 => "u32",
  };
  cx.path_ident(DUMMY_SP, cx.ident_of(name))
}

/// The `Path` to the type corresponding to the primitive type of
/// the given register
pub fn reg_primitive_type_path(cx: &ExtCtxt, reg: &node::Reg)
                               -> Option<ast::Path> {
  match reg.ty {
    node::RegType::RegPrim(ref width, _) => Some(primitive_type_path(cx, width)),
    _ => None,
  }
}

pub fn reg_primitive_type(cx: &ExtCtxt, reg: &node::Reg)
                          -> Option<P<ast::Ty>> {
  let path = reg_primitive_type_path(cx, reg);
  path.map(|p| cx.ty_path(p))
}

pub fn field_type_path(cx: &ExtCtxt, path: &Vec<String>,
    reg: &node::Reg, field: &node::Field) -> ast::Path {
  let span = field.ty.span;
  match field.ty.node {
    node::FieldType::UIntField => {
      match reg.ty {
        node::RegType::RegPrim(ref width, _) => primitive_type_path(cx, width),
        _  => panic!("The impossible happened: a union register with fields"),
      }
    },
    node::FieldType::BoolField => cx.path_ident(span, cx.ident_of("bool")),
    node::FieldType::EnumField { ref opt_name, ..} => {
      match opt_name {
        &Some(ref name) =>
          cx.path_ident(span, cx.ident_of(name.as_slice())),
        &None => {
          let mut name = path.clone();
          name.push(field.name.node.clone());
          cx.path_ident(span, cx.ident_of(name.connect("_").as_slice()))
        }
      }
    },
  }
}

/// Build an expression for the mask of a field
pub fn mask(cx: &ExtCtxt, field: &node::Field) -> P<ast::Expr> {
  expr_int(cx, ((1 << field.width) - 1) as i64)
}

/// Build an expression for the shift of a field (including the array
/// index if necessary)
pub fn shift(cx: &ExtCtxt, idx: Option<P<ast::Expr>>,
                 field: &node::Field) -> P<ast::Expr> {
  let low = expr_int(cx, field.low_bit as i64);
  match idx {
    Some(idx) => {
      let width = expr_int(cx, field.width as i64);
      quote_expr!(cx, $low + $idx * $width)
    },
    None => low,
  }
}

/// The name of the setter type for a register
pub fn setter_name(cx: &ExtCtxt, path: &Vec<String>) -> ast::Ident {
  let mut s = path.clone();
  s.push("Update".to_string());
  path_ident(cx, &s)
}

/// The name of the getter type for a register
pub fn getter_name(cx: &ExtCtxt, path: &Vec<String>) -> ast::Ident {
  let mut s = path.clone();
  s.push("Get".to_string());
  path_ident(cx, &s)
}

pub fn intern_string(cx: &ExtCtxt, s: String) -> token::InternedString {
  token::get_ident(cx.ident_of(s.as_slice()))
}
<commit_msg>ioreg: Fix integer overflow<commit_after>// Zinc, the bare metal stack for rust.
// Copyright 2014 Ben Gamari <bgamari@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::iter::FromIterator;

use syntax::ext::base::ExtCtxt;
use syntax::ast;
use syntax::ptr::P;
use syntax::codemap::DUMMY_SP;
use syntax::ext::build::AstBuilder;
use syntax::parse::token;

use super::super::node;

/// Generate an unsuffixed integer literal expression with a dummy span
pub fn expr_int(cx: &ExtCtxt, n: i64) -> P<ast::Expr> {
  let sign = if n < 0 {ast::Minus} else {ast::Plus};
  cx.expr_lit(DUMMY_SP, ast::LitInt(n as u64, ast::UnsuffixedIntLit(sign)))
}

/// The name of the structure representing a register
pub fn path_ident(cx: &ExtCtxt, path: &Vec<String>)
                      -> ast::Ident {
  cx.ident_of(path.clone().connect("_").as_slice())
}

/// Generate a `#[name(...)]` attribute of the given type
pub fn list_attribute(cx: &ExtCtxt, name: &'static str,
    list: Vec<&'static str>) -> ast::Attribute {
  let words =
   list.into_iter()
   .map(|word| cx.meta_word(DUMMY_SP, token::InternedString::new(word)));
  let allow = cx.meta_list(DUMMY_SP, token::InternedString::new(name),
                                FromIterator::from_iter(words));
  cx.attribute(DUMMY_SP, allow)
}

/// Generate a `#[doc="..."]` attribute of the given type
pub fn doc_attribute(cx: &ExtCtxt, docstring: token::InternedString)
                     -> ast::Attribute {
  let s: ast::Lit_ = ast::LitStr(docstring, ast::CookedStr);
  let attr =
    cx.meta_name_value(DUMMY_SP, token::InternedString::new("doc"), s);
  cx.attribute(DUMMY_SP, attr)
}

pub fn primitive_type_path(cx: &ExtCtxt, width: &node::RegWidth)
                           -> ast::Path {
  let name = match width {
    &node::RegWidth::Reg8  => "u8",
    &node::RegWidth::Reg16 => "u16",
    &node::RegWidth::Reg32 => "u32",
  };
  cx.path_ident(DUMMY_SP, cx.ident_of(name))
}

/// The `Path` to the type corresponding to the primitive type of
/// the given register
pub fn reg_primitive_type_path(cx: &ExtCtxt, reg: &node::Reg)
                               -> Option<ast::Path> {
  match reg.ty {
    node::RegType::RegPrim(ref width, _) => Some(primitive_type_path(cx, width)),
    _ => None,
  }
}

pub fn reg_primitive_type(cx: &ExtCtxt, reg: &node::Reg)
                          -> Option<P<ast::Ty>> {
  let path = reg_primitive_type_path(cx, reg);
  path.map(|p| cx.ty_path(p))
}

pub fn field_type_path(cx: &ExtCtxt, path: &Vec<String>,
    reg: &node::Reg, field: &node::Field) -> ast::Path {
  let span = field.ty.span;
  match field.ty.node {
    node::FieldType::UIntField => {
      match reg.ty {
        node::RegType::RegPrim(ref width, _) => primitive_type_path(cx, width),
        _  => panic!("The impossible happened: a union register with fields"),
      }
    },
    node::FieldType::BoolField => cx.path_ident(span, cx.ident_of("bool")),
    node::FieldType::EnumField { ref opt_name, ..} => {
      match opt_name {
        &Some(ref name) =>
          cx.path_ident(span, cx.ident_of(name.as_slice())),
        &None => {
          let mut name = path.clone();
          name.push(field.name.node.clone());
          cx.path_ident(span, cx.ident_of(name.connect("_").as_slice()))
        }
      }
    },
  }
}

/// Build an expression for the mask of a field
pub fn mask(cx: &ExtCtxt, field: &node::Field) -> P<ast::Expr> {
  expr_int(cx, ((1 << field.width as u64) - 1))
}

/// Build an expression for the shift of a field (including the array
/// index if necessary)
pub fn shift(cx: &ExtCtxt, idx: Option<P<ast::Expr>>,
                 field: &node::Field) -> P<ast::Expr> {
  let low = expr_int(cx, field.low_bit as i64);
  match idx {
    Some(idx) => {
      let width = expr_int(cx, field.width as i64);
      quote_expr!(cx, $low + $idx * $width)
    },
    None => low,
  }
}

/// The name of the setter type for a register
pub fn setter_name(cx: &ExtCtxt, path: &Vec<String>) -> ast::Ident {
  let mut s = path.clone();
  s.push("Update".to_string());
  path_ident(cx, &s)
}

/// The name of the getter type for a register
pub fn getter_name(cx: &ExtCtxt, path: &Vec<String>) -> ast::Ident {
  let mut s = path.clone();
  s.push("Get".to_string());
  path_ident(cx, &s)
}

pub fn intern_string(cx: &ExtCtxt, s: String) -> token::InternedString {
  token::get_ident(cx.ident_of(s.as_slice()))
}
<|endoftext|>


<commit_before>//! etcd's members API.
//!
//! These API endpoints are used to manage cluster membership.

use std::str::FromStr;

use futures::{Future, IntoFuture, Stream};
use hyper::{StatusCode, Uri};
use hyper::client::Connect;
use serde_json;

use async::first_ok;
use client::{Client, ClusterInfo, Response};
use error::{ApiError, Error};

/// An etcd server that is a member of a cluster.
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq)]
pub struct Member {
    /// An internal identifier for the cluster member.
    pub id: String,
    /// A human-readable name for the cluster member.
    pub name: String,
    /// URLs exposing this cluster member's peer API.
    #[serde(rename = "peerURLs")]
    pub peer_urls: Vec<String>,
    /// URLs exposing this cluster member's client API.
    #[serde(rename = "clientURLs")]
    pub client_urls: Vec<String>,
}

/// The request body for `POST /v2/members`.
#[derive(Debug, Serialize)]
struct AddMember {
    #[serde(rename = "peerURLs")]
    pub peer_urls: Vec<String>,
}

/// A small wrapper around `Member` to match the response of `GET /v2/members`.
#[derive(Debug, Deserialize)]
struct ListResponse {
    members: Vec<Member>,
}

/// Add a new member to the cluster.
///
/// # Parameters
///
/// * client: A `Client` to use to make the API call.
/// * peer_urls: URLs exposing this cluster member's peer API.
pub fn add_member<C>(
    client: &Client<C>,
    peer_urls: Vec<String>,
) -> Box<Future<Item = Response<()>, Error = Vec<Error>>>
where
    C: Clone + Connect,
{
    let add_member = AddMember { peer_urls };

    let body = match serde_json::to_string(&add_member) {
        Ok(body) => body,
        Err(error) => return Box::new(Err(vec![Error::Serialization(error)]).into_future()),
    };

    let http_client = client.http_client().clone();

    let result = first_ok(client.endpoints().to_vec(), move |member| {
        let url = build_url(member, "");
        let uri = Uri::from_str(url.as_str())
            .map_err(Error::from)
            .into_future();

        let body = body.clone();
        let http_client = http_client.clone();

        let response = uri.and_then(move |uri| http_client.post(uri, body).map_err(Error::from));

        let result = response.and_then(|response| {
            let status = response.status();
            let cluster_info = ClusterInfo::from(response.headers());
            let body = response.body().concat2().map_err(Error::from);

            body.and_then(move |ref body| if status == StatusCode::Created {
                Ok(Response {
                    data: (),
                    cluster_info,
                })
            } else {
                match serde_json::from_slice::<ApiError>(body) {
                    Ok(error) => Err(Error::Api(error)),
                    Err(error) => Err(Error::Serialization(error)),
                }
            })
        });

        Box::new(result)
    });

    Box::new(result)
}

/// Lists the members of the cluster.
///
/// # Parameters
///
/// * client: A `Client` to use to make the API call.
pub fn list<C>(client: &Client<C>) -> Box<Future<Item = Response<Vec<Member>>, Error = Vec<Error>>>
where
    C: Clone + Connect,
{
    let http_client = client.http_client().clone();

    let result = first_ok(client.endpoints().to_vec(), move |member| {
        let url = build_url(member, "");
        let uri = Uri::from_str(url.as_str())
            .map_err(Error::from)
            .into_future();

        let http_client = http_client.clone();

        let response = uri.and_then(move |uri| http_client.get(uri).map_err(Error::from));

        let result = response.and_then(|response| {
            let status = response.status();
            let cluster_info = ClusterInfo::from(response.headers());
            let body = response.body().concat2().map_err(Error::from);

            body.and_then(move |ref body| if status == StatusCode::Ok {
                match serde_json::from_slice::<ListResponse>(body) {
                    Ok(data) => Ok(Response {
                        data: data.members,
                        cluster_info,
                    }),
                    Err(error) => Err(Error::Serialization(error)),
                }
            } else {
                match serde_json::from_slice::<ApiError>(body) {
                    Ok(error) => Err(Error::Api(error)),
                    Err(error) => Err(Error::Serialization(error)),
                }
            })
        });

        Box::new(result)
    });

    Box::new(result)
}

/// Constructs the full URL for an API call.
fn build_url(endpoint: &Uri, path: &str) -> String {
    let maybe_slash = if endpoint.as_ref().ends_with("/") {
        ""
    } else {
        "/"
    };

    format!("{}{}v2/members{}", endpoint, maybe_slash, path)
}
<commit_msg>Add members::delete_member endpoint.<commit_after>//! etcd's members API.
//!
//! These API endpoints are used to manage cluster membership.

use std::str::FromStr;

use futures::{Future, IntoFuture, Stream};
use hyper::{StatusCode, Uri};
use hyper::client::Connect;
use serde_json;

use async::first_ok;
use client::{Client, ClusterInfo, Response};
use error::{ApiError, Error};

/// An etcd server that is a member of a cluster.
#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq)]
pub struct Member {
    /// An internal identifier for the cluster member.
    pub id: String,
    /// A human-readable name for the cluster member.
    pub name: String,
    /// URLs exposing this cluster member's peer API.
    #[serde(rename = "peerURLs")]
    pub peer_urls: Vec<String>,
    /// URLs exposing this cluster member's client API.
    #[serde(rename = "clientURLs")]
    pub client_urls: Vec<String>,
}

/// The request body for `POST /v2/members`.
#[derive(Debug, Serialize)]
struct AddMember {
    #[serde(rename = "peerURLs")]
    pub peer_urls: Vec<String>,
}

/// A small wrapper around `Member` to match the response of `GET /v2/members`.
#[derive(Debug, Deserialize)]
struct ListResponse {
    members: Vec<Member>,
}

/// Add a new member to the cluster.
///
/// # Parameters
///
/// * client: A `Client` to use to make the API call.
/// * peer_urls: URLs exposing this cluster member's peer API.
pub fn add_member<C>(
    client: &Client<C>,
    peer_urls: Vec<String>,
) -> Box<Future<Item = Response<()>, Error = Vec<Error>>>
where
    C: Clone + Connect,
{
    let add_member = AddMember { peer_urls };

    let body = match serde_json::to_string(&add_member) {
        Ok(body) => body,
        Err(error) => return Box::new(Err(vec![Error::Serialization(error)]).into_future()),
    };

    let http_client = client.http_client().clone();

    let result = first_ok(client.endpoints().to_vec(), move |member| {
        let url = build_url(member, "");
        let uri = Uri::from_str(url.as_str())
            .map_err(Error::from)
            .into_future();

        let body = body.clone();
        let http_client = http_client.clone();

        let response = uri.and_then(move |uri| http_client.post(uri, body).map_err(Error::from));

        let result = response.and_then(|response| {
            let status = response.status();
            let cluster_info = ClusterInfo::from(response.headers());
            let body = response.body().concat2().map_err(Error::from);

            body.and_then(move |ref body| if status == StatusCode::Created {
                Ok(Response {
                    data: (),
                    cluster_info,
                })
            } else {
                match serde_json::from_slice::<ApiError>(body) {
                    Ok(error) => Err(Error::Api(error)),
                    Err(error) => Err(Error::Serialization(error)),
                }
            })
        });

        Box::new(result)
    });

    Box::new(result)
}

/// Delete a member from the cluster.
///
/// # Parameters
///
/// * client: A `Client` to use to make the API call.
/// * id: The unique identifier of the member to delete.
pub fn delete_member<C>(
    client: &Client<C>,
    id: String,
) -> Box<Future<Item = Response<()>, Error = Vec<Error>>>
where
    C: Clone + Connect,
{
    let http_client = client.http_client().clone();

    let result = first_ok(client.endpoints().to_vec(), move |member| {
        let url = build_url(member, &format!("/{}", id));
        let uri = Uri::from_str(url.as_str())
            .map_err(Error::from)
            .into_future();

        let http_client = http_client.clone();

        let response = uri.and_then(move |uri| http_client.delete(uri).map_err(Error::from));

        let result = response.and_then(|response| {
            let status = response.status();
            let cluster_info = ClusterInfo::from(response.headers());
            let body = response.body().concat2().map_err(Error::from);

            body.and_then(move |ref body| if status == StatusCode::NoContent {
                Ok(Response {
                    data: (),
                    cluster_info,
                })
            } else {
                match serde_json::from_slice::<ApiError>(body) {
                    Ok(error) => Err(Error::Api(error)),
                    Err(error) => Err(Error::Serialization(error)),
                }
            })
        });

        Box::new(result)
    });

    Box::new(result)
}
/// Lists the members of the cluster.
///
/// # Parameters
///
/// * client: A `Client` to use to make the API call.
pub fn list<C>(client: &Client<C>) -> Box<Future<Item = Response<Vec<Member>>, Error = Vec<Error>>>
where
    C: Clone + Connect,
{
    let http_client = client.http_client().clone();

    let result = first_ok(client.endpoints().to_vec(), move |member| {
        let url = build_url(member, "");
        let uri = Uri::from_str(url.as_str())
            .map_err(Error::from)
            .into_future();

        let http_client = http_client.clone();

        let response = uri.and_then(move |uri| http_client.get(uri).map_err(Error::from));

        let result = response.and_then(|response| {
            let status = response.status();
            let cluster_info = ClusterInfo::from(response.headers());
            let body = response.body().concat2().map_err(Error::from);

            body.and_then(move |ref body| if status == StatusCode::Ok {
                match serde_json::from_slice::<ListResponse>(body) {
                    Ok(data) => Ok(Response {
                        data: data.members,
                        cluster_info,
                    }),
                    Err(error) => Err(Error::Serialization(error)),
                }
            } else {
                match serde_json::from_slice::<ApiError>(body) {
                    Ok(error) => Err(Error::Api(error)),
                    Err(error) => Err(Error::Serialization(error)),
                }
            })
        });

        Box::new(result)
    });

    Box::new(result)
}

/// Constructs the full URL for an API call.
fn build_url(endpoint: &Uri, path: &str) -> String {
    let maybe_slash = if endpoint.as_ref().ends_with("/") {
        ""
    } else {
        "/"
    };

    format!("{}{}v2/members{}", endpoint, maybe_slash, path)
}
<|endoftext|>