<commit_before>//! GPU-visible memory allocation and management.
//!
//! When you create a buffer or a texture with Vulkan, you have to bind it to a chunk of allocated
//! memory. To do so, you have to pass a type that implements the `MemorySource` trait.
//!
//! There are several implementations of the trait, ie. several things that you can pass to the
//! constructors of buffers and textures:
//!
//! - `&Arc<Device>`, which will simply allocate a new chunk of memory every time (easy but not
//!   very efficient).
//! - `MemorySource`, which is the same as `&Arc<Device>` except that it will use the
//!   already-allocated block.
//! - ... needs more ...
//!
//! # Synchronization
//!
//! In Vulkan, it's the job of the programmer to enforce memory safety. In other words, the
//! programmer must take care that two chunks of memory are not read and written simultaneously.
//!
//! In this library, this is enforced by the implementation of `MemorySource` or
//! `MemorySourceChunk`.
//!
//! There are two mechanisms in Vulkan that can provide synchronization: fences and semaphores.
//! Fences provide synchronization between the CPU and the GPU, and semaphores provide
//! synchronization between multiple queues of the GPU. See the `sync` module for more info.
//!
//! # Sparse resources
//!
//! **Not yet implemented**.
//!
//! Instead of creating a buffer or an image with a single chunk of memory, you also have the
//! possibility to create resources with *sparse memory*.
//!
//! For example you can bind the first half of the buffer to a memory chunk, and the second half of
//! the buffer to another memory chunk.
//!
//! There is a hierarchy of three features related to sparse resources:
//!
//!  - The `sparseBinding` feature allows you to use sparse resources.
//!  - The `sparseResidency` feature is a superset of `sparseBinding` and allows you to leave some
//!    parts of the resource unbinded before using it.
//!  - The `sparseResidencyAliased` feature is a superset of `sparseResidency` and allows you to
//!    bind the same memory chunk to multiple different resources at once.
//!
use std::mem;
use std::os::raw::c_void;
use std::slice;
use std::sync::Arc;

use sync::Fence;
use sync::Semaphore;

use device::Device;
use device::Queue;

use OomError;

pub use self::device_memory::DeviceMemory;
pub use self::device_memory::MappedDeviceMemory;
pub use self::single::DeviceLocal;
pub use self::single::DeviceLocalChunk;
pub use self::single::HostVisible;
pub use self::single::HostVisibleChunk;

mod device_memory;
mod single;

/// Trait for memory objects that can be accessed from the CPU.
pub unsafe trait CpuAccessible<'a, T: ?Sized> {
    type Read;

    /// Gives a read access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, blocks until it is available.
    // TODO: what happens if timeout is reached? a panic?
    fn read(&'a self, timeout_ns: u64) -> Self::Read;

    /// Tries to give a read access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, returns `None`.
    fn try_read(&'a self) -> Option<Self::Read>;
}

/// Trait for memory objects that be mutably accessed from the CPU.
pub unsafe trait CpuWriteAccessible<'a, T: ?Sized>: CpuAccessible<'a, T> {
    type Write;

    /// Gives a write access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, blocks until it is available.
    // TODO: what happens if timeout is reached? a panic?
    fn write(&'a self, timeout_ns: u64) -> Self::Write;

    /// Tries to give a write access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, returns `None`.
    fn try_write(&'a self) -> Option<Self::Write>;
}

/// Trait for objects that can be used to fill the memory requirements of a buffer or an image.
pub unsafe trait MemorySource {
    /// An object that represents one block of allocation. Returned by `allocate`.
    type Chunk: MemorySourceChunk;

    /// Returns true if the chunks allocated by this source will use sparse memory.
    // TODO: should return the level of the required sparse feature
    fn is_sparse(&self) -> bool;

    /// Allocates a block of memory to be used.
    ///
    /// `memory_type_bits` is a bitsfield which indicates from which memory type the memory can
    /// be allocated. For example if the bit 2 is set (`memory_type_bits & (1 << 2) != 0`), that
    /// means that the memory type whose ID is 2 can be used.
    ///
    /// The implementation is allowed to return a chunk with a larger size or alignment.
    fn allocate(self, &Arc<Device>, size: usize, alignment: usize, memory_type_bits: u32)
                -> Result<Self::Chunk, OomError>;
}

/// A chunk of GPU-visible memory.
pub unsafe trait MemorySourceChunk {
    /// Returns the properties of this chunk.
    fn properties(&self) -> ChunkProperties;

    /// Returns true if the `gpu_access` function should be passed a fence.
    #[inline]
    fn requires_fence(&self) -> bool {
        true
    }

    /// Returns true if the `gpu_access` function should be passed a semaphore.
    #[inline]
    fn requires_semaphore(&self) -> bool {
        true
    }

    /// Instructs the manager that a part of this chunk of memory is going to be used by the
    /// GPU soon in the future. The function should block if the memory is currently being
    /// accessed by the CPU.
    ///
    /// `write` indicates whether the GPU will write to the memory. If `false`, then it will only
    /// be written.
    ///
    /// `range` indicates the part of the chunk that is concerned.
    ///
    /// `queue` is the queue where the command buffer that accesses the memory will be submitted.
    /// If the `gpu_access` function submits something to that queue, it will thus be submitted
    /// beforehand. This behavior can be used for example to submit sparse binding commands.
    ///
    /// `fence` is a fence that will be signaled when this GPU access will stop. It should be
    /// waited upon whenever the user wants to read this memory from the CPU. If `requires_fence`
    /// returned false, then this value will be `None`.
    ///
    /// `semaphore` is a semaphore that will be signaled when this GPU access will stop. This value
    /// is intended to be returned later, in a follow-up call to `gpu_access`. If
    /// `requires_semaphore` returned false, then this value will be `None`.
    ///
    /// The manager must track whether this chunk of memory is being accessed by the CPU/GPU and
    /// return a semaphore that must be waited upon by the GPU before the access can start. The
    /// semaphore being returned is usually one that has been previously passed to this function,
    /// but it doesn't need to be the case.
    unsafe fn gpu_access(&self, write: bool, range: ChunkRange, queue: &mut Queue,
                         fence: Option<Arc<Fence>>, semaphore: Option<Arc<Semaphore>>)
                         -> Option<Arc<Semaphore>>;

    /// Returns true if this chunk of memory may be used, now or in the future, by multiple buffers
    /// or images (or a combination of both) simultaneously. If you're not sure, it's safer to
    /// return true.
    ///
    /// If this value is true, then the Vulkan implementation must be more conservative about
    /// reordering subpasses in a renderpass.
    fn may_alias(&self) -> bool;
}

pub enum ChunkRange {
    All,
    Range { offset: usize, size: usize }
}

pub enum ChunkProperties<'a> {
    Regular {
        memory: &'a DeviceMemory,
        offset: usize,
        size: usize,
    },

    Sparse,     // TODO: unimplemented
}

/// Trait for types of data that can be mapped.
pub unsafe trait Content {
    /// Builds a pointer to this type from a raw pointer.
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut Self>;

    /// Returns true if the size is suitable to store a type like this.
    fn is_size_suitable(usize) -> bool;
}

unsafe impl<T> Content for T where T: Copy {
    #[inline]
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut T> {
        if size != mem::size_of::<T>() {
            return None;
        }

        Some(ptr as *mut T)
    }

    #[inline]
    fn is_size_suitable(size: usize) -> bool {
        size == mem::size_of::<T>()
    }
}

unsafe impl<T> Content for [T] where T: Copy {
    #[inline]
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut [T]> {
        if size % mem::size_of::<T>() != 0 {
            return None;
        }

        let ptr = ptr as *mut T;
        let size = size / mem::size_of::<T>();
        Some(unsafe { slice::from_raw_parts_mut(&mut *ptr, size) as *mut [T] })
    }

    #[inline]
    fn is_size_suitable(size: usize) -> bool {
        size % mem::size_of::<T>() == 0
    }
}
<commit_msg>Relax rules for buffer content<commit_after>//! GPU-visible memory allocation and management.
//!
//! When you create a buffer or a texture with Vulkan, you have to bind it to a chunk of allocated
//! memory. To do so, you have to pass a type that implements the `MemorySource` trait.
//!
//! There are several implementations of the trait, ie. several things that you can pass to the
//! constructors of buffers and textures:
//!
//! - `&Arc<Device>`, which will simply allocate a new chunk of memory every time (easy but not
//!   very efficient).
//! - `MemorySource`, which is the same as `&Arc<Device>` except that it will use the
//!   already-allocated block.
//! - ... needs more ...
//!
//! # Synchronization
//!
//! In Vulkan, it's the job of the programmer to enforce memory safety. In other words, the
//! programmer must take care that two chunks of memory are not read and written simultaneously.
//!
//! In this library, this is enforced by the implementation of `MemorySource` or
//! `MemorySourceChunk`.
//!
//! There are two mechanisms in Vulkan that can provide synchronization: fences and semaphores.
//! Fences provide synchronization between the CPU and the GPU, and semaphores provide
//! synchronization between multiple queues of the GPU. See the `sync` module for more info.
//!
//! # Sparse resources
//!
//! **Not yet implemented**.
//!
//! Instead of creating a buffer or an image with a single chunk of memory, you also have the
//! possibility to create resources with *sparse memory*.
//!
//! For example you can bind the first half of the buffer to a memory chunk, and the second half of
//! the buffer to another memory chunk.
//!
//! There is a hierarchy of three features related to sparse resources:
//!
//!  - The `sparseBinding` feature allows you to use sparse resources.
//!  - The `sparseResidency` feature is a superset of `sparseBinding` and allows you to leave some
//!    parts of the resource unbinded before using it.
//!  - The `sparseResidencyAliased` feature is a superset of `sparseResidency` and allows you to
//!    bind the same memory chunk to multiple different resources at once.
//!
use std::mem;
use std::os::raw::c_void;
use std::slice;
use std::sync::Arc;

use sync::Fence;
use sync::Semaphore;

use device::Device;
use device::Queue;

use OomError;

pub use self::device_memory::DeviceMemory;
pub use self::device_memory::MappedDeviceMemory;
pub use self::single::DeviceLocal;
pub use self::single::DeviceLocalChunk;
pub use self::single::HostVisible;
pub use self::single::HostVisibleChunk;

mod device_memory;
mod single;

/// Trait for memory objects that can be accessed from the CPU.
pub unsafe trait CpuAccessible<'a, T: ?Sized> {
    type Read;

    /// Gives a read access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, blocks until it is available.
    // TODO: what happens if timeout is reached? a panic?
    fn read(&'a self, timeout_ns: u64) -> Self::Read;

    /// Tries to give a read access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, returns `None`.
    fn try_read(&'a self) -> Option<Self::Read>;
}

/// Trait for memory objects that be mutably accessed from the CPU.
pub unsafe trait CpuWriteAccessible<'a, T: ?Sized>: CpuAccessible<'a, T> {
    type Write;

    /// Gives a write access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, blocks until it is available.
    // TODO: what happens if timeout is reached? a panic?
    fn write(&'a self, timeout_ns: u64) -> Self::Write;

    /// Tries to give a write access to the content of the buffer.
    ///
    /// If the buffer is in use by the GPU, returns `None`.
    fn try_write(&'a self) -> Option<Self::Write>;
}

/// Trait for objects that can be used to fill the memory requirements of a buffer or an image.
pub unsafe trait MemorySource {
    /// An object that represents one block of allocation. Returned by `allocate`.
    type Chunk: MemorySourceChunk;

    /// Returns true if the chunks allocated by this source will use sparse memory.
    // TODO: should return the level of the required sparse feature
    fn is_sparse(&self) -> bool;

    /// Allocates a block of memory to be used.
    ///
    /// `memory_type_bits` is a bitsfield which indicates from which memory type the memory can
    /// be allocated. For example if the bit 2 is set (`memory_type_bits & (1 << 2) != 0`), that
    /// means that the memory type whose ID is 2 can be used.
    ///
    /// The implementation is allowed to return a chunk with a larger size or alignment.
    fn allocate(self, &Arc<Device>, size: usize, alignment: usize, memory_type_bits: u32)
                -> Result<Self::Chunk, OomError>;
}

/// A chunk of GPU-visible memory.
pub unsafe trait MemorySourceChunk {
    /// Returns the properties of this chunk.
    fn properties(&self) -> ChunkProperties;

    /// Returns true if the `gpu_access` function should be passed a fence.
    #[inline]
    fn requires_fence(&self) -> bool {
        true
    }

    /// Returns true if the `gpu_access` function should be passed a semaphore.
    #[inline]
    fn requires_semaphore(&self) -> bool {
        true
    }

    /// Instructs the manager that a part of this chunk of memory is going to be used by the
    /// GPU soon in the future. The function should block if the memory is currently being
    /// accessed by the CPU.
    ///
    /// `write` indicates whether the GPU will write to the memory. If `false`, then it will only
    /// be written.
    ///
    /// `range` indicates the part of the chunk that is concerned.
    ///
    /// `queue` is the queue where the command buffer that accesses the memory will be submitted.
    /// If the `gpu_access` function submits something to that queue, it will thus be submitted
    /// beforehand. This behavior can be used for example to submit sparse binding commands.
    ///
    /// `fence` is a fence that will be signaled when this GPU access will stop. It should be
    /// waited upon whenever the user wants to read this memory from the CPU. If `requires_fence`
    /// returned false, then this value will be `None`.
    ///
    /// `semaphore` is a semaphore that will be signaled when this GPU access will stop. This value
    /// is intended to be returned later, in a follow-up call to `gpu_access`. If
    /// `requires_semaphore` returned false, then this value will be `None`.
    ///
    /// The manager must track whether this chunk of memory is being accessed by the CPU/GPU and
    /// return a semaphore that must be waited upon by the GPU before the access can start. The
    /// semaphore being returned is usually one that has been previously passed to this function,
    /// but it doesn't need to be the case.
    unsafe fn gpu_access(&self, write: bool, range: ChunkRange, queue: &mut Queue,
                         fence: Option<Arc<Fence>>, semaphore: Option<Arc<Semaphore>>)
                         -> Option<Arc<Semaphore>>;

    /// Returns true if this chunk of memory may be used, now or in the future, by multiple buffers
    /// or images (or a combination of both) simultaneously. If you're not sure, it's safer to
    /// return true.
    ///
    /// If this value is true, then the Vulkan implementation must be more conservative about
    /// reordering subpasses in a renderpass.
    fn may_alias(&self) -> bool;
}

pub enum ChunkRange {
    All,
    Range { offset: usize, size: usize }
}

pub enum ChunkProperties<'a> {
    Regular {
        memory: &'a DeviceMemory,
        offset: usize,
        size: usize,
    },

    Sparse,     // TODO: unimplemented
}

/// Trait for types of data that can be mapped.
pub unsafe trait Content {
    /// Builds a pointer to this type from a raw pointer.
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut Self>;

    /// Returns true if the size is suitable to store a type like this.
    fn is_size_suitable(usize) -> bool;
}

unsafe impl<T> Content for T {
    #[inline]
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut T> {
        if size != mem::size_of::<T>() {
            return None;
        }

        Some(ptr as *mut T)
    }

    #[inline]
    fn is_size_suitable(size: usize) -> bool {
        size == mem::size_of::<T>()
    }
}

unsafe impl<T> Content for [T] {
    #[inline]
    fn ref_from_ptr<'a>(ptr: *mut c_void, size: usize) -> Option<*mut [T]> {
        if size % mem::size_of::<T>() != 0 {
            return None;
        }

        let ptr = ptr as *mut T;
        let size = size / mem::size_of::<T>();
        Some(unsafe { slice::from_raw_parts_mut(&mut *ptr, size) as *mut [T] })
    }

    #[inline]
    fn is_size_suitable(size: usize) -> bool {
        size % mem::size_of::<T>() == 0
    }
}
<|endoftext|>


<commit_before>use crate::metrics::{AtomicNumber, Descriptor, MetricsError, Number, NumberKind, Result};
use crate::sdk::export::metrics::{Buckets, Count, Histogram, Sum};
use crate::sdk::metrics::export::metrics::Aggregator;
use std::mem;
use std::sync::{Arc, RwLock};

/// Create a new histogram for the given descriptor with the given boundaries
pub fn histogram(desc: &Descriptor, boundaries: &[f64]) -> HistogramAggregator {
    let mut sorted_boundaries = boundaries.to_owned();
    sorted_boundaries.sort_by(|a, b| a.partial_cmp(&b).unwrap());
    let state = State::empty(&sorted_boundaries);

    HistogramAggregator {
        inner: RwLock::new(Inner {
            boundaries: sorted_boundaries,
            kind: desc.number_kind().clone(),
            state,
        }),
    }
}

/// This aggregator observes events and counts them in pre-determined buckets. It
/// also calculates the sum and count of all events.
#[derive(Debug)]
pub struct HistogramAggregator {
    inner: RwLock<Inner>,
}

#[derive(Debug)]
struct Inner {
    boundaries: Vec<f64>,
    kind: NumberKind,
    state: State,
}

#[derive(Debug)]
struct State {
    bucket_counts: Vec<f64>,
    count: AtomicNumber,
    sum: AtomicNumber,
}

impl State {
    fn empty(boundaries: &[f64]) -> Self {
        State {
            bucket_counts: vec![0.0; boundaries.len() + 1],
            count: NumberKind::U64.zero().to_atomic(),
            sum: NumberKind::U64.zero().to_atomic(),
        }
    }
}

impl Sum for HistogramAggregator {
    fn sum(&self) -> Result<Number> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| inner.state.sum.load())
    }
}

impl Count for HistogramAggregator {
    fn count(&self) -> Result<u64> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| inner.state.sum.load().to_u64(&NumberKind::U64))
    }
}

impl Histogram for HistogramAggregator {
    fn histogram(&self) -> Result<Buckets> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| Buckets::new(inner.boundaries.clone(), inner.state.bucket_counts.clone()))
    }
}

impl Aggregator for HistogramAggregator {
    fn update(&self, number: &Number, descriptor: &Descriptor) -> Result<()> {
        self.inner.write().map_err(From::from).map(|mut inner| {
            let kind = descriptor.number_kind();
            let as_float = number.to_f64(kind);

            let mut bucket_id = inner.boundaries.len();
            for (idx, boundary) in inner.boundaries.iter().enumerate() {
                if as_float < *boundary {
                    bucket_id = idx;
                    break;
                }
            }

            inner.state.count.fetch_add(&NumberKind::U64, &1u64.into());
            inner.state.sum.fetch_add(kind, number);
            inner.state.bucket_counts[bucket_id] += 1.0;
        })
    }

    fn synchronized_move(
        &self,
        other: &Arc<dyn Aggregator + Send + Sync>,
        _descriptor: &crate::metrics::Descriptor,
    ) -> Result<()> {
        if let Some(other) = other.as_any().downcast_ref::<Self>() {
            self.inner
                .write()
                .map_err(From::from)
                .and_then(|mut inner| {
                    other.inner.write().map_err(From::from).map(|mut other| {
                        let empty = State::empty(&inner.boundaries);
                        other.state = mem::replace(&mut inner.state, empty)
                    })
                })
        } else {
            Err(MetricsError::InconsistentAggregator(format!(
                "Expected {:?}, got: {:?}",
                self, other
            )))
        }
    }

    fn merge(&self, other: &(dyn Aggregator + Send + Sync), desc: &Descriptor) -> Result<()> {
        if let Some(other) = other.as_any().downcast_ref::<HistogramAggregator>() {
            self.inner
                .write()
                .map_err(From::from)
                .and_then(|mut inner| {
                    other.inner.read().map_err(From::from).map(|other| {
                        inner
                            .state
                            .sum
                            .fetch_add(desc.number_kind(), &other.state.sum.load());
                        inner
                            .state
                            .count
                            .fetch_add(&NumberKind::U64, &other.state.count.load());

                        for idx in 0..inner.state.bucket_counts.len() {
                            inner.state.bucket_counts[idx] += other.state.bucket_counts[idx];
                        }
                    })
                })
        } else {
            Err(MetricsError::InconsistentAggregator(format!(
                "Expected {:?}, got: {:?}",
                self, other
            )))
        }
    }

    fn as_any(&self) -> &dyn std::any::Any {
        self
    }
}
<commit_msg>fix: HistogramAggregator returns sum when asked for count. (#398)<commit_after>use crate::metrics::{AtomicNumber, Descriptor, MetricsError, Number, NumberKind, Result};
use crate::sdk::export::metrics::{Buckets, Count, Histogram, Sum};
use crate::sdk::metrics::export::metrics::Aggregator;
use std::mem;
use std::sync::{Arc, RwLock};

/// Create a new histogram for the given descriptor with the given boundaries
pub fn histogram(desc: &Descriptor, boundaries: &[f64]) -> HistogramAggregator {
    let mut sorted_boundaries = boundaries.to_owned();
    sorted_boundaries.sort_by(|a, b| a.partial_cmp(&b).unwrap());
    let state = State::empty(&sorted_boundaries);

    HistogramAggregator {
        inner: RwLock::new(Inner {
            boundaries: sorted_boundaries,
            kind: desc.number_kind().clone(),
            state,
        }),
    }
}

/// This aggregator observes events and counts them in pre-determined buckets. It
/// also calculates the sum and count of all events.
#[derive(Debug)]
pub struct HistogramAggregator {
    inner: RwLock<Inner>,
}

#[derive(Debug)]
struct Inner {
    boundaries: Vec<f64>,
    kind: NumberKind,
    state: State,
}

#[derive(Debug)]
struct State {
    bucket_counts: Vec<f64>,
    count: AtomicNumber,
    sum: AtomicNumber,
}

impl State {
    fn empty(boundaries: &[f64]) -> Self {
        State {
            bucket_counts: vec![0.0; boundaries.len() + 1],
            count: NumberKind::U64.zero().to_atomic(),
            sum: NumberKind::U64.zero().to_atomic(),
        }
    }
}

impl Sum for HistogramAggregator {
    fn sum(&self) -> Result<Number> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| inner.state.sum.load())
    }
}

impl Count for HistogramAggregator {
    fn count(&self) -> Result<u64> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| inner.state.count.load().to_u64(&NumberKind::U64))
    }
}

impl Histogram for HistogramAggregator {
    fn histogram(&self) -> Result<Buckets> {
        self.inner
            .read()
            .map_err(From::from)
            .map(|inner| Buckets::new(inner.boundaries.clone(), inner.state.bucket_counts.clone()))
    }
}

impl Aggregator for HistogramAggregator {
    fn update(&self, number: &Number, descriptor: &Descriptor) -> Result<()> {
        self.inner.write().map_err(From::from).map(|mut inner| {
            let kind = descriptor.number_kind();
            let as_float = number.to_f64(kind);

            let mut bucket_id = inner.boundaries.len();
            for (idx, boundary) in inner.boundaries.iter().enumerate() {
                if as_float < *boundary {
                    bucket_id = idx;
                    break;
                }
            }

            inner.state.count.fetch_add(&NumberKind::U64, &1u64.into());
            inner.state.sum.fetch_add(kind, number);
            inner.state.bucket_counts[bucket_id] += 1.0;
        })
    }

    fn synchronized_move(
        &self,
        other: &Arc<dyn Aggregator + Send + Sync>,
        _descriptor: &crate::metrics::Descriptor,
    ) -> Result<()> {
        if let Some(other) = other.as_any().downcast_ref::<Self>() {
            self.inner
                .write()
                .map_err(From::from)
                .and_then(|mut inner| {
                    other.inner.write().map_err(From::from).map(|mut other| {
                        let empty = State::empty(&inner.boundaries);
                        other.state = mem::replace(&mut inner.state, empty)
                    })
                })
        } else {
            Err(MetricsError::InconsistentAggregator(format!(
                "Expected {:?}, got: {:?}",
                self, other
            )))
        }
    }

    fn merge(&self, other: &(dyn Aggregator + Send + Sync), desc: &Descriptor) -> Result<()> {
        if let Some(other) = other.as_any().downcast_ref::<HistogramAggregator>() {
            self.inner
                .write()
                .map_err(From::from)
                .and_then(|mut inner| {
                    other.inner.read().map_err(From::from).map(|other| {
                        inner
                            .state
                            .sum
                            .fetch_add(desc.number_kind(), &other.state.sum.load());
                        inner
                            .state
                            .count
                            .fetch_add(&NumberKind::U64, &other.state.count.load());

                        for idx in 0..inner.state.bucket_counts.len() {
                            inner.state.bucket_counts[idx] += other.state.bucket_counts[idx];
                        }
                    })
                })
        } else {
            Err(MetricsError::InconsistentAggregator(format!(
                "Expected {:?}, got: {:?}",
                self, other
            )))
        }
    }

    fn as_any(&self) -> &dyn std::any::Any {
        self
    }
}
<|endoftext|>


<commit_before>use luminance::{Dim2, Flat, Framebuffer, Mode, Pipe, Pipeline, RGBA32F, RawTexture, RenderCommand,
                ShadingCommand, Tess, TessRender, Texture, Unit, Uniform};

use resource::Res;
use scene::Scene;
use shader::Program;

pub type Texture2D<A> = Texture<Flat, Dim2, A>;

const FORWARD_SOURCE: &'static Uniform<Unit> = &Uniform::new(0);

pub struct Forward {
  program: Res<Program>,
  quad: Tess,
  w: u32,
  h: u32
}

impl Forward {
  pub fn new(w: u32, h: u32, scene: &mut Scene) -> Self {
    let program = scene.get("spectra/compositors/forward.glsl", vec![FORWARD_SOURCE.sem("source")]).unwrap();

    Forward {
      program: program,
      quad: Tess::attributeless(Mode::TriangleStrip, 4),
      w: w,
      h: h
    }
  }

  pub fn to_screen(&self, source: &Texture2D<RGBA32F>) {
    let back_fb = Framebuffer::default((self.w, self.h));
    let textures: &[&RawTexture] = &[source];
    let tess_render = TessRender::one_whole(&self.quad);

    Pipeline::new(&back_fb, [0., 0., 0., 0.], textures, &[], vec![
      Pipe::empty()
        .uniforms(&[FORWARD_SOURCE.alter(Unit::new(0))])
        .unwrap(ShadingCommand::new(&self.program.borrow(), vec![
          Pipe::new(RenderCommand::new(None, true, vec![
            Pipe::new(tess_render)]))
          ]))
    ]).run();
  }

  pub fn black_screen(&self) {
    let back_fb = Framebuffer::default((self.w, self.h));
    Pipeline::new(&back_fb, [0., 0., 0., 0.], &[], &[], vec![]).run();
  }
}
<commit_msg>Updated the forward compositor to the new luminance’s pipeline system.<commit_after>use luminance::{Dim2, Flat, Framebuffer, Mode, Pipe, Pipeline, RGBA32F, RawTexture, RenderCommand,
                ShadingCommand, Tess, TessRender, Texture, Unit, Uniform};

use resource::Res;
use scene::Scene;
use shader::Program;

pub type Texture2D<A> = Texture<Flat, Dim2, A>;

const FORWARD_SOURCE: &'static Uniform<Unit> = &Uniform::new(0);

pub struct Forward {
  program: Res<Program>,
  quad: Tess,
  w: u32,
  h: u32
}

impl Forward {
  pub fn new(w: u32, h: u32, scene: &mut Scene) -> Self {
    let program = scene.get("spectra/compositors/forward.glsl", vec![FORWARD_SOURCE.sem("source")]).unwrap();

    Forward {
      program: program,
      quad: Tess::attributeless(Mode::TriangleStrip, 4),
      w: w,
      h: h
    }
  }

  pub fn to_screen(&self, source: &Texture2D<RGBA32F>) {
    let back_fb = Framebuffer::default((self.w, self.h));
    let textures: &[&RawTexture] = &[source];
    let tess_render = TessRender::one_whole(&self.quad);

    Pipeline::new(&back_fb, [0., 0., 0., 0.], textures, &[], &[
      Pipe::empty()
        .uniforms(&[FORWARD_SOURCE.alter(Unit::new(0))])
        .unwrap(ShadingCommand::new(&self.program.borrow(), &[
          Pipe::new(RenderCommand::new(None, true, &[
            Pipe::new(tess_render)]))
          ]))
    ]).run();
  }

  pub fn black_screen(&self) {
    let back_fb = Framebuffer::default((self.w, self.h));
    Pipeline::new(&back_fb, [0., 0., 0., 0.], &[], &[], &[]).run();
  }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::collections::HashSet;
use std::collections::hash_map::RandomState;
use std::iter::FromIterator;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;

use devicemapper::consts::SECTOR_SIZE;
use devicemapper::DM;
use devicemapper::{DataBlocks, Sectors};
use devicemapper::LinearDev;
use devicemapper::{ThinPoolDev, ThinPoolStatus, ThinPoolWorkingStatus};
use time::now;
use uuid::Uuid;
use serde_json;

use engine::EngineError;
use engine::EngineResult;
use engine::ErrorEnum;
use engine::Filesystem;
use engine::Pool;
use engine::RenameAction;
use engine::engine::Redundancy;
use engine::strat_engine::blockdev::wipe_sectors;
use consts::IEC::Mi;

use super::super::engine::{FilesystemUuid, HasName, HasUuid};
use super::super::structures::Table;

use super::serde_structs::StratSave;
use super::blockdev::{initialize, resolve_devices};
use super::blockdevmgr::BlockDevMgr;
use super::filesystem::{StratFilesystem, FilesystemStatus};
use super::metadata::MIN_MDA_SECTORS;

pub const DATA_BLOCK_SIZE: Sectors = Sectors(2048);
pub const META_LOWATER: u64 = 512;
pub const DATA_LOWATER: DataBlocks = DataBlocks(512);

pub const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);
pub const INITIAL_DATA_SIZE: Sectors = Sectors(512 * Mi / SECTOR_SIZE as u64);

#[derive(Debug)]
pub struct StratPool {
    name: String,
    pool_uuid: Uuid,
    pub block_devs: BlockDevMgr,
    pub filesystems: Table<StratFilesystem>,
    redundancy: Redundancy,
    thin_pool: ThinPoolDev,
}

impl StratPool {
    /// Initialize a Stratis Pool.
    /// 1. Initialize the block devices specified by paths.
    /// 2. Set up thinpool device to back filesystems.
    pub fn initialize(name: &str,
                      dm: &DM,
                      paths: &[&Path],
                      redundancy: Redundancy,
                      force: bool)
                      -> EngineResult<StratPool> {
        let pool_uuid = Uuid::new_v4();

        let devices = try!(resolve_devices(paths));
        let mut block_mgr =
            BlockDevMgr::new(try!(initialize(&pool_uuid, devices, MIN_MDA_SECTORS, force)));

        if block_mgr.avail_space() < INITIAL_META_SIZE + INITIAL_DATA_SIZE {
            let short_bytes = *(INITIAL_META_SIZE + INITIAL_DATA_SIZE - block_mgr.avail_space()) *
                              SECTOR_SIZE as u64;
            try!(block_mgr.destroy_all());
            return Err(EngineError::Engine(ErrorEnum::Invalid,
                                           format!("Initial total usable pool capacity too \
                                                    small by {} bytes",
                                                   short_bytes)));
        }

        let meta_regions = block_mgr
            .alloc_space(INITIAL_META_SIZE)
            .expect("blockmgr must not fail, already checked for space");
        let meta_dev = try!(LinearDev::new(&format!("stratis_{}_meta", name), dm, &meta_regions));

        // When constructing a thin-pool, Stratis reserves the first N
        // sectors on a block device by creating a linear device with a
        // starting offset. DM writes the super block in the first block.
        // DM requires this first block to be zeros when the meta data for
        // the thin-pool is initially created. If we don't zero the
        // superblock DM issue error messages because it triggers code paths
        // that are trying to re-adopt the device with the attributes that
        // have been passed.
        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));

        let data_regions = block_mgr
            .alloc_space(INITIAL_DATA_SIZE)
            .expect("blockmgr must not fail, already checked for space");
        let data_dev = try!(LinearDev::new(&format!("stratis_{}_data", name), dm, &data_regions));
        let length = try!(data_dev.size()).sectors();

        // TODO Fix hard coded data blocksize and low water mark.
        let thinpool_dev = try!(ThinPoolDev::new(&format!("stratis_{}_thinpool", name),
                                                 dm,
                                                 length,
                                                 DATA_BLOCK_SIZE,
                                                 DataBlocks(256000),
                                                 meta_dev,
                                                 data_dev));

        let mut pool = StratPool {
            name: name.to_owned(),
            pool_uuid: pool_uuid,
            block_devs: block_mgr,
            filesystems: Table::new(),
            redundancy: redundancy,
            thin_pool: thinpool_dev,
        };

        try!(pool.write_metadata());

        Ok(pool)
    }

    /// Return the metadata from the first blockdev with up-to-date, readable
    /// metadata.
    pub fn load_state(&self) -> Option<Vec<u8>> {
        self.block_devs.load_state()
    }

    // TODO: Check current time against global last updated, and use
    // alternate time value if earlier, as described in SWDD
    pub fn write_metadata(&mut self) -> EngineResult<()> {
        let data = try!(serde_json::to_string(&self.to_save()));
        self.block_devs
            .save_state(&now().to_timespec(), data.as_bytes())
    }

    pub fn to_save(&self) -> StratSave {
        StratSave {
            name: self.name.clone(),
            id: self.pool_uuid.simple().to_string(),
            block_devs: self.block_devs.to_save(),
        }
    }

    pub fn check(&mut self) -> () {
        let dm = DM::new().expect("Could not get DM handle");

        let result = match self.thin_pool.status(&dm) {
            Ok(r) => r,
            Err(_) => {
                error!("Could not get thinpool status");
                // TODO: Take pool offline?
                return;
            }
        };

        match result {
            ThinPoolStatus::Good(wstatus, usage) => {
                match wstatus {
                    ThinPoolWorkingStatus::Good => {}
                    ThinPoolWorkingStatus::ReadOnly => {
                        // TODO: why is pool r/o and how do we get it
                        // rw again?
                    }
                    ThinPoolWorkingStatus::OutOfSpace => {
                        // TODO: Add more space if possible, or
                        // prevent further usage
                        // Should never happen -- we should be extending first!
                    }
                    ThinPoolWorkingStatus::NeedsCheck => {
                        // TODO: Take pool offline?
                        // TODO: run thin_check
                    }
                }

                if usage.used_meta > usage.total_meta - META_LOWATER {
                    // TODO: Extend meta device
                }

                if usage.used_data > usage.total_data - DATA_LOWATER {
                    // TODO: Extend data device
                }
            }
            ThinPoolStatus::Fail => {
                // TODO: Take pool offline?
                // TODO: Run thin_check
            }
        }

        for fs in self.filesystems.iter_mut() {
            match fs.check(&dm) {
                Ok(f_status) => {
                    if let FilesystemStatus::Failed = f_status {
                        // TODO: recover fs? (how?)
                    }
                }
                Err(_e) => error!("fs.check() failed"),
            }
        }
    }

    /// Teardown a pool.
    /// Take down the device mapper devices belonging to the pool.
    /// This method and destroy() must keep their DM teardown operations
    /// in sync.
    pub fn teardown(self) -> EngineResult<()> {
        // TODO: any necessary clean up of filesystems
        if !self.filesystems.is_empty() {
            return Err(EngineError::Engine(ErrorEnum::Busy,
                                           format!("May be unsynced files on device.")));
        }
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));
        Ok(())
    }
}

impl Pool for StratPool {
    fn create_filesystems<'a, 'b>(&'a mut self,
                                  specs: &[&'b str])
                                  -> EngineResult<Vec<(&'b str, FilesystemUuid)>> {
        let names: HashSet<_, RandomState> = HashSet::from_iter(specs);
        for name in names.iter() {
            if self.filesystems.contains_name(name) {
                return Err(EngineError::Engine(ErrorEnum::AlreadyExists, name.to_string()));
            }
        }

        let mut result = Vec::new();
        for name in names.iter() {
            let uuid = Uuid::new_v4();
            let dm = try!(DM::new());
            let new_filesystem = try!(StratFilesystem::new(uuid, name, &dm, &mut self.thin_pool));
            self.filesystems.insert(new_filesystem);
            result.push((**name, uuid));
        }

        Ok(result)
    }

    fn add_blockdevs(&mut self, paths: &[&Path], force: bool) -> EngineResult<Vec<PathBuf>> {
        let bdev_paths = try!(self.block_devs.add(&self.pool_uuid, paths, force));
        try!(self.write_metadata());
        Ok(bdev_paths)
    }

    fn destroy(self) -> EngineResult<()> {
        // Ensure that DM teardown operations in this method are in sync
        // with operations in teardown().
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));
        try!(self.block_devs.destroy_all());

        Ok(())
    }

    fn destroy_filesystems<'a, 'b>(&'a mut self,
                                   fs_uuids: &[&'b FilesystemUuid])
                                   -> EngineResult<Vec<&'b FilesystemUuid>> {
        destroy_filesystems!{self; fs_uuids}
    }

    fn rename_filesystem(&mut self,
                         uuid: &FilesystemUuid,
                         new_name: &str)
                         -> EngineResult<RenameAction> {
        rename_filesystem!{self; uuid; new_name}
    }

    fn rename(&mut self, name: &str) {
        self.name = name.to_owned();
    }

    fn get_filesystem(&mut self, uuid: &FilesystemUuid) -> Option<&mut Filesystem> {
        get_filesystem!(self; uuid)
    }
}

impl HasUuid for StratPool {
    fn uuid(&self) -> &FilesystemUuid {
        &self.pool_uuid
    }
}

impl HasName for StratPool {
    fn name(&self) -> &str {
        &self.name
    }
}
<commit_msg>Make StratPool constants private<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::collections::HashSet;
use std::collections::hash_map::RandomState;
use std::iter::FromIterator;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;

use devicemapper::consts::SECTOR_SIZE;
use devicemapper::DM;
use devicemapper::{DataBlocks, Sectors};
use devicemapper::LinearDev;
use devicemapper::{ThinPoolDev, ThinPoolStatus, ThinPoolWorkingStatus};
use time::now;
use uuid::Uuid;
use serde_json;

use engine::EngineError;
use engine::EngineResult;
use engine::ErrorEnum;
use engine::Filesystem;
use engine::Pool;
use engine::RenameAction;
use engine::engine::Redundancy;
use engine::strat_engine::blockdev::wipe_sectors;
use consts::IEC::Mi;

use super::super::engine::{FilesystemUuid, HasName, HasUuid};
use super::super::structures::Table;

use super::serde_structs::StratSave;
use super::blockdev::{initialize, resolve_devices};
use super::blockdevmgr::BlockDevMgr;
use super::filesystem::{StratFilesystem, FilesystemStatus};
use super::metadata::MIN_MDA_SECTORS;

const DATA_BLOCK_SIZE: Sectors = Sectors(2048);
const META_LOWATER: u64 = 512;
const DATA_LOWATER: DataBlocks = DataBlocks(512);

const INITIAL_META_SIZE: Sectors = Sectors(16 * Mi / SECTOR_SIZE as u64);
const INITIAL_DATA_SIZE: Sectors = Sectors(512 * Mi / SECTOR_SIZE as u64);

#[derive(Debug)]
pub struct StratPool {
    name: String,
    pool_uuid: Uuid,
    pub block_devs: BlockDevMgr,
    pub filesystems: Table<StratFilesystem>,
    redundancy: Redundancy,
    thin_pool: ThinPoolDev,
}

impl StratPool {
    /// Initialize a Stratis Pool.
    /// 1. Initialize the block devices specified by paths.
    /// 2. Set up thinpool device to back filesystems.
    pub fn initialize(name: &str,
                      dm: &DM,
                      paths: &[&Path],
                      redundancy: Redundancy,
                      force: bool)
                      -> EngineResult<StratPool> {
        let pool_uuid = Uuid::new_v4();

        let devices = try!(resolve_devices(paths));
        let mut block_mgr =
            BlockDevMgr::new(try!(initialize(&pool_uuid, devices, MIN_MDA_SECTORS, force)));

        if block_mgr.avail_space() < INITIAL_META_SIZE + INITIAL_DATA_SIZE {
            let short_bytes = *(INITIAL_META_SIZE + INITIAL_DATA_SIZE - block_mgr.avail_space()) *
                              SECTOR_SIZE as u64;
            try!(block_mgr.destroy_all());
            return Err(EngineError::Engine(ErrorEnum::Invalid,
                                           format!("Initial total usable pool capacity too \
                                                    small by {} bytes",
                                                   short_bytes)));
        }

        let meta_regions = block_mgr
            .alloc_space(INITIAL_META_SIZE)
            .expect("blockmgr must not fail, already checked for space");
        let meta_dev = try!(LinearDev::new(&format!("stratis_{}_meta", name), dm, &meta_regions));

        // When constructing a thin-pool, Stratis reserves the first N
        // sectors on a block device by creating a linear device with a
        // starting offset. DM writes the super block in the first block.
        // DM requires this first block to be zeros when the meta data for
        // the thin-pool is initially created. If we don't zero the
        // superblock DM issue error messages because it triggers code paths
        // that are trying to re-adopt the device with the attributes that
        // have been passed.
        try!(wipe_sectors(&try!(meta_dev.devnode()), Sectors(0), INITIAL_META_SIZE));

        let data_regions = block_mgr
            .alloc_space(INITIAL_DATA_SIZE)
            .expect("blockmgr must not fail, already checked for space");
        let data_dev = try!(LinearDev::new(&format!("stratis_{}_data", name), dm, &data_regions));
        let length = try!(data_dev.size()).sectors();

        // TODO Fix hard coded data blocksize and low water mark.
        let thinpool_dev = try!(ThinPoolDev::new(&format!("stratis_{}_thinpool", name),
                                                 dm,
                                                 length,
                                                 DATA_BLOCK_SIZE,
                                                 DataBlocks(256000),
                                                 meta_dev,
                                                 data_dev));

        let mut pool = StratPool {
            name: name.to_owned(),
            pool_uuid: pool_uuid,
            block_devs: block_mgr,
            filesystems: Table::new(),
            redundancy: redundancy,
            thin_pool: thinpool_dev,
        };

        try!(pool.write_metadata());

        Ok(pool)
    }

    /// Return the metadata from the first blockdev with up-to-date, readable
    /// metadata.
    pub fn load_state(&self) -> Option<Vec<u8>> {
        self.block_devs.load_state()
    }

    // TODO: Check current time against global last updated, and use
    // alternate time value if earlier, as described in SWDD
    pub fn write_metadata(&mut self) -> EngineResult<()> {
        let data = try!(serde_json::to_string(&self.to_save()));
        self.block_devs
            .save_state(&now().to_timespec(), data.as_bytes())
    }

    pub fn to_save(&self) -> StratSave {
        StratSave {
            name: self.name.clone(),
            id: self.pool_uuid.simple().to_string(),
            block_devs: self.block_devs.to_save(),
        }
    }

    pub fn check(&mut self) -> () {
        let dm = DM::new().expect("Could not get DM handle");

        let result = match self.thin_pool.status(&dm) {
            Ok(r) => r,
            Err(_) => {
                error!("Could not get thinpool status");
                // TODO: Take pool offline?
                return;
            }
        };

        match result {
            ThinPoolStatus::Good(wstatus, usage) => {
                match wstatus {
                    ThinPoolWorkingStatus::Good => {}
                    ThinPoolWorkingStatus::ReadOnly => {
                        // TODO: why is pool r/o and how do we get it
                        // rw again?
                    }
                    ThinPoolWorkingStatus::OutOfSpace => {
                        // TODO: Add more space if possible, or
                        // prevent further usage
                        // Should never happen -- we should be extending first!
                    }
                    ThinPoolWorkingStatus::NeedsCheck => {
                        // TODO: Take pool offline?
                        // TODO: run thin_check
                    }
                }

                if usage.used_meta > usage.total_meta - META_LOWATER {
                    // TODO: Extend meta device
                }

                if usage.used_data > usage.total_data - DATA_LOWATER {
                    // TODO: Extend data device
                }
            }
            ThinPoolStatus::Fail => {
                // TODO: Take pool offline?
                // TODO: Run thin_check
            }
        }

        for fs in self.filesystems.iter_mut() {
            match fs.check(&dm) {
                Ok(f_status) => {
                    if let FilesystemStatus::Failed = f_status {
                        // TODO: recover fs? (how?)
                    }
                }
                Err(_e) => error!("fs.check() failed"),
            }
        }
    }

    /// Teardown a pool.
    /// Take down the device mapper devices belonging to the pool.
    /// This method and destroy() must keep their DM teardown operations
    /// in sync.
    pub fn teardown(self) -> EngineResult<()> {
        // TODO: any necessary clean up of filesystems
        if !self.filesystems.is_empty() {
            return Err(EngineError::Engine(ErrorEnum::Busy,
                                           format!("May be unsynced files on device.")));
        }
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));
        Ok(())
    }
}

impl Pool for StratPool {
    fn create_filesystems<'a, 'b>(&'a mut self,
                                  specs: &[&'b str])
                                  -> EngineResult<Vec<(&'b str, FilesystemUuid)>> {
        let names: HashSet<_, RandomState> = HashSet::from_iter(specs);
        for name in names.iter() {
            if self.filesystems.contains_name(name) {
                return Err(EngineError::Engine(ErrorEnum::AlreadyExists, name.to_string()));
            }
        }

        let mut result = Vec::new();
        for name in names.iter() {
            let uuid = Uuid::new_v4();
            let dm = try!(DM::new());
            let new_filesystem = try!(StratFilesystem::new(uuid, name, &dm, &mut self.thin_pool));
            self.filesystems.insert(new_filesystem);
            result.push((**name, uuid));
        }

        Ok(result)
    }

    fn add_blockdevs(&mut self, paths: &[&Path], force: bool) -> EngineResult<Vec<PathBuf>> {
        let bdev_paths = try!(self.block_devs.add(&self.pool_uuid, paths, force));
        try!(self.write_metadata());
        Ok(bdev_paths)
    }

    fn destroy(self) -> EngineResult<()> {
        // Ensure that DM teardown operations in this method are in sync
        // with operations in teardown().
        let dm = try!(DM::new());
        try!(self.thin_pool.teardown(&dm));
        try!(self.block_devs.destroy_all());

        Ok(())
    }

    fn destroy_filesystems<'a, 'b>(&'a mut self,
                                   fs_uuids: &[&'b FilesystemUuid])
                                   -> EngineResult<Vec<&'b FilesystemUuid>> {
        destroy_filesystems!{self; fs_uuids}
    }

    fn rename_filesystem(&mut self,
                         uuid: &FilesystemUuid,
                         new_name: &str)
                         -> EngineResult<RenameAction> {
        rename_filesystem!{self; uuid; new_name}
    }

    fn rename(&mut self, name: &str) {
        self.name = name.to_owned();
    }

    fn get_filesystem(&mut self, uuid: &FilesystemUuid) -> Option<&mut Filesystem> {
        get_filesystem!(self; uuid)
    }
}

impl HasUuid for StratPool {
    fn uuid(&self) -> &FilesystemUuid {
        &self.pool_uuid
    }
}

impl HasName for StratPool {
    fn name(&self) -> &str {
        &self.name
    }
}
<|endoftext|>


<commit_before>use super::placement::sm::client::SMClient;
use bifrost::rpc::*;
use bifrost_plugins::hash_ident;
use client::AsyncClient;
use dovahkiin::types::custom_types::id::Id;
use index::btree::storage::store_changed_nodes;
use index::lsmtree::service::inner::LSMTreeIns;
use index::lsmtree::split::check_and_split;
use index::lsmtree::tree::{LSMTree, LSMTreeResult};
use index::{EntryKey, Ordering};
use itertools::Itertools;
use parking_lot::RwLock;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use server::NebServer;
use smallvec::SmallVec;
use std::collections::HashMap;
use std::path::Component::CurDir;
use std::sync::atomic;
use std::sync::atomic::AtomicU64;
use std::thread;

mod inner;
#[cfg(test)]
mod test;

pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(LSM_TREE_RPC_SERVICE) as u64;

#[derive(Serialize, Deserialize, Debug)]
pub enum LSMTreeSvrError {
    TreeNotFound,
    SMError,
}

#[derive(Serialize, Deserialize)]
pub struct LSMTreeSummary {
    id: Id,
    count: u64,
    epoch: u64,
    range: (Vec<u8>, Vec<u8>),
}

service! {
    rpc seek(tree_id: Id, key: Vec<u8>, ordering: Ordering, epoch: u64) -> LSMTreeResult<u64> | LSMTreeSvrError;
    rpc next(tree_id: Id, cursor_id: u64) -> Option<bool> | LSMTreeSvrError;
    rpc current(tree_id: Id, cursor_id: u64) -> Option<Option<Vec<u8>>> | LSMTreeSvrError;
    rpc complete(tree_id: Id, cursor_id: u64) -> bool | LSMTreeSvrError;
    rpc new_tree(start: Vec<u8>, end: Vec<u8>, id: Id) -> bool;
    rpc summary() -> Vec<LSMTreeSummary>;

    rpc insert(tree_id: Id, key: Vec<u8>, epoch: u64) -> LSMTreeResult<bool> | LSMTreeSvrError;
    rpc merge(tree_id: Id, keys: Vec<Vec<u8>>, epoch: u64) -> LSMTreeResult<()> | LSMTreeSvrError;
    rpc set_epoch(tree_id: Id, epoch: u64) -> u64 | LSMTreeSvrError;
}

pub struct LSMTreeService {
    neb_server: Arc<NebServer>,
    sm: Arc<SMClient>,
    counter: AtomicU64,
    trees: Arc<RwLock<HashMap<Id, Arc<LSMTreeIns>>>>,
}

dispatch_rpc_service_functions!(LSMTreeService);

impl Service for LSMTreeService {
    fn seek(
        &self,
        tree_id: Id,
        key: Vec<u8>,
        ordering: Ordering,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<u64>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| {
                    tree.with_epoch_check(epoch, || tree.seek(&SmallVec::from(key), ordering))
                }),
        )
    }

    fn next(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = Option<bool>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.next(&cursor_id)),
        )
    }

    fn current(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = Option<Option<Vec<u8>>>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.current(&cursor_id)),
        )
    }

    fn complete(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = bool, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.complete(&cursor_id)),
        )
    }

    fn new_tree(
        &self,
        start: Vec<u8>,
        end: Vec<u8>,
        id: Id,
    ) -> Box<Future<Item = bool, Error = ()>> {
        let mut trees = self.trees.write();
        let succeed = if trees.contains_key(&id) {
            false
        } else {
            trees.insert(
                id,
                Arc::new(LSMTreeIns::new(
                    (EntryKey::from(start), EntryKey::from(end)),
                    id,
                )),
            );
            true
        };
        box future::ok(succeed)
    }

    fn summary(&self) -> Box<Future<Item = Vec<LSMTreeSummary>, Error = ()>> {
        let trees = self.trees.read();
        box future::ok(
            trees
                .iter()
                .map(|(id, tree)| LSMTreeSummary {
                    id: *id,
                    count: tree.count(),
                    epoch: tree.epoch(),
                    range: tree.range(),
                })
                .sorted_by_key(|tree| tree.range.0.clone())
                .collect(),
        )
    }

    fn insert(
        &self,
        tree_id: Id,
        key: Vec<u8>,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<bool>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let neb = self.neb_server.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.with_epoch_check(epoch, || tree.insert(SmallVec::from(key)))),
        )
        .and_then(|o| persist(neb, o))
    }

    fn merge(
        &self,
        tree_id: Id,
        keys: Vec<Vec<u8>>,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<()>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let neb = self.neb_server.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| {
                    tree.with_epoch_check(epoch, || {
                        tree.merge(box keys.into_iter().map(|key| SmallVec::from(key)).collect())
                    })
                }),
        )
        .and_then(|o| persist(neb, o))
    }

    fn set_epoch(
        &self,
        tree_id: Id,
        epoch: u64,
    ) -> Box<Future<Item = u64, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let sm = self.sm.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.set_epoch(epoch)),
        )
        .and_then(move |_| {
            sm.update_epoch(&tree_id, &epoch)
                .map_err(|_| LSMTreeSvrError::SMError)
                .and_then(|r| r.map_err(|_| LSMTreeSvrError::SMError))
        })
    }
}

impl LSMTreeService {
    pub fn new(neb_server: &Arc<NebServer>, sm: &Arc<SMClient>) -> Arc<Self> {
        Arc::new(Self {
            neb_server: neb_server.clone(),
            counter: AtomicU64::new(0),
            trees: Arc::new(RwLock::new(HashMap::new())),
            sm: sm.clone(),
        })
    }
}

fn persist<T, E>(neb: Arc<NebServer>, val: T) -> impl Future<Item = T, Error = E> {
    store_changed_nodes(neb).then(move |_| future::ok(val))
}
<commit_msg>persist on new<commit_after>use super::placement::sm::client::SMClient;
use bifrost::rpc::*;
use bifrost_plugins::hash_ident;
use client::AsyncClient;
use dovahkiin::types::custom_types::id::Id;
use index::btree::storage::store_changed_nodes;
use index::lsmtree::service::inner::LSMTreeIns;
use index::lsmtree::split::check_and_split;
use index::lsmtree::tree::{LSMTree, LSMTreeResult};
use index::{EntryKey, Ordering};
use itertools::Itertools;
use parking_lot::RwLock;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use server::NebServer;
use smallvec::SmallVec;
use std::collections::HashMap;
use std::path::Component::CurDir;
use std::sync::atomic;
use std::sync::atomic::AtomicU64;
use std::thread;

mod inner;
#[cfg(test)]
mod test;

pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(LSM_TREE_RPC_SERVICE) as u64;

#[derive(Serialize, Deserialize, Debug)]
pub enum LSMTreeSvrError {
    TreeNotFound,
    SMError,
}

#[derive(Serialize, Deserialize)]
pub struct LSMTreeSummary {
    id: Id,
    count: u64,
    epoch: u64,
    range: (Vec<u8>, Vec<u8>),
}

service! {
    rpc seek(tree_id: Id, key: Vec<u8>, ordering: Ordering, epoch: u64) -> LSMTreeResult<u64> | LSMTreeSvrError;
    rpc next(tree_id: Id, cursor_id: u64) -> Option<bool> | LSMTreeSvrError;
    rpc current(tree_id: Id, cursor_id: u64) -> Option<Option<Vec<u8>>> | LSMTreeSvrError;
    rpc complete(tree_id: Id, cursor_id: u64) -> bool | LSMTreeSvrError;
    rpc new_tree(start: Vec<u8>, end: Vec<u8>, id: Id) -> bool;
    rpc summary() -> Vec<LSMTreeSummary>;

    rpc insert(tree_id: Id, key: Vec<u8>, epoch: u64) -> LSMTreeResult<bool> | LSMTreeSvrError;
    rpc merge(tree_id: Id, keys: Vec<Vec<u8>>, epoch: u64) -> LSMTreeResult<()> | LSMTreeSvrError;
    rpc set_epoch(tree_id: Id, epoch: u64) -> u64 | LSMTreeSvrError;
}

pub struct LSMTreeService {
    neb_server: Arc<NebServer>,
    sm: Arc<SMClient>,
    counter: AtomicU64,
    trees: Arc<RwLock<HashMap<Id, Arc<LSMTreeIns>>>>,
}

dispatch_rpc_service_functions!(LSMTreeService);

impl Service for LSMTreeService {
    fn seek(
        &self,
        tree_id: Id,
        key: Vec<u8>,
        ordering: Ordering,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<u64>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| {
                    tree.with_epoch_check(epoch, || tree.seek(&SmallVec::from(key), ordering))
                }),
        )
    }

    fn next(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = Option<bool>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.next(&cursor_id)),
        )
    }

    fn current(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = Option<Option<Vec<u8>>>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.current(&cursor_id)),
        )
    }

    fn complete(
        &self,
        tree_id: Id,
        cursor_id: u64,
    ) -> Box<Future<Item = bool, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.complete(&cursor_id)),
        )
    }

    fn new_tree(
        &self,
        start: Vec<u8>,
        end: Vec<u8>,
        id: Id,
    ) -> Box<Future<Item = bool, Error = ()>> {
        let mut trees = self.trees.write();
        let succeed = if trees.contains_key(&id) {
            false
        } else {
            trees.insert(
                id,
                Arc::new(LSMTreeIns::new(
                    (EntryKey::from(start), EntryKey::from(end)),
                    id,
                )),
            );
            true
        };
        box persist(self.neb_server.clone(), succeed)
    }

    fn summary(&self) -> Box<Future<Item = Vec<LSMTreeSummary>, Error = ()>> {
        let trees = self.trees.read();
        box future::ok(
            trees
                .iter()
                .map(|(id, tree)| LSMTreeSummary {
                    id: *id,
                    count: tree.count(),
                    epoch: tree.epoch(),
                    range: tree.range(),
                })
                .sorted_by_key(|tree| tree.range.0.clone())
                .collect(),
        )
    }

    fn insert(
        &self,
        tree_id: Id,
        key: Vec<u8>,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<bool>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let neb = self.neb_server.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.with_epoch_check(epoch, || tree.insert(SmallVec::from(key)))),
        )
        .and_then(|o| persist(neb, o))
    }

    fn merge(
        &self,
        tree_id: Id,
        keys: Vec<Vec<u8>>,
        epoch: u64,
    ) -> Box<Future<Item = LSMTreeResult<()>, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let neb = self.neb_server.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| {
                    tree.with_epoch_check(epoch, || {
                        tree.merge(box keys.into_iter().map(|key| SmallVec::from(key)).collect())
                    })
                }),
        )
        .and_then(|o| persist(neb, o))
    }

    fn set_epoch(
        &self,
        tree_id: Id,
        epoch: u64,
    ) -> Box<Future<Item = u64, Error = LSMTreeSvrError>> {
        let trees = self.trees.read();
        let sm = self.sm.clone();
        box future::result(
            trees
                .get(&tree_id)
                .ok_or(LSMTreeSvrError::TreeNotFound)
                .map(|tree| tree.set_epoch(epoch)),
        )
        .and_then(move |_| {
            sm.update_epoch(&tree_id, &epoch)
                .map_err(|_| LSMTreeSvrError::SMError)
                .and_then(|r| r.map_err(|_| LSMTreeSvrError::SMError))
        })
    }
}

impl LSMTreeService {
    pub fn new(neb_server: &Arc<NebServer>, sm: &Arc<SMClient>) -> Arc<Self> {
        Arc::new(Self {
            neb_server: neb_server.clone(),
            counter: AtomicU64::new(0),
            trees: Arc::new(RwLock::new(HashMap::new())),
            sm: sm.clone(),
        })
    }
}

fn persist<T, E>(neb: Arc<NebServer>, val: T) -> impl Future<Item = T, Error = E> {
    store_changed_nodes(neb).then(move |_| future::ok(val))
}
<|endoftext|>


<commit_before>use std::io::Write;
use std::ops::{Deref, DerefMut};

use TerminalError;

/// A terminal restorer, which keeps the previous state of the terminal, and restores it, when
/// dropped.
#[cfg(target_os = "redox")]
pub struct TerminalRestorer<W> {
    output: W,
}

#[cfg(target_os = "redox")]
impl<W: Write> Drop for TerminalRestorer<W> {
    fn drop(&mut self) {
        use TermControl;
        self.csi(b"R");
    }
}

#[cfg(not(target_os = "redox"))]
use termios::Termios;
/// A terminal restorer, which keeps the previous state of the terminal, and restores it, when
/// dropped.
#[cfg(not(target_os = "redox"))]
pub struct TerminalRestorer<W> {
    prev_ios: Termios,
    output: W,
}

#[cfg(not(target_os = "redox"))]
impl<W> Drop for TerminalRestorer<W> {
    fn drop(&mut self) {
        use termios::set_terminal_attr;
        set_terminal_attr(&mut self.prev_ios as *mut _);
    }
}

impl<W> Deref for TerminalRestorer<W> {
    type Target = W;

    fn deref(&self) -> &W {
        &self.output
    }
}
impl<W> DerefMut for TerminalRestorer<W> {
    fn deref_mut(&mut self) -> &mut W {
        &mut self.output
    }
}

pub trait IntoRawMode: Sized {
    /// Switch to raw mode.
    ///
    /// Raw mode means that stdin won't be printed (it will instead have to be written manually by the
    /// program). Furthermore, the input isn't canonicalised or buffered (that is, you can read from
    /// stdin one byte of a time). The output is neither modified in any way.
    fn into_raw_mode(self) -> Result<TerminalRestorer<Self>, TerminalError>;
}

impl<W: Write> IntoRawMode for W {
    #[cfg(not(target_os = "redox"))]
    fn into_raw_mode(self) -> Result<TerminalRestorer<W>, TerminalError> {
        use termios::{cfmakeraw, get_terminal_attr, set_terminal_attr};

        let (mut ios, err) = get_terminal_attr();
        let prev_ios = ios.clone();
        if err != 0 {
            return Err(TerminalError::LoadAttrError);
        }

        unsafe {
            cfmakeraw(&mut ios);
        }

        if set_terminal_attr(&mut ios as *mut _) != 0 {
            Err(TerminalError::SetAttrError)
        } else {
            Ok(TerminalRestorer {
                prev_ios: prev_ios,
                output: self,
            })
        }
    }
    #[cfg(target_os = "redox")]
    fn into_raw_mode(self) -> Result<TerminalRestorer<W>, TerminalError> {
        if let Err(_) = self.csi("r") {
            Err(TerminalError::StdoutError)
        } else {
            Ok(TerminalRestorer {
                output: self,
            })
        }
    }
}
<commit_msg>Fixes for Redox<commit_after>use std::io::Write;
use std::ops::{Deref, DerefMut};

use TerminalError;

/// A terminal restorer, which keeps the previous state of the terminal, and restores it, when
/// dropped.
#[cfg(target_os = "redox")]
pub struct TerminalRestorer<W> {
    output: W,
}

#[cfg(target_os = "redox")]
impl<W: Write> Drop for TerminalRestorer<W> {
    fn drop(&mut self) {
        use TermControl;
        self.csi(b"R");
    }
}

#[cfg(not(target_os = "redox"))]
use termios::Termios;
/// A terminal restorer, which keeps the previous state of the terminal, and restores it, when
/// dropped.
#[cfg(not(target_os = "redox"))]
pub struct TerminalRestorer<W> {
    prev_ios: Termios,
    output: W,
}

#[cfg(not(target_os = "redox"))]
impl<W> Drop for TerminalRestorer<W> {
    fn drop(&mut self) {
        use termios::set_terminal_attr;
        set_terminal_attr(&mut self.prev_ios as *mut _);
    }
}

impl<W> Deref for TerminalRestorer<W> {
    type Target = W;

    fn deref(&self) -> &W {
        &self.output
    }
}
impl<W> DerefMut for TerminalRestorer<W> {
    fn deref_mut(&mut self) -> &mut W {
        &mut self.output
    }
}

pub trait IntoRawMode: Sized {
    /// Switch to raw mode.
    ///
    /// Raw mode means that stdin won't be printed (it will instead have to be written manually by the
    /// program). Furthermore, the input isn't canonicalised or buffered (that is, you can read from
    /// stdin one byte of a time). The output is neither modified in any way.
    fn into_raw_mode(self) -> Result<TerminalRestorer<Self>, TerminalError>;
}

impl<W: Write> IntoRawMode for W {
    #[cfg(not(target_os = "redox"))]
    fn into_raw_mode(self) -> Result<TerminalRestorer<W>, TerminalError> {
        use termios::{cfmakeraw, get_terminal_attr, set_terminal_attr};

        let (mut ios, err) = get_terminal_attr();
        let prev_ios = ios.clone();
        if err != 0 {
            return Err(TerminalError::LoadAttrError);
        }

        unsafe {
            cfmakeraw(&mut ios);
        }

        if set_terminal_attr(&mut ios as *mut _) != 0 {
            Err(TerminalError::SetAttrError)
        } else {
            Ok(TerminalRestorer {
                prev_ios: prev_ios,
                output: self,
            })
        }
    }
    #[cfg(target_os = "redox")]
    fn into_raw_mode(self) -> Result<TerminalRestorer<W>, TerminalError> {
        use TermControl;

        if let Err(_) = self.csi("r") {
            Err(TerminalError::StdoutError)
        } else {
            Ok(TerminalRestorer {
                output: self,
            })
        }
    }
}
<|endoftext|>


<commit_before>use std::fmt;
use std::marker::PhantomData;
use serde::de::{self, Deserializer, DeserializeOwned};

#[derive(Debug, Deserialize, Clone)]
pub struct Instance {
    #[serde(rename = "InstanceId")]
    pub id: String,
    #[serde(rename = "InstanceName")]
    pub name: String,
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "PublicIpAddress")]
    pub public_ip_address: Vec<String>,
}

impl Instance {
    pub fn ip(&self) -> &str {
        &self.public_ip_address[0]
    }
}

#[derive(Debug, Deserialize, Clone)]
pub struct Instances {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "Instances")]
    pub instances: Vec<Instance>,
    #[serde(rename = "TotalCount")]
    pub total: usize,
    #[serde(rename = "PageNumber")]
    pub page: usize,
    #[serde(rename = "PageSize")]
    pub size: usize,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceStatus {
    #[serde(rename = "InstanceId")]
    pub id: String,
    #[serde(rename = "Status")]
    pub status: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceStatuses {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "InstanceStatuses")]
    pub instance_statuses: Vec<InstanceStatus>,
    #[serde(rename = "TotalCount")]
    pub total: usize,
    #[serde(rename = "PageNumber")]
    pub page: usize,
    #[serde(rename = "PageSize")]
    pub size: usize,
}

#[derive(Debug, Deserialize, Clone)]
pub struct Region {
    #[serde(rename = "RegionId")]
    pub id: String,
    #[serde(rename = "LocalName")]
    pub name: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct Regions {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "Regions")]
    pub regions: Vec<Region>,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceCreateResponse {
    #[serde(rename = "InstanceId")]
    pub id: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct MonitorData {
    #[serde(rename = "InstanceId")]
    pub instance_id: String,
    #[serde(rename = "CPU")]
    pub cpu: u8,
    #[serde(rename = "IntranetRX")]
    pub intranet_rx: u32,
    #[serde(rename = "IntranetTX")]
    pub intranet_tx: u32,
    #[serde(rename = "IntranetBandwidth")]
    pub intranet_bandwidth: u32,
    #[serde(rename = "InternetRX")]
    pub internet_rx: u32,
    #[serde(rename = "InternetTX")]
    pub internet_tx: u32,
    #[serde(rename = "InternetBandwidth")]
    pub internet_bandwidth: u32,
    #[serde(rename = "IOPSRead")]
    pub iops_read: u32,
    #[serde(rename = "IOPSWrite")]
    pub iops_write: u32,
    #[serde(rename = "BPSRead")]
    pub bps_read: u32,
    #[serde(rename = "BPSWrite")]
    pub bps_write: u32,
    #[serde(rename = "TimeStamp")]
    pub timestamp: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct MonitorResponse {
    #[serde(rename = "RequestId")]
    pub request_id: String,
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "MonitorData")]
    pub monitor_data: Vec<MonitorData>,
}

fn deserialize_single_key_map<'de, V, D>(d: D) -> Result<V, D::Error>
    where D: Deserializer<'de>,
          V: DeserializeOwned
{
    struct SingleKeyMapVisitor<V: DeserializeOwned>(PhantomData<V>);

    impl<'de, V> de::Visitor<'de> for SingleKeyMapVisitor<V>
        where V: DeserializeOwned
    {
        type Value = V;

        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
            formatter.write_str("a map")
        }

        #[inline]
        fn visit_map<T>(self, mut visitor: T) -> Result<V, T::Error>
            where T: de::MapAccess<'de>
        {
            let item: Option<(String, V)> = visitor.next_entry()?;
            if let Some((_, value)) = item {
                return Ok(value);
            }
            Err(de::Error::custom("No single key value in map"))
        }
    }
    d.deserialize_map(SingleKeyMapVisitor(PhantomData))
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_deserialize_instances() {
        let json = r#"{
  "Instances": {
    "Instance": [
      {
        "CreationTime": "2015-07-27T07:08Z",
        "DeviceAvailable": true,
        "EipAddress": {},
        "ExpiredTime": "2011-09-08T16:00Z",
        "HostName": "iZ94t3s0jxkZ",
        "ImageId": "centos6u5_64_20G_aliaegis_20150130.vhd",
        "InnerIpAddress": {
          "IpAddress": [
            "10.170.106.80"
          ]
        },
        "InstanceChargeType": "PostPaid",
        "InstanceId": "i-94t3s0jxk",
        "InstanceName": "dd\u6027\u80fd\u6d4b\u8bd5",
        "InstanceNetworkType": "classic",
        "InstanceType": "ecs.s2.large",
        "InternetChargeType": "PayByTraffic",
        "InternetMaxBandwidthIn": -1,
        "InternetMaxBandwidthOut": 1,
        "IoOptimized": false,
        "OperationLocks": {
          "LockReason": []
        },
        "PublicIpAddress": {
          "IpAddress": [
            "120.25.13.106"
          ]
        },
        "RegionId": "cn-shenzhen",
        "SecurityGroupIds": {
          "SecurityGroupId": [
            "sg-94kd0cyg0"
          ]
        },
        "SerialNumber": "51d1353b-22bf-4567-a176-8b3e12e43135",
        "Status": "Running",
        "VpcAttributes": {
          "PrivateIpAddress": {
            "IpAddress": []
          }
        },
        "ZoneId": "cn-shenzhen-a"
      }
    ]
  },
  "PageNumber": 1,
  "PageSize": 10,
  "RequestId": "14A07460-EBE7-47CA-9757-12CC4761D47A",
  "TotalCount": 1
}"#;
        let instances = ::serde_json::from_str::<super::Instances>(json).unwrap();
        assert_eq!(instances.page, 1);
        assert_eq!(instances.size, 10);
        assert_eq!(instances.total, 1);
        assert_eq!(instances.instances.len(), 1);
        let instance = &instances.instances[0];
        assert_eq!(instance.id, "i-94t3s0jxk");
        assert_eq!(instance.ip(), "120.25.13.106");
    }
}
<commit_msg>Use serde flatten for pagination (#23)<commit_after>use std::fmt;
use std::marker::PhantomData;
use serde::de::{self, Deserializer, DeserializeOwned};

#[derive(Debug, Deserialize, Clone)]
pub struct Pagination {
    #[serde(rename = "TotalCount")]
    pub total: usize,
    #[serde(rename = "PageNumber")]
    pub page: usize,
    #[serde(rename = "PageSize")]
    pub size: usize,
}

#[derive(Debug, Deserialize, Clone)]
pub struct Instance {
    #[serde(rename = "InstanceId")]
    pub id: String,
    #[serde(rename = "InstanceName")]
    pub name: String,
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "PublicIpAddress")]
    pub public_ip_address: Vec<String>,
}

impl Instance {
    pub fn ip(&self) -> &str {
        &self.public_ip_address[0]
    }
}

#[derive(Debug, Deserialize, Clone)]
pub struct Instances {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "Instances")]
    pub instances: Vec<Instance>,
    #[serde(flatten)]
    pub pagination: Pagination,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceStatus {
    #[serde(rename = "InstanceId")]
    pub id: String,
    #[serde(rename = "Status")]
    pub status: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceStatuses {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "InstanceStatuses")]
    pub instance_statuses: Vec<InstanceStatus>,
    #[serde(flatten)]
    pub pagination: Pagination,
}

#[derive(Debug, Deserialize, Clone)]
pub struct Region {
    #[serde(rename = "RegionId")]
    pub id: String,
    #[serde(rename = "LocalName")]
    pub name: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct Regions {
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "Regions")]
    pub regions: Vec<Region>,
}

#[derive(Debug, Deserialize, Clone)]
pub struct InstanceCreateResponse {
    #[serde(rename = "InstanceId")]
    pub id: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct MonitorData {
    #[serde(rename = "InstanceId")]
    pub instance_id: String,
    #[serde(rename = "CPU")]
    pub cpu: u8,
    #[serde(rename = "IntranetRX")]
    pub intranet_rx: u32,
    #[serde(rename = "IntranetTX")]
    pub intranet_tx: u32,
    #[serde(rename = "IntranetBandwidth")]
    pub intranet_bandwidth: u32,
    #[serde(rename = "InternetRX")]
    pub internet_rx: u32,
    #[serde(rename = "InternetTX")]
    pub internet_tx: u32,
    #[serde(rename = "InternetBandwidth")]
    pub internet_bandwidth: u32,
    #[serde(rename = "IOPSRead")]
    pub iops_read: u32,
    #[serde(rename = "IOPSWrite")]
    pub iops_write: u32,
    #[serde(rename = "BPSRead")]
    pub bps_read: u32,
    #[serde(rename = "BPSWrite")]
    pub bps_write: u32,
    #[serde(rename = "TimeStamp")]
    pub timestamp: String,
}

#[derive(Debug, Deserialize, Clone)]
pub struct MonitorResponse {
    #[serde(rename = "RequestId")]
    pub request_id: String,
    #[serde(deserialize_with = "deserialize_single_key_map")]
    #[serde(rename = "MonitorData")]
    pub monitor_data: Vec<MonitorData>,
}

fn deserialize_single_key_map<'de, V, D>(d: D) -> Result<V, D::Error>
    where D: Deserializer<'de>,
          V: DeserializeOwned
{
    struct SingleKeyMapVisitor<V: DeserializeOwned>(PhantomData<V>);

    impl<'de, V> de::Visitor<'de> for SingleKeyMapVisitor<V>
        where V: DeserializeOwned
    {
        type Value = V;

        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
            formatter.write_str("a map")
        }

        #[inline]
        fn visit_map<T>(self, mut visitor: T) -> Result<V, T::Error>
            where T: de::MapAccess<'de>
        {
            let item: Option<(String, V)> = visitor.next_entry()?;
            if let Some((_, value)) = item {
                return Ok(value);
            }
            Err(de::Error::custom("No single key value in map"))
        }
    }
    d.deserialize_map(SingleKeyMapVisitor(PhantomData))
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_deserialize_instances() {
        let json = r#"{
  "Instances": {
    "Instance": [
      {
        "CreationTime": "2015-07-27T07:08Z",
        "DeviceAvailable": true,
        "EipAddress": {},
        "ExpiredTime": "2011-09-08T16:00Z",
        "HostName": "iZ94t3s0jxkZ",
        "ImageId": "centos6u5_64_20G_aliaegis_20150130.vhd",
        "InnerIpAddress": {
          "IpAddress": [
            "10.170.106.80"
          ]
        },
        "InstanceChargeType": "PostPaid",
        "InstanceId": "i-94t3s0jxk",
        "InstanceName": "dd\u6027\u80fd\u6d4b\u8bd5",
        "InstanceNetworkType": "classic",
        "InstanceType": "ecs.s2.large",
        "InternetChargeType": "PayByTraffic",
        "InternetMaxBandwidthIn": -1,
        "InternetMaxBandwidthOut": 1,
        "IoOptimized": false,
        "OperationLocks": {
          "LockReason": []
        },
        "PublicIpAddress": {
          "IpAddress": [
            "120.25.13.106"
          ]
        },
        "RegionId": "cn-shenzhen",
        "SecurityGroupIds": {
          "SecurityGroupId": [
            "sg-94kd0cyg0"
          ]
        },
        "SerialNumber": "51d1353b-22bf-4567-a176-8b3e12e43135",
        "Status": "Running",
        "VpcAttributes": {
          "PrivateIpAddress": {
            "IpAddress": []
          }
        },
        "ZoneId": "cn-shenzhen-a"
      }
    ]
  },
  "PageNumber": 1,
  "PageSize": 10,
  "RequestId": "14A07460-EBE7-47CA-9757-12CC4761D47A",
  "TotalCount": 1
}"#;
        let instances = ::serde_json::from_str::<super::Instances>(json).unwrap();
        assert_eq!(instances.pagination.page, 1);
        assert_eq!(instances.pagination.size, 10);
        assert_eq!(instances.pagination.total, 1);
        assert_eq!(instances.instances.len(), 1);
        let instance = &instances.instances[0];
        assert_eq!(instance.id, "i-94t3s0jxk");
        assert_eq!(instance.ip(), "120.25.13.106");
    }
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct GistFile {
  pub size: i64,
  pub raw_url: String,
  // type: String
//  pub truncated: bool,
  pub language: Option<String>
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Gist {
  pub url: String,
  pub forks_url: String,
  pub commits_url: String,
  pub id: String,
  pub description: Option<String>,
  pub public: bool,
  pub owner: User,
  pub user: Option<User>,
  pub files: HashMap<String, GistFile>,
  pub comments: i64,
  pub comments_url: String,
  pub html_url: String,
  pub git_pull_url: String,
  pub git_push_url: String,
  pub created_at: String,
  pub updated_at: String
}

#[derive(Debug, RustcDecodable)]
pub struct GistFork {
  user: User,
  url: String,
  id: String,
  created_at: String,
  updated_at: String
}

#[derive(Debug, RustcEncodable)]
pub struct Content {
  pub filename: Option<&'static str>,
  pub content: &'static str
}

impl Content {
  pub fn new(filename: Option<&'static str>, content: &'static str) -> Content {
    Content { filename: filename, content: content }
  }
}

#[derive(Debug, RustcEncodable)]
pub struct GistReq {
  pub description: Option<&'static str>,
  pub public: Option<bool>,
  pub files: HashMap<&'static str, Content>
}

impl GistReq {
  pub fn new(desc: Option<&'static str>, public: bool, files: HashMap<&'static str, &'static str>) -> GistReq {
    let mut contents = HashMap::new();
    for (k,v) in files {
      contents.insert(k, Content::new(None, v));
    }
    GistReq {
      description: desc,
      public: Some(public),
      files: contents
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Permissions {
  pub admin: bool,
  pub push: bool,
  pub pull: bool
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Repo {
  pub id: i64,
  pub owner: User,
  pub name: String,
  pub full_name: String,
  pub description: String,
  // private (keyword)
  pub fork: bool,
  pub url: String,
  html_url: String,
  archive_url: String,
  assignees_url: String,
  blobs_url: String,
  branches_url: String,
  clone_url: String,
  collaborators_url: String,
  comments_url: String,
  commits_url: String,
  compare_url: String,
  contents_url: String,
  contributors_url: String,
  downloads_url: String,
  events_url: String,
  forks_url: String,
  git_commits_url: String,
  git_refs_url: String,
  git_tags_url: String,
  git_url: String,
  hooks_url: String,
  issue_comment_url: String,
  issue_events_url: String,
  issues_url: String,
  keys_url: String,
  labels_url: String,
  languages_url: String,
  merges_url: String,
  milestones_url: String,
  mirror_url: Option<String>,
  notifications_url: String,
  pulls_url: String,
  releases_url: String,
  ssh_url: String,
  stargazers_url: String,
  statuses_url: String,
  subscribers_url: String,
  subscription_url: String,
  svn_url: String,
  tags_url: String,
  teams_url: String,
  trees_url: String,
  homepage: Option<String>,
  language: Option<String>,
  forks_count: i64,
  stargazers_count: i64,
  watchers_count: i64,
  size: i64,
  default_branch: String,
  open_issues_count: i64,
  has_issues: bool,
  has_wiki: bool,
  has_pages: bool,
  has_downloads: bool,
  pushed_at: String,
  created_at: String,
  updated_at: String,
//  permissions: Permissions
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct User {
  pub login: String,
  pub id: u64,
  pub avatar_url: String,
  pub gravatar_id: String,
  pub url: String,
  pub html_url: String,
  pub followers_url: String,
  following_url: String,
  gists_url: String,
  starred_url: String,
  subscriptions_url: String,
  organizations_url: String,
  repos_url: String,
  events_url: String,
  received_events_url: String,
  // type (keyword)
  site_admin: bool
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Commit {
  pub label: String,
  // ref (keyword)
  pub sha: String,
  pub user: User,
  pub repo: Option<Repo>
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Label {
  pub url: String,
  pub name: String,
  pub color: String
}

#[derive(Debug, RustcEncodable)]
pub struct PullEdit {
  title: Option<&'static str>,
  body: Option<&'static str>,
  state: Option<&'static str>
}

impl PullEdit {
  pub fn new(title: Option<&'static str>, body: Option<&'static str>, state: Option<&'static str>) -> PullEdit {
    PullEdit { title: title, body: body, state: state }
  }
}

#[derive(Debug, RustcEncodable)]
pub struct PullReq {
  pub title: &'static str,
  pub head: &'static str,
  pub base: &'static str,
  pub body: Option<&'static str>
}

impl PullReq {
  pub fn new(title: &'static str, head: &'static str, base: &'static str, body: Option<&'static str>) -> PullReq {
    PullReq {
      title: title,
      head: head,
      base: base,
      body: body
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Pull {
  pub id: u64,
  pub url: String,
  pub html_url: String,
  pub diff_url: String,
  pub patch_url: String,
  pub issue_url: String,
  pub commits_url: String,
  pub review_comments_url: String,
  pub review_comment_url: String,
  pub comments_url: String,
  pub statuses_url: String,
  pub number: u64,
  pub state: String,
  pub title: String,
  pub body: String,
  pub created_at: String,
  pub updated_at: String,
  pub closed_at: Option<String>,
  pub merged_at: Option<String>,
  //pub head: Commit,
//  pub base: Commit,
  // links
  pub user: User,
  pub merge_commit_sha: Option<String>,
  pub mergeable: Option<bool>,
  pub merged_by: Option<User>,
  pub comments: Option<u64>,
  pub commits: Option<u64>,
  pub additions: Option<u64>,
  pub deletions: Option<u64>,
  pub changed_files: Option<u64>
}

#[derive(Debug, RustcEncodable)]
pub struct IssueReq {
  pub title: &'static str,
  pub body: Option<&'static str>,
  pub assignee: Option<&'static str>,
  pub milestone: Option<i64>,
  pub labels: Vec<&'static str>
}

impl IssueReq {
  pub fn new(title: &'static str, body: Option<&'static str>, assignee: Option<&'static str>,
             milestone: Option<i64>, labels: Vec<&'static str>) -> IssueReq {
    IssueReq {
      title: title,
      body: body,
      assignee: assignee,
      milestone: milestone,
      labels: labels
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Issue {
  pub title: String
}
<commit_msg>fill in issue rep<commit_after>use std::collections::HashMap;

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct GistFile {
  pub size: i64,
  pub raw_url: String,
  // type: String
//  pub truncated: bool,
  pub language: Option<String>
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Gist {
  pub url: String,
  pub forks_url: String,
  pub commits_url: String,
  pub id: String,
  pub description: Option<String>,
  pub public: bool,
  pub owner: User,
  pub user: Option<User>,
  pub files: HashMap<String, GistFile>,
  pub comments: i64,
  pub comments_url: String,
  pub html_url: String,
  pub git_pull_url: String,
  pub git_push_url: String,
  pub created_at: String,
  pub updated_at: String
}

#[derive(Debug, RustcDecodable)]
pub struct GistFork {
  user: User,
  url: String,
  id: String,
  created_at: String,
  updated_at: String
}

#[derive(Debug, RustcEncodable)]
pub struct Content {
  pub filename: Option<&'static str>,
  pub content: &'static str
}

impl Content {
  pub fn new(filename: Option<&'static str>, content: &'static str) -> Content {
    Content { filename: filename, content: content }
  }
}

#[derive(Debug, RustcEncodable)]
pub struct GistReq {
  pub description: Option<&'static str>,
  pub public: Option<bool>,
  pub files: HashMap<&'static str, Content>
}

impl GistReq {
  pub fn new(desc: Option<&'static str>, public: bool, files: HashMap<&'static str, &'static str>) -> GistReq {
    let mut contents = HashMap::new();
    for (k,v) in files {
      contents.insert(k, Content::new(None, v));
    }
    GistReq {
      description: desc,
      public: Some(public),
      files: contents
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Permissions {
  pub admin: bool,
  pub push: bool,
  pub pull: bool
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Repo {
  pub id: i64,
  pub owner: User,
  pub name: String,
  pub full_name: String,
  pub description: String,
  // private (keyword)
  pub fork: bool,
  pub url: String,
  html_url: String,
  archive_url: String,
  assignees_url: String,
  blobs_url: String,
  branches_url: String,
  clone_url: String,
  collaborators_url: String,
  comments_url: String,
  commits_url: String,
  compare_url: String,
  contents_url: String,
  contributors_url: String,
  downloads_url: String,
  events_url: String,
  forks_url: String,
  git_commits_url: String,
  git_refs_url: String,
  git_tags_url: String,
  git_url: String,
  hooks_url: String,
  issue_comment_url: String,
  issue_events_url: String,
  issues_url: String,
  keys_url: String,
  labels_url: String,
  languages_url: String,
  merges_url: String,
  milestones_url: String,
  mirror_url: Option<String>,
  notifications_url: String,
  pulls_url: String,
  releases_url: String,
  ssh_url: String,
  stargazers_url: String,
  statuses_url: String,
  subscribers_url: String,
  subscription_url: String,
  svn_url: String,
  tags_url: String,
  teams_url: String,
  trees_url: String,
  homepage: Option<String>,
  language: Option<String>,
  forks_count: i64,
  stargazers_count: i64,
  watchers_count: i64,
  size: i64,
  default_branch: String,
  open_issues_count: i64,
  has_issues: bool,
  has_wiki: bool,
  has_pages: bool,
  has_downloads: bool,
  pushed_at: String,
  created_at: String,
  updated_at: String,
//  permissions: Permissions
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct User {
  pub login: String,
  pub id: u64,
  pub avatar_url: String,
  pub gravatar_id: String,
  pub url: String,
  pub html_url: String,
  pub followers_url: String,
  following_url: String,
  gists_url: String,
  starred_url: String,
  subscriptions_url: String,
  organizations_url: String,
  repos_url: String,
  events_url: String,
  received_events_url: String,
  // type (keyword)
  site_admin: bool
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Commit {
  pub label: String,
  // ref (keyword)
  pub sha: String,
  pub user: User,
  pub repo: Option<Repo>
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Label {
  pub url: String,
  pub name: String,
  pub color: String
}

#[derive(Debug, RustcEncodable)]
pub struct PullEdit {
  title: Option<&'static str>,
  body: Option<&'static str>,
  state: Option<&'static str>
}

impl PullEdit {
  pub fn new(title: Option<&'static str>, body: Option<&'static str>, state: Option<&'static str>) -> PullEdit {
    PullEdit { title: title, body: body, state: state }
  }
}

#[derive(Debug, RustcEncodable)]
pub struct PullReq {
  pub title: &'static str,
  pub head: &'static str,
  pub base: &'static str,
  pub body: Option<&'static str>
}

impl PullReq {
  pub fn new(title: &'static str, head: &'static str, base: &'static str, body: Option<&'static str>) -> PullReq {
    PullReq {
      title: title,
      head: head,
      base: base,
      body: body
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Pull {
  pub id: u64,
  pub url: String,
  pub html_url: String,
  pub diff_url: String,
  pub patch_url: String,
  pub issue_url: String,
  pub commits_url: String,
  pub review_comments_url: String,
  pub review_comment_url: String,
  pub comments_url: String,
  pub statuses_url: String,
  pub number: u64,
  pub state: String,
  pub title: String,
  pub body: String,
  pub created_at: String,
  pub updated_at: String,
  pub closed_at: Option<String>,
  pub merged_at: Option<String>,
  //pub head: Commit,
//  pub base: Commit,
  // links
  pub user: User,
  pub merge_commit_sha: Option<String>,
  pub mergeable: Option<bool>,
  pub merged_by: Option<User>,
  pub comments: Option<u64>,
  pub commits: Option<u64>,
  pub additions: Option<u64>,
  pub deletions: Option<u64>,
  pub changed_files: Option<u64>
}

#[derive(Debug, RustcEncodable)]
pub struct IssueReq {
  pub title: &'static str,
  pub body: Option<&'static str>,
  pub assignee: Option<&'static str>,
  pub milestone: Option<i64>,
  pub labels: Vec<&'static str>
}

impl IssueReq {
  pub fn new(title: &'static str, body: Option<&'static str>, assignee: Option<&'static str>,
             milestone: Option<i64>, labels: Vec<&'static str>) -> IssueReq {
    IssueReq {
      title: title,
      body: body,
      assignee: assignee,
      milestone: milestone,
      labels: labels
    }
  }
}

#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct Issue {
  pub id: i64,
  pub url: String,
  pub labels_url: String,
  pub comments_url: String,
  pub events_url: String,
  pub html_url: String,
  pub number: i64,
  pub state: String,
  pub title: String,
  pub body: String,
  pub user: User,
  pub labels: Vec<Label>,
  pub assignee: Option<User>,
  pub locked: bool,
  pub comments: i64,
  pub closed_at: Option<String>,
  pub created_at: String,
  pub updated_at: String
}
<|endoftext|>