<commit_before>mod rope;
mod seq;
mod small;

use {alloc, Buf};
use self::seq::Seq;
use self::small::Small;
use self::rope::{Rope, RopeBuf};
use std::{cmp, fmt, ops};
use std::io::Cursor;
use std::sync::Arc;

#[derive(Clone)]
pub struct Bytes {
    kind: Kind,
}

#[derive(Clone)]
enum Kind {
    Seq(Seq),
    Small(Small),
    Rope(Arc<Rope>),
}

pub struct BytesBuf<'a> {
    kind: BufKind<'a>,
}

enum BufKind<'a> {
    Cursor(Cursor<&'a [u8]>),
    Rope(RopeBuf<'a>),
}

impl Bytes {
    /// Return an empty `Bytes`
    pub fn empty() -> Bytes {
        Bytes { kind: Kind::Small(Small::empty()) }
    }

    /// Creates a new `Bytes` from a `MemRef`, an offset, and a length.
    ///
    /// This function is unsafe as there are no guarantees that the given
    /// arguments are valid.
    #[inline]
    pub unsafe fn from_mem_ref(mem: alloc::MemRef, pos: u32, len: u32) -> Bytes {
        Small::from_slice(&mem.bytes_slice(pos as usize, pos as usize + len as usize))
            .map(|b| Bytes { kind: Kind::Small(b) })
            .unwrap_or_else(|| {
                let seq = Seq::from_mem_ref(mem, pos, len);
                Bytes { kind: Kind::Seq(seq) }
            })
    }

    pub fn buf(&self) -> BytesBuf {
        let kind = match self.kind {
            Kind::Seq(ref v) => BufKind::Cursor(v.buf()),
            Kind::Small(ref v) => BufKind::Cursor(v.buf()),
            Kind::Rope(ref v) => BufKind::Rope(v.buf()),
        };

        BytesBuf { kind: kind }
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn len(&self) -> usize {
        match self.kind {
            Kind::Seq(ref v) => v.len(),
            Kind::Small(ref v) => v.len(),
            Kind::Rope(ref v) => v.len(),
        }
    }

    pub fn concat(&self, other: &Bytes) -> Bytes {
        Rope::concat(self.clone(), other.clone())
    }

    /// Returns a new ByteStr value containing the byte range between `begin`
    /// (inclusive) and `end` (exclusive)
    pub fn slice(&self, begin: usize, end: usize) -> Bytes {
        match self.kind {
            Kind::Seq(ref v) => v.slice(begin, end),
            Kind::Small(ref v) => v.slice(begin, end),
            Kind::Rope(ref v) => v.slice(begin, end),
        }
    }

    /// Returns a new ByteStr value containing the byte range starting from
    /// `begin` (inclusive) to the end of the byte str.
    ///
    /// Equivalent to `bytes.slice(begin, bytes.len())`
    pub fn slice_from(&self, begin: usize) -> Bytes {
        self.slice(begin, self.len())
    }

    /// Returns a new ByteStr value containing the byte range from the start up
    /// to `end` (exclusive).
    ///
    /// Equivalent to `bytes.slice(0, end)`
    pub fn slice_to(&self, end: usize) -> Bytes {
        self.slice(0, end)
    }

    /// Returns the Rope depth
    fn depth(&self) -> u16 {
        match self.kind {
            Kind::Rope(ref r) => r.depth(),
            _ => 0,
        }
    }

    fn into_rope(self) -> Result<Arc<Rope>, Bytes> {
        match self.kind {
            Kind::Rope(r) => Ok(r),
            _ => Err(self),
        }
    }
}

impl ops::Index<usize> for Bytes {
    type Output = u8;

    fn index(&self, index: usize) -> &u8 {
        match self.kind {
            Kind::Seq(ref v) => v.index(index),
            Kind::Small(ref v) => v.index(index),
            Kind::Rope(ref v) => v.index(index),
        }
    }
}

impl<T: AsRef<[u8]>> From<T> for Bytes {
    fn from(src: T) -> Bytes {
        Small::from_slice(src.as_ref())
            .map(|b| Bytes { kind: Kind::Small(b) })
            .unwrap_or_else(|| Seq::from_slice(src.as_ref()))
    }
}

impl cmp::PartialEq<Bytes> for Bytes {
    fn eq(&self, other: &Bytes) -> bool {
        if self.len() != other.len() {
            return false;
        }

        let mut buf1 = self.buf();
        let mut buf2 = self.buf();

        while buf1.has_remaining() {
            let len;

            {
                let b1 = buf1.bytes();
                let b2 = buf2.bytes();

                len = cmp::min(b1.len(), b2.len());

                if b1[..len] != b2[..len] {
                    return false;
                }
            }

            buf1.advance(len);
            buf2.advance(len);
        }

        true
    }

    fn ne(&self, other: &Bytes) -> bool {
        return !self.eq(other)
    }
}

impl<'a> Buf for BytesBuf<'a> {
    fn remaining(&self) -> usize {
        match self.kind {
            BufKind::Cursor(ref v) => v.remaining(),
            BufKind::Rope(ref v) => v.remaining(),
        }
    }

    fn bytes(&self) -> &[u8] {
        match self.kind {
            BufKind::Cursor(ref v) => v.bytes(),
            BufKind::Rope(ref v) => v.bytes(),
        }
    }

    fn advance(&mut self, cnt: usize) {
        match self.kind {
            BufKind::Cursor(ref mut v) => v.advance(cnt),
            BufKind::Rope(ref mut v) => v.advance(cnt),
        }
    }
}

/*
 *
 * ===== Internal utilities =====
 *
 */

impl fmt::Debug for Bytes {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let mut buf = self.buf();

        try!(write!(fmt, "Bytes[len={}; ", self.len()));

        let mut rem = 128;

        while buf.has_remaining() {
            let byte = buf.read_u8();

            if rem > 0 {
                if is_ascii(byte) {
                    try!(write!(fmt, "{}", byte as char));
                } else {
                    try!(write!(fmt, "\\x{:02X}", byte));
                }

                rem -= 1;
            } else {
                try!(write!(fmt, " ... "));
                break;
            }
        }

        try!(write!(fmt, "]"));

        Ok(())
    }
}

fn is_ascii(byte: u8) -> bool {
    match byte {
        10 | 13 | 32...126 => true,
        _ => false,
    }
}
<commit_msg>Add some docs<commit_after>mod rope;
mod seq;
mod small;

use {alloc, Buf};
use self::seq::Seq;
use self::small::Small;
use self::rope::{Rope, RopeBuf};
use std::{cmp, fmt, ops};
use std::io::Cursor;
use std::sync::Arc;

/// An immutable sequence of bytes
#[derive(Clone)]
pub struct Bytes {
    kind: Kind,
}

#[derive(Clone)]
enum Kind {
    Seq(Seq),
    Small(Small),
    Rope(Arc<Rope>),
}

pub struct BytesBuf<'a> {
    kind: BufKind<'a>,
}

enum BufKind<'a> {
    Cursor(Cursor<&'a [u8]>),
    Rope(RopeBuf<'a>),
}

impl Bytes {
    /// Return an empty `Bytes`
    pub fn empty() -> Bytes {
        Bytes { kind: Kind::Small(Small::empty()) }
    }

    /// Creates a new `Bytes` from a `MemRef`, an offset, and a length.
    ///
    /// This function is unsafe as there are no guarantees that the given
    /// arguments are valid.
    #[inline]
    pub unsafe fn from_mem_ref(mem: alloc::MemRef, pos: u32, len: u32) -> Bytes {
        Small::from_slice(&mem.bytes_slice(pos as usize, pos as usize + len as usize))
            .map(|b| Bytes { kind: Kind::Small(b) })
            .unwrap_or_else(|| {
                let seq = Seq::from_mem_ref(mem, pos, len);
                Bytes { kind: Kind::Seq(seq) }
            })
    }

    pub fn buf(&self) -> BytesBuf {
        let kind = match self.kind {
            Kind::Seq(ref v) => BufKind::Cursor(v.buf()),
            Kind::Small(ref v) => BufKind::Cursor(v.buf()),
            Kind::Rope(ref v) => BufKind::Rope(v.buf()),
        };

        BytesBuf { kind: kind }
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn len(&self) -> usize {
        match self.kind {
            Kind::Seq(ref v) => v.len(),
            Kind::Small(ref v) => v.len(),
            Kind::Rope(ref v) => v.len(),
        }
    }

    pub fn concat(&self, other: &Bytes) -> Bytes {
        Rope::concat(self.clone(), other.clone())
    }

    /// Returns a new ByteStr value containing the byte range between `begin`
    /// (inclusive) and `end` (exclusive)
    pub fn slice(&self, begin: usize, end: usize) -> Bytes {
        match self.kind {
            Kind::Seq(ref v) => v.slice(begin, end),
            Kind::Small(ref v) => v.slice(begin, end),
            Kind::Rope(ref v) => v.slice(begin, end),
        }
    }

    /// Returns a new ByteStr value containing the byte range starting from
    /// `begin` (inclusive) to the end of the byte str.
    ///
    /// Equivalent to `bytes.slice(begin, bytes.len())`
    pub fn slice_from(&self, begin: usize) -> Bytes {
        self.slice(begin, self.len())
    }

    /// Returns a new ByteStr value containing the byte range from the start up
    /// to `end` (exclusive).
    ///
    /// Equivalent to `bytes.slice(0, end)`
    pub fn slice_to(&self, end: usize) -> Bytes {
        self.slice(0, end)
    }

    /// Returns the Rope depth
    fn depth(&self) -> u16 {
        match self.kind {
            Kind::Rope(ref r) => r.depth(),
            _ => 0,
        }
    }

    fn into_rope(self) -> Result<Arc<Rope>, Bytes> {
        match self.kind {
            Kind::Rope(r) => Ok(r),
            _ => Err(self),
        }
    }
}

impl ops::Index<usize> for Bytes {
    type Output = u8;

    fn index(&self, index: usize) -> &u8 {
        match self.kind {
            Kind::Seq(ref v) => v.index(index),
            Kind::Small(ref v) => v.index(index),
            Kind::Rope(ref v) => v.index(index),
        }
    }
}

impl<T: AsRef<[u8]>> From<T> for Bytes {
    fn from(src: T) -> Bytes {
        Small::from_slice(src.as_ref())
            .map(|b| Bytes { kind: Kind::Small(b) })
            .unwrap_or_else(|| Seq::from_slice(src.as_ref()))
    }
}

impl cmp::PartialEq<Bytes> for Bytes {
    fn eq(&self, other: &Bytes) -> bool {
        if self.len() != other.len() {
            return false;
        }

        let mut buf1 = self.buf();
        let mut buf2 = self.buf();

        while buf1.has_remaining() {
            let len;

            {
                let b1 = buf1.bytes();
                let b2 = buf2.bytes();

                len = cmp::min(b1.len(), b2.len());

                if b1[..len] != b2[..len] {
                    return false;
                }
            }

            buf1.advance(len);
            buf2.advance(len);
        }

        true
    }

    fn ne(&self, other: &Bytes) -> bool {
        return !self.eq(other)
    }
}

impl<'a> Buf for BytesBuf<'a> {
    fn remaining(&self) -> usize {
        match self.kind {
            BufKind::Cursor(ref v) => v.remaining(),
            BufKind::Rope(ref v) => v.remaining(),
        }
    }

    fn bytes(&self) -> &[u8] {
        match self.kind {
            BufKind::Cursor(ref v) => v.bytes(),
            BufKind::Rope(ref v) => v.bytes(),
        }
    }

    fn advance(&mut self, cnt: usize) {
        match self.kind {
            BufKind::Cursor(ref mut v) => v.advance(cnt),
            BufKind::Rope(ref mut v) => v.advance(cnt),
        }
    }
}

/*
 *
 * ===== Internal utilities =====
 *
 */

impl fmt::Debug for Bytes {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let mut buf = self.buf();

        try!(write!(fmt, "Bytes[len={}; ", self.len()));

        let mut rem = 128;

        while buf.has_remaining() {
            let byte = buf.read_u8();

            if rem > 0 {
                if is_ascii(byte) {
                    try!(write!(fmt, "{}", byte as char));
                } else {
                    try!(write!(fmt, "\\x{:02X}", byte));
                }

                rem -= 1;
            } else {
                try!(write!(fmt, " ... "));
                break;
            }
        }

        try!(write!(fmt, "]"));

        Ok(())
    }
}

fn is_ascii(byte: u8) -> bool {
    match byte {
        10 | 13 | 32...126 => true,
        _ => false,
    }
}
<|endoftext|>


<commit_before>
// Copyright 2017 The gltf Library Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use futures::{self, future};
use import::{self, config, data};
use json;

use futures::{Future, Poll};
use image_crate::{load_from_memory, load_from_memory_with_format};
use image_crate::GenericImage;
use image_crate::ImageFormat as Format;
use image_crate::ImageFormat::{JPEG as Jpeg, PNG as Png};
use image_crate::ImageResult;
use import::{Config, Source};
use root::Root;
use {Data, DynamicImage, Gltf};

/// The contents of a .glb file.
#[derive(Clone, Debug)]
struct Glb(Box<[u8]>);

/// The header section of a .glb file.
#[derive(Copy, Clone, Debug)]
#[repr(C)]
struct Header {
    /// Must be `b"glTF"`.
    magic: [u8; 4],

    /// Must be `2`.
    version: u32,

    /// Must match the length of the parent .glb file.
    length: u32,
}

/// The header of the JSON or BIN section of a .glb file.
#[derive(Copy, Clone, Debug)]
#[repr(C)]
struct ChunkInfo {
    /// The length of the chunk data in byte excluding the header.
    length: u32,

    /// Either `b"JSON"` or `b"BIN\0"`.
    kind: [u8; 4],

    // Data follows...
}

/// Represents a subset of `Importer::glb`.
#[derive(Copy, Clone, Debug, Default)]
struct Slice {
    /// Offset into `glb` in bytes.
    offset: usize,

    /// Length of the slice in bytes.
    length: usize,
}

enum AsyncImage<S: import::Source> {
    /// Image data is borrowed from a buffer.
    Borrowed {
        /// The buffer index.
        index: usize,

        /// Byte offset into the indexed buffer where the image data begins.
        offset: usize,

        /// Length of the image past the offset in bytes.
        len: usize,

        /// The image format.
        format: Format,
    },

    /// Image data is owned.
    Owned {
        /// A `Future` that drives the loading of external image data.
        data: data::Async<S>,

        /// The image format.
        format: Option<Format>,
    },
}

impl<S: import::Source> Future for AsyncImage<S> {
    type Item = EncodedImage;
    type Error = import::Error<S>;
    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        match self {
            &mut AsyncImage::Borrowed { index, offset, len, format } => {
                Ok(futures::Async::Ready(EncodedImage::Borrowed {
                    index: index,
                    offset: offset,
                    len: len,
                    format: format,
                }))
            },
            &mut AsyncImage::Owned { ref mut data, format } => {
                data.poll()
                    .map(|async| {
                        async.map(|data| {
                            EncodedImage::Owned {
                                data: data,
                                format: format,
                            }
                        })
                    })
            },
        }
    }
}

enum EncodedImage {
    /// Image data is borrowed from a buffer.
    Borrowed {
        /// The buffer index.
        index: usize,

        /// Byte offset into the indexed buffer where the image data begins.
        offset: usize,

        /// Length of the image past the offset in bytes.
        len: usize,

        /// The image format.
        format: Format,
    },

    /// Image data is owned.
    Owned {
        /// A `Future` that drives the loading of external image data.
        data: Data,

        /// The image format.
        format: Option<Format>,
    },
}

/// The header, JSON, and BIN sections of a .glb file, respectively.
type Split = (Header, Slice, Option<Slice>);

impl Glb {
    /// Obtains a slice of the GLB data.
    fn slice(&self, slice: Slice) -> &[u8] {
        let begin = slice.offset;
        let end = begin + slice.length;
        &self.0[begin..end]
    }

    /// Splits loaded GLB into its three chunks.
    ///
    /// * Mandatory GLB header.
    /// * Mandatory JSON chunk.
    /// * Optional BIN chunk.
    fn split<S: import::Source>(&self) -> Result<Split, import::Error<S>> {
        use std::mem::{size_of, transmute};
        let err = |reason: &str| Err(import::Error::MalformedGlb(reason.to_string()));
        let glb = &self.0;
        let glb_ptr = glb.as_ptr();
        if glb.len() < size_of::<Header>() + size_of::<ChunkInfo>() {
            return err("GLB missing header");
        }

        // Offset in bytes into `glb`.
        let mut offset = 0isize;

        let header = unsafe {
            let x: *const Header = transmute(glb_ptr.offset(offset));
            if &(*x).magic != b"glTF" {
                return err("GLB does not start with `glTF`");
            }
            if (*x).length as usize != glb.len() {
                return err("length does not match length of data");
            }
            if (*x).version != 2 {
                return err("Not GLB version 2.0");
            }
            *x
        };

        offset += size_of::<Header>() as isize;
        let json_chunk = unsafe {
            let x: *const ChunkInfo = transmute(glb_ptr.offset(offset));
            offset += size_of::<ChunkInfo>() as isize;
            if (*x).length as usize > (glb.len() as isize - offset) as usize {
                return err("JSON chunkLength exceeds length of data");
            }
            if &(*x).kind != b"JSON" {
                return err("JSON chunkType is not `JSON`");
            }
            Slice {
                offset: offset as usize,
                length: (*x).length as usize,
            }
        };

        offset += json_chunk.length as isize;
        let blob_chunk = if glb.len() as isize - offset == 0 {
            None
        } else {
            unsafe {
                let x: *const ChunkInfo = transmute(glb_ptr.offset(offset));
                offset += size_of::<ChunkInfo>() as isize;
                if (*x).length as usize > (glb.len() as isize - offset) as usize {
                    return err("BIN chunkLength exceeds length of data");
                }
                if &(*x).kind != b"BIN\0" {
                    return err("BIN chunkType is not `BIN\0`");
                }
                Some(Slice {
                    offset: offset as usize,
                    length: (*x).length as usize,
                })
            }
        };

        Ok((header, json_chunk, blob_chunk))
    }
}

fn source_buffers<S: import::Source>(
    root: &Root,
    source: &S,
    blob: Option<Box<[u8]>>,
) -> Vec<data::Async<S>> {
    let json = root.as_json();
    let mut iter = json.buffers.iter();
    let mut buffers = vec![];
    if let Some(data) = blob {
        let future = Box::new(future::ok(data));
        // Skip the first buffer entry.
        let _ = iter.next();
        buffers.push(data::Async::full(future));
    }
    for entry in iter {
        let uri = entry.uri.as_ref().unwrap();
        let future = Box::new(source.source_external_data(uri));
        buffers.push(data::Async::full(future));
    }
    buffers
}

fn source_images<S: import::Source>(
    root: &Root,
    source: &S,
) -> Vec<AsyncImage<S>> {
    root.as_json().images
        .iter()
        .map(|entry| {
            let format = entry.mime_type.as_ref().map(|x| match x.0.as_str() {
                "image/jpeg" => Jpeg,
                "image/png" => Png,
                _ => unreachable!(),
            });
            if let Some(uri) = entry.uri.as_ref() {
                let future = Box::new(source.source_external_data(uri));
                AsyncImage::Owned {
                    data: data::Async::full(future),
                    format: format,
                }
            } else if let Some(index) = entry.buffer_view.as_ref() {
                let buffer_view = &root.as_json().buffer_views[index.value()];
                AsyncImage::Borrowed {
                    index: buffer_view.buffer.value(),
                    offset: buffer_view.byte_offset as usize,
                    len: buffer_view.byte_length as usize,
                    format: format.unwrap(),
                }
            } else {
                unreachable!()
            }
        })
        .collect()
}

fn decode_images(
    buffers: &[Data],
    images: Vec<EncodedImage>,
) -> ImageResult<Vec<DynamicImage>> {
    images
        .iter()
        .map(|entry| {
            match entry {
                &EncodedImage::Borrowed { index, offset, len, format } => {
                    let data = &buffers[index][offset..(offset + len)];
                    load_from_memory_with_format(data, format)
                },
                &EncodedImage::Owned { ref data, format: Some(format) } => {
                    load_from_memory_with_format(data, format)
                },
                &EncodedImage::Owned { ref data, format: None } => {
                    load_from_memory(data)
                },
            }
        })
        .collect()
}

fn validate<S: Source>(
    json: json::Root,
    strategy: config::ValidationStrategy,
    has_blob: bool,
) -> Box<Future<Item = Root, Error = import::Error<S>>> {
    use validation::{Error as Reason, Validate};
    Box::new(future::lazy(move || {
        let mut errs = vec![];
        match strategy {
            config::ValidationStrategy::Skip => {
                return Ok(Root::new(json));
            },
            config::ValidationStrategy::Minimal => {
                json.validate_minimally(
                    &json,
                    || json::Path::new(),
                    &mut |path, err| {
                        errs.push((path(), err));
                    },
                )
            },
            config::ValidationStrategy::Complete => {
                json.validate_completely(
                    &json,
                    || json::Path::new(),
                    &mut |path, err| {
                        errs.push((path(), err));
                    },
                )
            }
        }

        // Required for the `Minimal` and `Complete` validation strategies.
        for (index, buffer) in json.buffers.iter().enumerate() {
            let path = || {
                json::Path::new()
                    .field("buffers")
                    .index(index)
                    .field("uri")
            };
            match index {
                0 if has_blob => if buffer.uri.is_some() {
                    errs.push((path(), Reason::Missing));
                },
                _ if buffer.uri.is_none() => {
                    errs.push((path(), Reason::Missing));
                },
                _ => {},
            }
        }

        if errs.is_empty() {
            Ok(Root::new(json))
        } else {
            Err(import::Error::Validation(errs))
        }
    }))
}

/// Imports some glTF from the given data source.
pub fn import<S: import::Source>(
    data: Box<[u8]>,
    source: S,
    config: Config,
) -> Box<Future<Item = Gltf, Error = import::Error<S>>> {
    Box::new(future::lazy(move || {
        let glb = Glb(data);
        let (_, json_chunk, blob_chunk) = glb.split()?;
        let begin = json_chunk.offset;
        let end = begin + json_chunk.length;
        let json: json::Root = json::from_slice(&glb.0[begin..end])?;
        let blob = blob_chunk.map(|chunk| {
            glb.slice(chunk).to_vec().into_boxed_slice()
        });
        Ok((json, blob))
    })
        .and_then(move |(json, blob)| {
            let config = config;
            let has_blob = blob.is_some();
            validate(json, config.validation_strategy, has_blob)
                .join(future::ok(blob))
        })
        .and_then(move |(root, blob)| {
            let source = source;
            let buffers = source_buffers(&root, &source, blob);
            let images = source_images(&root, &source);
            future::ok(root)
                .join3(
                    future::join_all(buffers),
                    future::join_all(images),
                )
        })
        .and_then(|(root, buffers, images)| {
            let decoded_images = decode_images(&buffers, images)?;
            Ok((root, buffers, decoded_images))
        })
        .and_then(|(root, buffers, images)| {
            Ok(Gltf::new(root, buffers, images))
        }))
}

<commit_msg>Remove unused import<commit_after>
// Copyright 2017 The gltf Library Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use futures::{self, future};
use import::{self, config, data};
use json;

use futures::{Future, Poll};
use image_crate::{load_from_memory, load_from_memory_with_format};
use image_crate::ImageFormat as Format;
use image_crate::ImageFormat::{JPEG as Jpeg, PNG as Png};
use image_crate::ImageResult;
use import::{Config, Source};
use root::Root;
use {Data, DynamicImage, Gltf};

/// The contents of a .glb file.
#[derive(Clone, Debug)]
struct Glb(Box<[u8]>);

/// The header section of a .glb file.
#[derive(Copy, Clone, Debug)]
#[repr(C)]
struct Header {
    /// Must be `b"glTF"`.
    magic: [u8; 4],

    /// Must be `2`.
    version: u32,

    /// Must match the length of the parent .glb file.
    length: u32,
}

/// The header of the JSON or BIN section of a .glb file.
#[derive(Copy, Clone, Debug)]
#[repr(C)]
struct ChunkInfo {
    /// The length of the chunk data in byte excluding the header.
    length: u32,

    /// Either `b"JSON"` or `b"BIN\0"`.
    kind: [u8; 4],

    // Data follows...
}

/// Represents a subset of `Importer::glb`.
#[derive(Copy, Clone, Debug, Default)]
struct Slice {
    /// Offset into `glb` in bytes.
    offset: usize,

    /// Length of the slice in bytes.
    length: usize,
}

enum AsyncImage<S: import::Source> {
    /// Image data is borrowed from a buffer.
    Borrowed {
        /// The buffer index.
        index: usize,

        /// Byte offset into the indexed buffer where the image data begins.
        offset: usize,

        /// Length of the image past the offset in bytes.
        len: usize,

        /// The image format.
        format: Format,
    },

    /// Image data is owned.
    Owned {
        /// A `Future` that drives the loading of external image data.
        data: data::Async<S>,

        /// The image format.
        format: Option<Format>,
    },
}

impl<S: import::Source> Future for AsyncImage<S> {
    type Item = EncodedImage;
    type Error = import::Error<S>;
    fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
        match self {
            &mut AsyncImage::Borrowed { index, offset, len, format } => {
                Ok(futures::Async::Ready(EncodedImage::Borrowed {
                    index: index,
                    offset: offset,
                    len: len,
                    format: format,
                }))
            },
            &mut AsyncImage::Owned { ref mut data, format } => {
                data.poll()
                    .map(|async| {
                        async.map(|data| {
                            EncodedImage::Owned {
                                data: data,
                                format: format,
                            }
                        })
                    })
            },
        }
    }
}

enum EncodedImage {
    /// Image data is borrowed from a buffer.
    Borrowed {
        /// The buffer index.
        index: usize,

        /// Byte offset into the indexed buffer where the image data begins.
        offset: usize,

        /// Length of the image past the offset in bytes.
        len: usize,

        /// The image format.
        format: Format,
    },

    /// Image data is owned.
    Owned {
        /// A `Future` that drives the loading of external image data.
        data: Data,

        /// The image format.
        format: Option<Format>,
    },
}

/// The header, JSON, and BIN sections of a .glb file, respectively.
type Split = (Header, Slice, Option<Slice>);

impl Glb {
    /// Obtains a slice of the GLB data.
    fn slice(&self, slice: Slice) -> &[u8] {
        let begin = slice.offset;
        let end = begin + slice.length;
        &self.0[begin..end]
    }

    /// Splits loaded GLB into its three chunks.
    ///
    /// * Mandatory GLB header.
    /// * Mandatory JSON chunk.
    /// * Optional BIN chunk.
    fn split<S: import::Source>(&self) -> Result<Split, import::Error<S>> {
        use std::mem::{size_of, transmute};
        let err = |reason: &str| Err(import::Error::MalformedGlb(reason.to_string()));
        let glb = &self.0;
        let glb_ptr = glb.as_ptr();
        if glb.len() < size_of::<Header>() + size_of::<ChunkInfo>() {
            return err("GLB missing header");
        }

        // Offset in bytes into `glb`.
        let mut offset = 0isize;

        let header = unsafe {
            let x: *const Header = transmute(glb_ptr.offset(offset));
            if &(*x).magic != b"glTF" {
                return err("GLB does not start with `glTF`");
            }
            if (*x).length as usize != glb.len() {
                return err("length does not match length of data");
            }
            if (*x).version != 2 {
                return err("Not GLB version 2.0");
            }
            *x
        };

        offset += size_of::<Header>() as isize;
        let json_chunk = unsafe {
            let x: *const ChunkInfo = transmute(glb_ptr.offset(offset));
            offset += size_of::<ChunkInfo>() as isize;
            if (*x).length as usize > (glb.len() as isize - offset) as usize {
                return err("JSON chunkLength exceeds length of data");
            }
            if &(*x).kind != b"JSON" {
                return err("JSON chunkType is not `JSON`");
            }
            Slice {
                offset: offset as usize,
                length: (*x).length as usize,
            }
        };

        offset += json_chunk.length as isize;
        let blob_chunk = if glb.len() as isize - offset == 0 {
            None
        } else {
            unsafe {
                let x: *const ChunkInfo = transmute(glb_ptr.offset(offset));
                offset += size_of::<ChunkInfo>() as isize;
                if (*x).length as usize > (glb.len() as isize - offset) as usize {
                    return err("BIN chunkLength exceeds length of data");
                }
                if &(*x).kind != b"BIN\0" {
                    return err("BIN chunkType is not `BIN\0`");
                }
                Some(Slice {
                    offset: offset as usize,
                    length: (*x).length as usize,
                })
            }
        };

        Ok((header, json_chunk, blob_chunk))
    }
}

fn source_buffers<S: import::Source>(
    root: &Root,
    source: &S,
    blob: Option<Box<[u8]>>,
) -> Vec<data::Async<S>> {
    let json = root.as_json();
    let mut iter = json.buffers.iter();
    let mut buffers = vec![];
    if let Some(data) = blob {
        let future = Box::new(future::ok(data));
        // Skip the first buffer entry.
        let _ = iter.next();
        buffers.push(data::Async::full(future));
    }
    for entry in iter {
        let uri = entry.uri.as_ref().unwrap();
        let future = Box::new(source.source_external_data(uri));
        buffers.push(data::Async::full(future));
    }
    buffers
}

fn source_images<S: import::Source>(
    root: &Root,
    source: &S,
) -> Vec<AsyncImage<S>> {
    root.as_json().images
        .iter()
        .map(|entry| {
            let format = entry.mime_type.as_ref().map(|x| match x.0.as_str() {
                "image/jpeg" => Jpeg,
                "image/png" => Png,
                _ => unreachable!(),
            });
            if let Some(uri) = entry.uri.as_ref() {
                let future = Box::new(source.source_external_data(uri));
                AsyncImage::Owned {
                    data: data::Async::full(future),
                    format: format,
                }
            } else if let Some(index) = entry.buffer_view.as_ref() {
                let buffer_view = &root.as_json().buffer_views[index.value()];
                AsyncImage::Borrowed {
                    index: buffer_view.buffer.value(),
                    offset: buffer_view.byte_offset as usize,
                    len: buffer_view.byte_length as usize,
                    format: format.unwrap(),
                }
            } else {
                unreachable!()
            }
        })
        .collect()
}

fn decode_images(
    buffers: &[Data],
    images: Vec<EncodedImage>,
) -> ImageResult<Vec<DynamicImage>> {
    images
        .iter()
        .map(|entry| {
            match entry {
                &EncodedImage::Borrowed { index, offset, len, format } => {
                    let data = &buffers[index][offset..(offset + len)];
                    load_from_memory_with_format(data, format)
                },
                &EncodedImage::Owned { ref data, format: Some(format) } => {
                    load_from_memory_with_format(data, format)
                },
                &EncodedImage::Owned { ref data, format: None } => {
                    load_from_memory(data)
                },
            }
        })
        .collect()
}

fn validate<S: Source>(
    json: json::Root,
    strategy: config::ValidationStrategy,
    has_blob: bool,
) -> Box<Future<Item = Root, Error = import::Error<S>>> {
    use validation::{Error as Reason, Validate};
    Box::new(future::lazy(move || {
        let mut errs = vec![];
        match strategy {
            config::ValidationStrategy::Skip => {
                return Ok(Root::new(json));
            },
            config::ValidationStrategy::Minimal => {
                json.validate_minimally(
                    &json,
                    || json::Path::new(),
                    &mut |path, err| {
                        errs.push((path(), err));
                    },
                )
            },
            config::ValidationStrategy::Complete => {
                json.validate_completely(
                    &json,
                    || json::Path::new(),
                    &mut |path, err| {
                        errs.push((path(), err));
                    },
                )
            }
        }

        // Required for the `Minimal` and `Complete` validation strategies.
        for (index, buffer) in json.buffers.iter().enumerate() {
            let path = || {
                json::Path::new()
                    .field("buffers")
                    .index(index)
                    .field("uri")
            };
            match index {
                0 if has_blob => if buffer.uri.is_some() {
                    errs.push((path(), Reason::Missing));
                },
                _ if buffer.uri.is_none() => {
                    errs.push((path(), Reason::Missing));
                },
                _ => {},
            }
        }

        if errs.is_empty() {
            Ok(Root::new(json))
        } else {
            Err(import::Error::Validation(errs))
        }
    }))
}

/// Imports some glTF from the given data source.
pub fn import<S: import::Source>(
    data: Box<[u8]>,
    source: S,
    config: Config,
) -> Box<Future<Item = Gltf, Error = import::Error<S>>> {
    Box::new(future::lazy(move || {
        let glb = Glb(data);
        let (_, json_chunk, blob_chunk) = glb.split()?;
        let begin = json_chunk.offset;
        let end = begin + json_chunk.length;
        let json: json::Root = json::from_slice(&glb.0[begin..end])?;
        let blob = blob_chunk.map(|chunk| {
            glb.slice(chunk).to_vec().into_boxed_slice()
        });
        Ok((json, blob))
    })
        .and_then(move |(json, blob)| {
            let config = config;
            let has_blob = blob.is_some();
            validate(json, config.validation_strategy, has_blob)
                .join(future::ok(blob))
        })
        .and_then(move |(root, blob)| {
            let source = source;
            let buffers = source_buffers(&root, &source, blob);
            let images = source_images(&root, &source);
            future::ok(root)
                .join3(
                    future::join_all(buffers),
                    future::join_all(images),
                )
        })
        .and_then(|(root, buffers, images)| {
            let decoded_images = decode_images(&buffers, images)?;
            Ok((root, buffers, decoded_images))
        })
        .and_then(|(root, buffers, images)| {
            Ok(Gltf::new(root, buffers, images))
        }))
}

<|endoftext|>


<commit_before>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::sstable::*;
use super::*;
use client::AsyncClient;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
const LEVEL_DIFF_MULTIPLIER: usize = 10;
const LEVEL_M: usize = super::btree::NUM_KEYS;
const LEVEL_1: usize = LEVEL_M * LEVEL_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<SSLevelTree>>;

macro_rules! with_levels {
    ($($sym:ident, $level_size:ident;)*) => {
        $(
            type $sym = [EntryKey; $level_size];
            impl_sspage_slice!($sym, EntryKey, $level_size);
        )*

        fn init_lsm_level_trees(neb_client: &Arc<AsyncClient>) -> (LevelTrees, Vec<usize>, Vec<usize>) {
            let mut trees = LevelTrees::new();
            let mut max_elements_list = Vec::new();
            let mut max_elements = LEVEL_M_MAX_ELEMENTS_COUNT;
            let mut page_sizes = Vec::new();
            $(
                max_elements *= LEVEL_ELEMENTS_MULTIPLIER;
                trees.push(box LevelTree::<$sym>::new(neb_client));
                max_elements_list.push(max_elements);
                page_sizes.push($level_size);
                const_assert!($level_size * KEY_SIZE <= MAX_SEGMENT_SIZE);
            )*
            return (trees, max_elements_list, page_sizes);
        }
    };
}

with_levels! {
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    L4, LEVEL_4;
}

pub struct LSMTree {
    pub level_m: BPlusTree,
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
    page_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        let (trees, max_sizes, page_sizes) = init_lsm_level_trees(neb_client);
        let lsm_tree = Arc::new(LSMTree {
            level_m: BPlusTree::new(neb_client),
            trees,
            max_sizes,
            page_sizes,
        });
        let tree_clone = lsm_tree.clone();
        thread::spawn(move || {
            tree_clone.sentinel();
        });
        lsm_tree
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.level_m.insert(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        let m_deleted = self.level_m.remove(&key);
        let levels_deleted = self
            .trees
            .iter()
            .map(|tree| tree.mark_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d);
        m_deleted || levels_deleted
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![box self.level_m.seek(&key, ordering)];
        for tree in &self.trees {
            cursors.push(tree.seek(&key, ordering));
        }
        return LSMTreeCursor::new(
            cursors,
            self.trees.iter().map(|t| t.tombstones()).collect_vec(),
        );
    }

    fn sentinel(&self) {
        self.check_and_merge(
            LEVEL_M_MAX_ELEMENTS_COUNT,
            self.page_sizes[0],
            &mut BTreeSet::new(),
            &self.level_m,
            &*self.trees[0],
        );
        for i in 0..self.trees.len() - 1 {
            let upper = &*self.trees[i];
            let lower = &*self.trees[i + 1];
            let upper_tombstons_ref = upper.tombstones();
            let mut upper_tombstones = upper_tombstons_ref.write();
            self.check_and_merge(
                self.max_sizes[i],
                self.page_sizes[i + 1],
                &mut *upper_tombstones,
                upper,
                lower,
            );
        }
        thread::sleep(Duration::from_millis(100));
    }

    fn check_and_merge<U, L>(
        &self,
        max_elements: usize,
        upper_page_size: usize,
        upper_tombstones: &mut TombstonesInner,
        upper_level: &U,
        lower_level: &L,
    ) where
        U: MergeableTree + ?Sized,
        L: SSLevelTree + ?Sized,
    {
        let elements = upper_level.elements();
        if elements > max_elements {
            debug!(
                "upper level have elements {} exceeds {}, start level merging",
                elements, max_elements
            );
            let entries_to_merge = upper_page_size;
            let guard = upper_level.prepare_level_merge();
            let mut pages = vec![guard.last_page()];
            let mut collected_entries = pages.last().unwrap().keys().len();
            while collected_entries < entries_to_merge {
                let next_page = pages.last().unwrap().next();
                collected_entries += next_page.keys().len();
                pages.push(next_page);
            }
            let mut entries = vec![];
            for page in &pages {
                entries.append(&mut Vec::from(page.keys()));
            }
            debug!("going to merge {} entries", entries.len());
            lower_level.merge(entries.as_mut_slice(), upper_tombstones);
            guard.remove_pages(pages.iter().map(|p| p.keys()).collect_vec().as_slice())
        }
    }

    pub fn len(&self) -> usize {
        let mem_len = self.level_m.len();
        let levels_len_sum = self.trees.iter().map(|tree| tree.len()).sum::<usize>();
        return mem_len + levels_len_sum;
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
    tombstones: Vec<Arc<Tombstones>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>, tombstones: Vec<Arc<Tombstones>>) -> Self {
        LSMTreeCursor {
            tombstones,
            level_cursors: cursors,
        }
    }

    fn next_candidate(&mut self) -> (bool, usize) {
        // TODO: redo this part
        let min_tree = self
            .level_cursors
            .iter()
            .enumerate()
            .map(|(i, cursor)| (i, cursor.current()))
            .filter_map(|(i, current)| current.map(|current_val| (i, current_val)))
            .min_by_key(|(i, val)| val.clone())
            .map(|(id, _)| id);
        if let Some(id) = min_tree {
            let min_has_next = self.level_cursors[id].next();
            if !min_has_next {
                let next = self
                    .level_cursors
                    .iter()
                    .enumerate()
                    .filter_map(|(level, cursor)| cursor.current().map(|key| (id, key)))
                    .min_by_key(|(level, key)| key.clone());
                if let Some((id, _)) = next {
                    return (true, id);
                }
            } else {
                return (true, id);
            }
        }
        return (false, 0);
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        loop {
            let (has_next, candidate_level) = self.next_candidate();
            if !has_next {
                break;
            }
            if candidate_level == 0 {
                return has_next;
            }
            let candidate_key = self.level_cursors[candidate_level].current().unwrap();
            if self.tombstones[candidate_level + 1]
                .read()
                .contains(candidate_key)
            {
                // marked in tombstone, skip
                continue;
            }
            return true;
        }
        return false;
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5600");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 4 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        thread::spawn(move || {
            loop {
                thread::sleep(Duration::from_secs(10));
                let tree_len = tree_clone.len();
                debug!(
                    "LSM-Tree now have {}/{} elements, total {:.2}%",
                    tree_len,
                    num,
                    tree_len as f32 / num as f32 * 100.0
                );
            }
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        if tree.len() > LEVEL_M_MAX_ELEMENTS_COUNT {
            debug!("Sleep 5 minute for level merge");
            thread::sleep(Duration::from_secs(5 * 60));
        }
        debug!("Start validations");
        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let mut rng = rand::rngs::OsRng::new().unwrap();
            let die_range = Uniform::new_inclusive(1, 6);
            let mut roll_die = rng.sample_iter(&die_range);
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
            if roll_die.next().unwrap() == 6 {
                for j in i..num {
                    let id = Id::new(0, j);
                    let key_slice = u64_to_slice(j);
                    let mut key = SmallVec::from_slice(&key_slice);
                    key_with_id(&mut key, &id);
                    assert_eq!(cursor.current(), Some(&key), "{}/{}", i, j);
                    assert_eq!(cursor.next(), j != num - 1, "{}/{}", i, j);
                }
            }
        });
    }
}
<commit_msg>not parallel for validation<commit_after>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::sstable::*;
use super::*;
use client::AsyncClient;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
const LEVEL_DIFF_MULTIPLIER: usize = 10;
const LEVEL_M: usize = super::btree::NUM_KEYS;
const LEVEL_1: usize = LEVEL_M * LEVEL_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<SSLevelTree>>;

macro_rules! with_levels {
    ($($sym:ident, $level_size:ident;)*) => {
        $(
            type $sym = [EntryKey; $level_size];
            impl_sspage_slice!($sym, EntryKey, $level_size);
        )*

        fn init_lsm_level_trees(neb_client: &Arc<AsyncClient>) -> (LevelTrees, Vec<usize>, Vec<usize>) {
            let mut trees = LevelTrees::new();
            let mut max_elements_list = Vec::new();
            let mut max_elements = LEVEL_M_MAX_ELEMENTS_COUNT;
            let mut page_sizes = Vec::new();
            $(
                max_elements *= LEVEL_ELEMENTS_MULTIPLIER;
                trees.push(box LevelTree::<$sym>::new(neb_client));
                max_elements_list.push(max_elements);
                page_sizes.push($level_size);
                const_assert!($level_size * KEY_SIZE <= MAX_SEGMENT_SIZE);
            )*
            return (trees, max_elements_list, page_sizes);
        }
    };
}

with_levels! {
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    L4, LEVEL_4;
}

pub struct LSMTree {
    pub level_m: BPlusTree,
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
    page_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        let (trees, max_sizes, page_sizes) = init_lsm_level_trees(neb_client);
        let lsm_tree = Arc::new(LSMTree {
            level_m: BPlusTree::new(neb_client),
            trees,
            max_sizes,
            page_sizes,
        });
        let tree_clone = lsm_tree.clone();
        thread::spawn(move || {
            tree_clone.sentinel();
        });
        lsm_tree
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.level_m.insert(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        let m_deleted = self.level_m.remove(&key);
        let levels_deleted = self
            .trees
            .iter()
            .map(|tree| tree.mark_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d);
        m_deleted || levels_deleted
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![box self.level_m.seek(&key, ordering)];
        for tree in &self.trees {
            cursors.push(tree.seek(&key, ordering));
        }
        return LSMTreeCursor::new(
            cursors,
            self.trees.iter().map(|t| t.tombstones()).collect_vec(),
        );
    }

    fn sentinel(&self) {
        self.check_and_merge(
            LEVEL_M_MAX_ELEMENTS_COUNT,
            self.page_sizes[0],
            &mut BTreeSet::new(),
            &self.level_m,
            &*self.trees[0],
        );
        for i in 0..self.trees.len() - 1 {
            let upper = &*self.trees[i];
            let lower = &*self.trees[i + 1];
            let upper_tombstons_ref = upper.tombstones();
            let mut upper_tombstones = upper_tombstons_ref.write();
            self.check_and_merge(
                self.max_sizes[i],
                self.page_sizes[i + 1],
                &mut *upper_tombstones,
                upper,
                lower,
            );
        }
        thread::sleep(Duration::from_millis(100));
    }

    fn check_and_merge<U, L>(
        &self,
        max_elements: usize,
        upper_page_size: usize,
        upper_tombstones: &mut TombstonesInner,
        upper_level: &U,
        lower_level: &L,
    ) where
        U: MergeableTree + ?Sized,
        L: SSLevelTree + ?Sized,
    {
        let elements = upper_level.elements();
        if elements > max_elements {
            debug!(
                "upper level have elements {} exceeds {}, start level merging",
                elements, max_elements
            );
            let entries_to_merge = upper_page_size;
            let guard = upper_level.prepare_level_merge();
            let mut pages = vec![guard.last_page()];
            let mut collected_entries = pages.last().unwrap().keys().len();
            while collected_entries < entries_to_merge {
                let next_page = pages.last().unwrap().next();
                collected_entries += next_page.keys().len();
                pages.push(next_page);
            }
            let mut entries = vec![];
            for page in &pages {
                entries.append(&mut Vec::from(page.keys()));
            }
            debug!("going to merge {} entries", entries.len());
            lower_level.merge(entries.as_mut_slice(), upper_tombstones);
            guard.remove_pages(pages.iter().map(|p| p.keys()).collect_vec().as_slice())
        }
    }

    pub fn len(&self) -> usize {
        let mem_len = self.level_m.len();
        let levels_len_sum = self.trees.iter().map(|tree| tree.len()).sum::<usize>();
        return mem_len + levels_len_sum;
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
    tombstones: Vec<Arc<Tombstones>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>, tombstones: Vec<Arc<Tombstones>>) -> Self {
        LSMTreeCursor {
            tombstones,
            level_cursors: cursors,
        }
    }

    fn next_candidate(&mut self) -> (bool, usize) {
        // TODO: redo this part
        let min_tree = self
            .level_cursors
            .iter()
            .enumerate()
            .map(|(i, cursor)| (i, cursor.current()))
            .filter_map(|(i, current)| current.map(|current_val| (i, current_val)))
            .min_by_key(|(i, val)| val.clone())
            .map(|(id, _)| id);
        if let Some(id) = min_tree {
            let min_has_next = self.level_cursors[id].next();
            if !min_has_next {
                let next = self
                    .level_cursors
                    .iter()
                    .enumerate()
                    .filter_map(|(level, cursor)| cursor.current().map(|key| (id, key)))
                    .min_by_key(|(level, key)| key.clone());
                if let Some((id, _)) = next {
                    return (true, id);
                }
            } else {
                return (true, id);
            }
        }
        return (false, 0);
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        loop {
            let (has_next, candidate_level) = self.next_candidate();
            if !has_next {
                break;
            }
            if candidate_level == 0 {
                return has_next;
            }
            let candidate_key = self.level_cursors[candidate_level].current().unwrap();
            if self.tombstones[candidate_level + 1]
                .read()
                .contains(candidate_key)
            {
                // marked in tombstone, skip
                continue;
            }
            return true;
        }
        return false;
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5600");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 4 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(super::get_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        thread::spawn(move || {
            loop {
                thread::sleep(Duration::from_secs(10));
                let tree_len = tree_clone.len();
                debug!(
                    "LSM-Tree now have {}/{} elements, total {:.2}%",
                    tree_len,
                    num,
                    tree_len as f32 / num as f32 * 100.0
                );
            }
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        if tree.len() > LEVEL_M_MAX_ELEMENTS_COUNT {
            debug!("Sleep 5 minute for level merge");
            thread::sleep(Duration::from_secs(5 * 60));
        }
        debug!("Start validations");
        (0..num).collect::<Vec<_>>().iter().for_each(|i| {
            let mut rng = rand::rngs::OsRng::new().unwrap();
            let die_range = Uniform::new_inclusive(1, 6);
            let mut roll_die = rng.sample_iter(&die_range);
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
            if roll_die.next().unwrap() == 6 {
                for j in i..num {
                    let id = Id::new(0, j);
                    let key_slice = u64_to_slice(j);
                    let mut key = SmallVec::from_slice(&key_slice);
                    key_with_id(&mut key, &id);
                    assert_eq!(cursor.current(), Some(&key), "{}/{}", i, j);
                    assert_eq!(cursor.next(), j != num - 1, "{}/{}", i, j);
                }
            }
        });
    }
}
<|endoftext|>


<commit_before>use itertools::Itertools;

fn day10(input: &str) -> (usize, usize) {

    let mut i = 1;
    let mut next = String::from(input);
    let mut part1 = 0;
    let mut part2 = 0;
    loop {
        next = next.chars()
                   .group_by(|&x| x)
                   .map(|(n, l)| {
                       let mut s = l.len().to_string();
                       s.push(n);
                       s
                   })
                   .collect::<String>();
        if i == 40 {
            part1 = next.len();
        }
        if i == 50 {
            part2 = next.len();
        }
        i += 1;
        if i == 51 {
            break;
        }
    }
    return (part1, part2);
}

pub fn main() {
    let (part1, part2) = day10("3113322113");
    println!("Part 1: {}\nPart 2: {}", part1, part2);
}

#[test]
fn test() {
    let (part1, part2) = day10("3113322113");
    assert_eq!(part1, 329356);
    assert_eq!(part2, 4666278);
}
<commit_msg>make rust day10 faster<commit_after>use itertools::Itertools;

fn day10(input: &str) -> (usize, usize) {
    let mut i = 1;
    let mut next = String::from(input);
    let mut part1 = 0;
    let mut part2 = 0;
    loop {
        let mut buf = String::new();
        for (number, list) in next.chars().group_by(|&x| x) {
            buf.push_str(&list.len().to_string());
            buf.push(number);
        }
        next = buf;
        if i == 40 {
            part1 = next.len();
        }
        if i == 50 {
            part2 = next.len();
        }
        i += 1;
        if i == 51 {
            break;
        }
    }
    return (part1, part2);
}

pub fn main() {
    let (part1, part2) = day10("3113322113");
    println!("Part 1: {}\nPart 2: {}", part1, part2);
}

#[test]
fn test() {
    let (part1, part2) = day10("3113322113");
    assert_eq!(part1, 329356);
    assert_eq!(part2, 4666278);
}
<|endoftext|>


<commit_before>use std::str::Chars;
use std::iter::Peekable;
use std::fmt::{Show,Formatter};
use std::fmt;

pub struct TokenValue<'a> {
  token: Token,
  value: &'a str
}

/// Lexer
pub struct Lexer<'a> {
  line: int,
  column: int,
  pos: int,
  iter: Peekable<char, Chars<'a>>,
  curr_ch: Option<char>,
  done: bool
}

/// Lexer tokens.
pub enum Token {
  PLUS,
  MINUS,
  INTEGER, // Any type of integer
  IDEN, // Identifier
  LPAREN,
  RPAREN,
  COLON
}

impl<'a> TokenValue<'a> {
  pub fn new(tok: Token, val: &'a str) -> TokenValue<'a> {
    TokenValue {
      token: tok,
      value: val
    }
  }
}

impl Eq for Token {
  fn eq(&self, other: &Token) -> bool {
    match *self {
      PLUS => match *other { PLUS => true, _ => false },
      MINUS => match *other { MINUS => true, _ => false },
      INTEGER => match *other { INTEGER => true, _ => false },
      IDEN => match *other { IDEN => true, _ => false },
      LPAREN => match *other { LPAREN => true, _ => false },
      RPAREN => match *other { RPAREN => true, _ => false },
      COLON => match *other { COLON => true, _ => false }
    }
  }
}

impl Show for Token {
  fn fmt(&self, f: &mut Formatter) -> fmt::Result {
    let c = match *self {
      PLUS => &"+",
      MINUS => &"-",
      INTEGER => &"integer",
      IDEN => &"iden",
      LPAREN => &"(",
      RPAREN => &")",
      COLON => ":"
    };

    write!(f.buf, "({})", c)
  }
}

impl<'a> Lexer<'a> {
  pub fn new(input: &'a str) -> Lexer<'a> {
    let it = input.chars().peekable();
    Lexer {
      line: 0,
      column: 0,
      pos: 0,
      iter: it,
      curr_ch: None,
      done: false
    }
  }

  pub fn bump(&mut self) {
    self.curr_ch = self.iter.next();
  }

  pub fn nextch_is(&mut self, ch: char) -> bool {
    self.bump();
    self.curr_ch.unwrap() == ch
  }

  pub fn peekch_is(&mut self, ch: char) -> bool {
    *self.iter.peek().unwrap() == ch
  }

  pub fn next_token(&mut self) -> Option<TokenValue<'a>> {
    return Some(TokenValue::new(LPAREN, &"9"));
    for c in self.iter {
      match c {
        //'(' => return Some(TokenValue::new(LPAREN, 9)),
        // ')' => return Some(TokenValue::new(RPAREN, ')')),
        // '+' => return Some(TokenValue::new(PLUS, '+')),
        // '-' => return Some(TokenValue::new(MINUS, '-')),
        // ':' => return Some(TokenValue::new(COLON, ':')),
        // '9' => return Some(TokenValue::new(INTEGER, 9)),
        // '8' => return Some(TokenValue::new(INTEGER, 8)),
        // '7' => return Some(TokenValue::new(INTEGER, 7)),
        // '6' => return Some(TokenValue::new(INTEGER, 6)),
        // '5' => return Some(TokenValue::new(INTEGER, 5)),
        // '4' => return Some(TokenValue::new(INTEGER, 4)),
        // '3' => return Some(TokenValue::new(INTEGER, 3)),
        // '2' => return Some(TokenValue::new(INTEGER, 2)),
        // '1' => return Some(TokenValue::new(INTEGER, 1)),
        // '0' => return Some(TokenValue::new(INTEGER, 0)),
        _ => return None
      }
    }

    None
  }

}

#[cfg(test)]
mod test {
  use super::Lexer;
  use super::LPAREN;
  use super::TokenValue;

  #[test]
  fn bump() {
    let mut lex = Lexer::new(&"Hello World");
    lex.bump();
    assert_eq!(lex.curr_ch.unwrap(), 'H');
  }

  #[test]
  fn nextch() {
    let mut lex = Lexer::new(&"Woot");
    assert!(lex.nextch_is('W'));
  }

  #[test]
  fn peekable() {
    let mut lex = Lexer::new(&"Haha");
    lex.bump();
    assert!(lex.peekch_is('a'));
  }

  #[test]
  fn next_token() {
    // Should spit out two tokens (LPAREN,RPAREN)
    let mut lex = Lexer::new(&"()");
    assert!(lex.next_token().unwrap().value == &"9"); //TokenValue::new(LPAREN, "("));
  }

}<commit_msg>uncommented<commit_after>use std::str::Chars;
use std::iter::Peekable;
use std::fmt::{Show,Formatter};
use std::fmt;

pub struct TokenValue<'a> {
  token: Token,
  value: &'a str
}

/// Lexer
pub struct Lexer<'a> {
  line: int,
  column: int,
  pos: int,
  iter: Peekable<char, Chars<'a>>,
  curr_ch: Option<char>,
  done: bool
}

/// Lexer tokens.
pub enum Token {
  PLUS,
  MINUS,
  INTEGER, // Any type of integer
  IDEN, // Identifier
  LPAREN,
  RPAREN,
  COLON
}

impl<'a> TokenValue<'a> {
  pub fn new(tok: Token, val: &'a str) -> TokenValue<'a> {
    TokenValue {
      token: tok,
      value: val
    }
  }
}

impl Eq for Token {
  fn eq(&self, other: &Token) -> bool {
    match *self {
      PLUS => match *other { PLUS => true, _ => false },
      MINUS => match *other { MINUS => true, _ => false },
      INTEGER => match *other { INTEGER => true, _ => false },
      IDEN => match *other { IDEN => true, _ => false },
      LPAREN => match *other { LPAREN => true, _ => false },
      RPAREN => match *other { RPAREN => true, _ => false },
      COLON => match *other { COLON => true, _ => false }
    }
  }
}

impl Show for Token {
  fn fmt(&self, f: &mut Formatter) -> fmt::Result {
    let c = match *self {
      PLUS => &"+",
      MINUS => &"-",
      INTEGER => &"integer",
      IDEN => &"iden",
      LPAREN => &"(",
      RPAREN => &")",
      COLON => ":"
    };

    write!(f.buf, "({})", c)
  }
}

impl<'a> Lexer<'a> {
  pub fn new(input: &'a str) -> Lexer<'a> {
    let it = input.chars().peekable();
    Lexer {
      line: 0,
      column: 0,
      pos: 0,
      iter: it,
      curr_ch: None,
      done: false
    }
  }

  pub fn bump(&mut self) {
    self.curr_ch = self.iter.next();
  }

  pub fn nextch_is(&mut self, ch: char) -> bool {
    self.bump();
    self.curr_ch.unwrap() == ch
  }

  pub fn peekch_is(&mut self, ch: char) -> bool {
    *self.iter.peek().unwrap() == ch
  }

  pub fn next_token(&mut self) -> Option<TokenValue<'a>> {

    for c in self.iter {
      match c {
        '(' => return Some(TokenValue::new(LPAREN, &"(")),
        ')' => return Some(TokenValue::new(RPAREN, &")")),
        '+' => return Some(TokenValue::new(PLUS, &"+")),
        '-' => return Some(TokenValue::new(MINUS, &"-")),
        ':' => return Some(TokenValue::new(COLON, &":")),
        '9' => return Some(TokenValue::new(INTEGER, &"9")),
        '8' => return Some(TokenValue::new(INTEGER, &"8")),
        '7' => return Some(TokenValue::new(INTEGER, &"7")),
        '6' => return Some(TokenValue::new(INTEGER, &"6")),
        '5' => return Some(TokenValue::new(INTEGER, &"5")),
        '4' => return Some(TokenValue::new(INTEGER, &"4")),
        '3' => return Some(TokenValue::new(INTEGER, &"3")),
        '2' => return Some(TokenValue::new(INTEGER, &"2")),
        '1' => return Some(TokenValue::new(INTEGER, &"1")),
        '0' => return Some(TokenValue::new(INTEGER, &"0")),
        _ => return None
      }
    }

    None
  }

}

#[cfg(test)]
mod test {
  use super::Lexer;
  use super::LPAREN;
  use super::TokenValue;

  #[test]
  fn bump() {
    let mut lex = Lexer::new(&"Hello World");
    lex.bump();
    assert_eq!(lex.curr_ch.unwrap(), 'H');
  }

  #[test]
  fn nextch() {
    let mut lex = Lexer::new(&"Woot");
    assert!(lex.nextch_is('W'));
  }

  #[test]
  fn peekable() {
    let mut lex = Lexer::new(&"Haha");
    lex.bump();
    assert!(lex.peekch_is('a'));
  }

  #[test]
  fn next_token() {
    // Should spit out two tokens (LPAREN,RPAREN)
    let mut lex = Lexer::new(&"()");
    assert!(lex.next_token().unwrap().value == &"("); //TokenValue::new(LPAREN, "("));
  }

}<|endoftext|>


<commit_before>use libc;
use glib::translate::*;
use itertools::Itertools;
use std::borrow::Cow;

#[repr(C)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum XmlSpace {
    Default,
    Preserve
}

/// Implements `xml:space` handling per the SVG spec
///
/// Normalizes a string as it comes out of the XML parser's handler
/// for character data according to the SVG rules in
/// https://www.w3.org/TR/SVG/text.html#WhiteSpace
pub fn xml_space_normalize(mode: XmlSpace, s: &str) -> String {
    match mode {
        XmlSpace::Default => normalize_default(s),
        XmlSpace::Preserve => normalize_preserve(s)
    }
}

// From https://www.w3.org/TR/SVG/text.html#WhiteSpace
//
// When xml:space="default", the SVG user agent will do the following
// using a copy of the original character data content. First, it will
// remove all newline characters. Then it will convert all tab
// characters into space characters. Then, it will strip off all
// leading and trailing space characters. Then, all contiguous space
// characters will be consolidated.
fn normalize_default<'a, S: Into<Cow<'a, str>>>(s: S) -> String {
    let s = s.into();
    s.trim()
        .chars()
        .filter(|ch| *ch != '\n')
        .map(|ch| {
            match ch {
                '\t' => ' ',
                c => c
            }
        })
        .coalesce(|current, next| {
            if current == ' ' && next == ' ' {
                Ok(' ')
            } else {
                Err((current, next))
            }
        })
        .collect::<String>()
}

// From https://www.w3.org/TR/SVG/text.html#WhiteSpace
//
// When xml:space="preserve", the SVG user agent will do the following
// using a copy of the original character data content. It will
// convert all newline and tab characters into space characters. Then,
// it will draw all space characters, including leading, trailing and
// multiple contiguous space characters. Thus, when drawn with
// xml:space="preserve", the string "a   b" (three spaces between "a"
// and "b") will produce a larger separation between "a" and "b" than
// "a b" (one space between "a" and "b").
fn normalize_preserve(s: &str) -> String {
    s.chars()
        .map(|ch| {
            match ch {
                '\n' | '\t' => ' ',

                c => c
            }
        })
        .collect()
}

#[no_mangle]
pub extern fn rsvg_xml_space_normalize(mode: XmlSpace, s: *const libc::c_char) -> *const libc::c_char {
    let rs = unsafe { String::from_glib_none (s) };

    xml_space_normalize(mode, &rs).to_glib_full()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn xml_space_default() {
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\n    WS example\n    indented lines\n  "),
                   "WS example indented lines");
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\n  \t  \tWS \t\t\texample\n  \t  indented liines\t\t  \n  "),
                   "WS example indented liines");

        assert_eq!(xml_space_normalize(XmlSpace::Default, "\nWS example\nnon-indented lines\n  "),
                   "WS examplenon-indented lines");
    }

    #[test]
    fn xml_space_preserve() {
        assert_eq!(xml_space_normalize(XmlSpace::Preserve, "\n    WS example\n    indented lines\n  "),
                   "     WS example     indented lines   ");
        assert_eq!(xml_space_normalize(XmlSpace::Preserve, "\n  \t  \tWS \t\t\texample\n  \t  indented lines\t\t  \n  "),
                   "       WS    example      indented lines       ");

    }
}
<commit_msg>Added extra test cases.<commit_after>use libc;
use glib::translate::*;
use itertools::Itertools;
use std::borrow::Cow;

#[repr(C)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum XmlSpace {
    Default,
    Preserve
}

/// Implements `xml:space` handling per the SVG spec
///
/// Normalizes a string as it comes out of the XML parser's handler
/// for character data according to the SVG rules in
/// https://www.w3.org/TR/SVG/text.html#WhiteSpace
pub fn xml_space_normalize(mode: XmlSpace, s: &str) -> String {
    match mode {
        XmlSpace::Default => normalize_default(s),
        XmlSpace::Preserve => normalize_preserve(s)
    }
}

// From https://www.w3.org/TR/SVG/text.html#WhiteSpace
//
// When xml:space="default", the SVG user agent will do the following
// using a copy of the original character data content. First, it will
// remove all newline characters. Then it will convert all tab
// characters into space characters. Then, it will strip off all
// leading and trailing space characters. Then, all contiguous space
// characters will be consolidated.
fn normalize_default<'a, S: Into<Cow<'a, str>>>(s: S) -> String {
    let s = s.into();
    s.trim()
        .chars()
        .filter(|ch| *ch != '\n')
        .map(|ch| match ch {
            '\t' => ' ',
            c => c,
        })
        .coalesce(|current, next| match (current, next) {
            (' ', ' ') | (' ', '\t') => Ok(' '),
            (_, _) => Err((current, next)),
        })
        .collect::<String>()
}

// From https://www.w3.org/TR/SVG/text.html#WhiteSpace
//
// When xml:space="preserve", the SVG user agent will do the following
// using a copy of the original character data content. It will
// convert all newline and tab characters into space characters. Then,
// it will draw all space characters, including leading, trailing and
// multiple contiguous space characters. Thus, when drawn with
// xml:space="preserve", the string "a   b" (three spaces between "a"
// and "b") will produce a larger separation between "a" and "b" than
// "a b" (one space between "a" and "b").
fn normalize_preserve(s: &str) -> String {
    s.chars()
        .map(|ch| {
            match ch {
                '\n' | '\t' => ' ',

                c => c
            }
        })
        .collect()
}

#[no_mangle]
pub extern fn rsvg_xml_space_normalize(mode: XmlSpace, s: *const libc::c_char) -> *const libc::c_char {
    let rs = unsafe { String::from_glib_none (s) };

    xml_space_normalize(mode, &rs).to_glib_full()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn xml_space_default() {
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\n    WS example\n    indented lines\n  "),
                   "WS example indented lines");
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\n  \t  \tWS \t\t\texample\n  \t  indented lines\t\t  \n  "),
                   "WS example indented lines");
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\n  \t  \tWS \t\t\texample\n  \t  duplicate letters\t\t  \n  "),
                   "WS example duplicate letters");
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\nWS example\nnon-indented lines\n  "),
                   "WS examplenon-indented lines");
        assert_eq!(xml_space_normalize(XmlSpace::Default, "\nWS example\tnon-indented lines\n  "),
                   "WS example non-indented lines");
    }

    #[test]
    fn xml_space_preserve() {
        assert_eq!(xml_space_normalize(XmlSpace::Preserve, "\n    WS example\n    indented lines\n  "),
                   "     WS example     indented lines   ");
        assert_eq!(xml_space_normalize(XmlSpace::Preserve, "\n  \t  \tWS \t\t\texample\n  \t  indented lines\t\t  \n  "),
                   "       WS    example      indented lines       ");
        assert_eq!(xml_space_normalize(XmlSpace::Preserve, "\n  \t  \tWS \t\t\texample\n  \t  duplicate letters\t\t  \n  "),
                   "       WS    example      duplicate letters       ");
    }
}
<|endoftext|>


<commit_before>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use chrono::naive::datetime::NaiveDateTime;

use libimagstore::storeid::StoreId;

use datepath::accuracy::Accuracy;
use datepath::format::Format;
use datepath::result::Result;

pub struct DatePathCompiler {
    accuracy : Accuracy,
    format   : Format,
}

impl DatePathCompiler {

    pub fn new(accuracy: Accuracy, format: Format) -> DatePathCompiler {
        DatePathCompiler {
            accuracy : accuracy,
            format   : format,
        }
    }

    /// Compile a NaiveDateTime object into a StoreId object.
    ///
    /// # More information
    ///
    /// See the documentations of the `Format` and the `Accuracy` types as well.
    ///
    /// # Warnings
    ///
    /// This does _not_ guarantee that the StoreId can be created, is not yet in the store or
    /// anything else. Overall, this is just a `spec->path` compiler which is really stupid.
    ///
    /// # Return value
    ///
    /// The `StoreId` object on success.
    ///
    pub fn compile(&self, datetime: &NaiveDateTime) -> Result<StoreId> {
        unimplemented!()
    }

}
<commit_msg>Add compiler impl<commit_after>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::path::PathBuf;

use chrono::naive::datetime::NaiveDateTime;
use chrono::Datelike;
use chrono::Timelike;

use libimagstore::storeid::StoreId;

use datepath::accuracy::Accuracy;
use datepath::format::Format;
use datepath::result::Result;
use datepath::error::DatePathCompilerErrorKind as DPCEK;
use datepath::error::MapErrInto;

pub struct DatePathCompiler {
    accuracy : Accuracy,
    format   : Format,
}

impl DatePathCompiler {

    pub fn new(accuracy: Accuracy, format: Format) -> DatePathCompiler {
        DatePathCompiler {
            accuracy : accuracy,
            format   : format,
        }
    }

    /// Compile a NaiveDateTime object into a StoreId object.
    ///
    /// # More information
    ///
    /// See the documentations of the `Format` and the `Accuracy` types as well.
    ///
    /// # Warnings
    ///
    /// This does _not_ guarantee that the StoreId can be created, is not yet in the store or
    /// anything else. Overall, this is just a `spec->path` compiler which is really stupid.
    ///
    /// # Return value
    ///
    /// The `StoreId` object on success.
    ///
    pub fn compile(&self, module_name: &str, datetime: &NaiveDateTime) -> Result<StoreId> {
        let mut s = format!("{}/", module_name);

        if self.accuracy.has_year_accuracy() /* always true */ {
            s.push_str(&format!("{}", datetime.year()));
        }

        if self.accuracy.has_month_accuracy() {
            match self.format {
                Format::ElementIsFolder
                    | Format::DaysAreFolder
                    | Format::MonthIsFolder
                    | Format::YearIsFolder
                    => s.push_str(&format!("/{}", datetime.month())),
            }
        }

        if self.accuracy.has_day_accuracy() {
            match self.format {
                Format::ElementIsFolder
                    | Format::DaysAreFolder
                    | Format::MonthIsFolder
                    => s.push_str(&format!("/{}", datetime.day())),
                Format::YearIsFolder
                    => s.push_str(&format!("-{}", datetime.day())),
            }
        }

        if self.accuracy.has_hour_accuracy() {
            match self.format {
                Format::ElementIsFolder
                    | Format::DaysAreFolder
                    => s.push_str(&format!("/{}", datetime.hour())),
                Format::YearIsFolder
                    | Format::MonthIsFolder
                    => s.push_str(&format!("-{}", datetime.hour())),
            }
        }

        if self.accuracy.has_minute_accuracy() {
            match self.format {
                Format::ElementIsFolder
                    => s.push_str(&format!("/{}", datetime.minute())),
                Format::YearIsFolder
                    | Format::MonthIsFolder
                    | Format::DaysAreFolder
                    => s.push_str(&format!("-{}", datetime.minute())),
            }
        }

        if self.accuracy.has_second_accuracy() {
            match self.format {
                Format::ElementIsFolder
                    => s.push_str(&format!("/{}", datetime.second())),
                Format::YearIsFolder
                    | Format::MonthIsFolder
                    | Format::DaysAreFolder
                    => s.push_str(&format!("-{}", datetime.second())),
            }
        }

        StoreId::new_baseless(PathBuf::from(s))
            .map_err_into(DPCEK::StoreIdBuildFailed)
    }

}
<|endoftext|>


<commit_before>use std::path::PathBuf;
use std::path::Path;
use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use toml::Value;

use libimagerror::trace::trace_error;
use libimagstore::hook::Hook;
use libimagstore::hook::accessor::StoreIdAccessor;
use libimagstore::hook::accessor::{HookDataAccessor, HookDataAccessorProvider};
use libimagstore::hook::position::HookPosition;
use libimagstore::hook::result::HookResult;
use libimagstore::storeid::StoreId;
use libimagutil::debug_result::*;

use vcs::git::error::GitHookErrorKind as GHEK;
use vcs::git::error::MapErrInto;
use vcs::git::runtime::Runtime as GRuntime;

/// The `UpdateHook` type
///
/// Represents a hook which is executed whenever a entry in the store is updated (written to disk).
///
/// # Time of execution
///
/// This hook is executed _after_ the store operation succeeded, so _after_ the file is written to
/// disk.
pub struct UpdateHook {
    storepath: PathBuf,

    runtime: GRuntime,

    position: HookPosition,
}

impl UpdateHook {

    pub fn new(storepath: PathBuf, p: HookPosition) -> UpdateHook {
        UpdateHook {
            runtime: GRuntime::new(&storepath),
            storepath: storepath,
            position: p,
        }
    }

}

impl Debug for UpdateHook {
    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "UpdateHook(storepath={:?}, repository={}, pos={:?}, cfg={:?})",
               self.storepath,
               (if self.runtime.has_repository() { "Some(_)" } else { "None" }),
               self.position,
               self.runtime.has_config())
    }
}

impl Hook for UpdateHook {

    fn name(&self) -> &'static str {
        "stdhook_git_update"
    }

    /// Set the configuration of the hook. See
    /// `libimagstorestdhook::vcs::git::runtime::Runtime::set_config()`.
    ///
    /// This function traces the error (using `trace_error()`) that
    /// `libimagstorestdhook::vcs::git::runtime::Runtime::set_config()`
    /// returns, if any.
    fn set_config(&mut self, config: &Value) {
        if let Err(e) = self.runtime.set_config(config) {
            trace_error(&e);
        }
    }

}

impl HookDataAccessorProvider for UpdateHook {

    fn accessor(&self) -> HookDataAccessor {
        HookDataAccessor::StoreIdAccess(self)
    }
}

impl StoreIdAccessor for UpdateHook {

    /// The implementation of the UpdateHook
    ///
    /// # Scope
    ///
    /// This hook takes the git index and commits it either interactively or with a default message,
    /// if there is no configuration for an interactive commit.
    ///
    fn access(&self, id: &StoreId) -> HookResult<()> {
        use vcs::git::action::StoreAction;
        use vcs::git::config::commit_message;
        use vcs::git::error::MapIntoHookError;
        use vcs::git::util::fetch_index;
        use git2::{ADD_DEFAULT, STATUS_WT_NEW, STATUS_WT_MODIFIED, IndexMatchedPath};

        debug!("[GIT UPDATE HOOK]: {:?}", id);

        let action    = StoreAction::Update;
        let cfg       = try!(self.runtime.config_value_or_err(&action));
        let repo      = try!(self.runtime.repository(&action));
        let mut index = try!(fetch_index(repo, &action));

        if !self.runtime.repo_is_dirty(&index) {
            debug!("[GIT UPDATE HOOK]: Repository seems to be clean. I'm done.");
            return Ok(())
        } else {
            debug!("[GIT UPDATE HOOK]: Repository seems to be dirty. Continuing.");
        }

        let signature = try!(
            repo.signature()
                .map_err_into(GHEK::MkSignature)
                .map_dbg_err_str("Failed to fetch signature")
                .map_into_hook_error()
        );

        let head = try!(
            repo.head()
                .map_err_into(GHEK::HeadFetchError)
                .map_dbg_err_str("Failed to fetch HEAD")
                .map_into_hook_error()
        );

        let file_status = try!(
            repo
                .status_file(id.local())
                .map_dbg_err_str("Failed to fetch file status")
                .map_dbg_err(|e| format!("\t->  {:?}", e))
                .map_err_into(GHEK::RepositoryFileStatusError)
                .map_into_hook_error()
        );

        let cb = &mut |path: &Path, _matched_spec: &[u8]| -> i32 {
            if file_status.contains(STATUS_WT_NEW) || file_status.contains(STATUS_WT_MODIFIED) {
                debug!("[GIT CREATE HOOK]: File is modified/new: {}", path.display());
                0
            } else {
                debug!("[GIT CREATE HOOK]: Ignoring file: {}", path.display());
                1
            }
        };

        try!(
            index.add_all(&[id.local()], ADD_DEFAULT, Some(cb as &mut IndexMatchedPath))
                .map_err_into(GHEK::RepositoryPathAddingError)
                .map_dbg_err_str("Failed to add to index")
                .map_into_hook_error()
        );

        let tree_id = try!(
            index.write_tree()
                .map_err_into(GHEK::RepositoryIndexWritingError)
                .map_dbg_err_str("Failed to write tree")
                .map_into_hook_error()
        );

        let mut parents = Vec::new();
        {
            let commit = try!(
                repo.find_commit(head.target().unwrap())
                    .map_err_into(GHEK::RepositoryParentFetchingError)
                    .map_dbg_err_str("Failed to find commit HEAD")
                    .map_into_hook_error()
            );
            parents.push(commit);
        }

        // for converting from Vec<Commit> to Vec<&Commit>
        let parents = parents.iter().collect::<Vec<_>>();

        let tree = try!(
            repo.find_tree(tree_id)
                .map_err_into(GHEK::RepositoryParentFetchingError)
                .map_dbg_err_str("Failed to find tree")
                .map_into_hook_error()
        );

        let message = try!(commit_message(&repo, cfg, StoreAction::Update)
                .map_dbg_err_str("Failed to get commit message"));

        try!(repo.commit(Some("HEAD"), &signature, &signature, &message, &tree, &parents)
            .map_dbg_str("Committed")
            .map_dbg_err_str("Failed to commit")
            .map_err_into(GHEK::RepositoryCommittingError)
            .map_into_hook_error()
        );

        index.write()
            .map_err_into(GHEK::RepositoryIndexWritingError)
            .map_dbg_err_str("Failed to write tree")
            .map_into_hook_error()
            .map(|_| ())
    }

}

<commit_msg>Add debug output for UpdateHook<commit_after>use std::path::PathBuf;
use std::path::Path;
use std::fmt::{Debug, Formatter, Error as FmtError};
use std::result::Result as RResult;

use toml::Value;

use libimagerror::trace::trace_error;
use libimagstore::hook::Hook;
use libimagstore::hook::accessor::StoreIdAccessor;
use libimagstore::hook::accessor::{HookDataAccessor, HookDataAccessorProvider};
use libimagstore::hook::position::HookPosition;
use libimagstore::hook::result::HookResult;
use libimagstore::storeid::StoreId;
use libimagutil::debug_result::*;

use vcs::git::error::GitHookErrorKind as GHEK;
use vcs::git::error::MapErrInto;
use vcs::git::runtime::Runtime as GRuntime;

/// The `UpdateHook` type
///
/// Represents a hook which is executed whenever a entry in the store is updated (written to disk).
///
/// # Time of execution
///
/// This hook is executed _after_ the store operation succeeded, so _after_ the file is written to
/// disk.
pub struct UpdateHook {
    storepath: PathBuf,

    runtime: GRuntime,

    position: HookPosition,
}

impl UpdateHook {

    pub fn new(storepath: PathBuf, p: HookPosition) -> UpdateHook {
        UpdateHook {
            runtime: GRuntime::new(&storepath),
            storepath: storepath,
            position: p,
        }
    }

}

impl Debug for UpdateHook {
    fn fmt(&self, fmt: &mut Formatter) -> RResult<(), FmtError> {
        write!(fmt, "UpdateHook(storepath={:?}, repository={}, pos={:?}, cfg={:?})",
               self.storepath,
               (if self.runtime.has_repository() { "Some(_)" } else { "None" }),
               self.position,
               self.runtime.has_config())
    }
}

impl Hook for UpdateHook {

    fn name(&self) -> &'static str {
        "stdhook_git_update"
    }

    /// Set the configuration of the hook. See
    /// `libimagstorestdhook::vcs::git::runtime::Runtime::set_config()`.
    ///
    /// This function traces the error (using `trace_error()`) that
    /// `libimagstorestdhook::vcs::git::runtime::Runtime::set_config()`
    /// returns, if any.
    fn set_config(&mut self, config: &Value) {
        if let Err(e) = self.runtime.set_config(config) {
            trace_error(&e);
        }
    }

}

impl HookDataAccessorProvider for UpdateHook {

    fn accessor(&self) -> HookDataAccessor {
        HookDataAccessor::StoreIdAccess(self)
    }
}

impl StoreIdAccessor for UpdateHook {

    /// The implementation of the UpdateHook
    ///
    /// # Scope
    ///
    /// This hook takes the git index and commits it either interactively or with a default message,
    /// if there is no configuration for an interactive commit.
    ///
    fn access(&self, id: &StoreId) -> HookResult<()> {
        use vcs::git::action::StoreAction;
        use vcs::git::config::commit_message;
        use vcs::git::error::MapIntoHookError;
        use vcs::git::util::fetch_index;
        use git2::{ADD_DEFAULT, STATUS_WT_NEW, STATUS_WT_MODIFIED, IndexMatchedPath};

        debug!("[GIT UPDATE HOOK]: {:?}", id);

        let action    = StoreAction::Update;
        let cfg       = try!(self.runtime.config_value_or_err(&action));
        let repo      = try!(self.runtime.repository(&action));
        let mut index = try!(fetch_index(repo, &action));

        if !self.runtime.repo_is_dirty(&index) {
            debug!("[GIT UPDATE HOOK]: Repository seems to be clean. I'm done.");
            return Ok(())
        } else {
            debug!("[GIT UPDATE HOOK]: Repository seems to be dirty. Continuing.");
        }

        let signature = try!(
            repo.signature()
                .map_err_into(GHEK::MkSignature)
                .map_dbg_err_str("Failed to fetch signature")
                .map_dbg_str("[GIT UPDATE HOOK]: Fetched signature object")
                .map_into_hook_error()
        );

        let head = try!(
            repo.head()
                .map_err_into(GHEK::HeadFetchError)
                .map_dbg_err_str("Failed to fetch HEAD")
                .map_dbg_str("[GIT UPDATE HOOK]: Fetched HEAD")
                .map_into_hook_error()
        );

        let file_status = try!(
            repo
                .status_file(id.local())
                .map_dbg_err_str("Failed to fetch file status")
                .map_dbg_err(|e| format!("\t->  {:?}", e))
                .map_dbg_str("[GIT UPDATE HOOK]: Fetched file status")
                .map_err_into(GHEK::RepositoryFileStatusError)
                .map_into_hook_error()
        );

        let cb = &mut |path: &Path, _matched_spec: &[u8]| -> i32 {
            if file_status.contains(STATUS_WT_NEW) || file_status.contains(STATUS_WT_MODIFIED) {
                debug!("[GIT CREATE HOOK]: File is modified/new: {}", path.display());
                0
            } else {
                debug!("[GIT CREATE HOOK]: Ignoring file: {}", path.display());
                1
            }
        };

        try!(
            index.add_all(&[id.local()], ADD_DEFAULT, Some(cb as &mut IndexMatchedPath))
                .map_err_into(GHEK::RepositoryPathAddingError)
                .map_dbg_err_str("Failed to add to index")
                .map_dbg(|_| format!("[GIT UPDATE HOOK]: Added id ({:?}) to index", id))
                .map_into_hook_error()
        );

        let tree_id = try!(
            index.write_tree()
                .map_err_into(GHEK::RepositoryIndexWritingError)
                .map_dbg_err_str("Failed to write tree")
                .map_dbg_str("[GIT UPDATE HOOK]: Wrote tree")
                .map_into_hook_error()
        );

        let mut parents = Vec::new();
        {
            let commit = try!(
                repo.find_commit(head.target().unwrap())
                    .map_err_into(GHEK::RepositoryParentFetchingError)
                    .map_dbg_err_str("Failed to find commit HEAD")
                    .map_dbg_str("[GIT UPDATE HOOK]: Fetched commit parents of HEAD")
                    .map_into_hook_error()
            );
            parents.push(commit);
        }

        // for converting from Vec<Commit> to Vec<&Commit>
        let parents = parents.iter().collect::<Vec<_>>();

        let tree = try!(
            repo.find_tree(tree_id)
                .map_err_into(GHEK::RepositoryParentFetchingError)
                .map_dbg_err_str("Failed to find tree")
                .map_dbg_str("[GIT UPDATE HOOK]: Found Tree for parents")
                .map_into_hook_error()
        );

        let message = try!(commit_message(&repo, cfg, StoreAction::Update)
                .map_dbg_err_str("Failed to get commit message")
                .map_dbg_str("[GIT UPDATE HOOK]: Fetched commit message"));

        try!(repo.commit(Some("HEAD"), &signature, &signature, &message, &tree, &parents)
            .map_dbg_str("Committed")
            .map_dbg_err_str("Failed to commit")
            .map_err_into(GHEK::RepositoryCommittingError)
            .map_into_hook_error()
        );

        index.write()
            .map_err_into(GHEK::RepositoryIndexWritingError)
            .map_dbg_err_str("Failed to write tree")
            .map_into_hook_error()
            .map(|_| ())
    }

}

<|endoftext|>