<commit_before>//! A stably addressed token buffer supporting efficient traversal based on a
//! cheaply copyable cursor.
//!
//! *This module is available only if Syn is built with the `"parsing"` feature.*

// This module is heavily commented as it contains most of the unsafe code in
// Syn, and caution should be used when editing it. The public-facing interface
// is 100% safe but the implementation is fragile internally.

#[cfg(all(
    not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
    feature = "proc-macro"
))]
use crate::proc_macro as pm;
use crate::Lifetime;
use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::marker::PhantomData;
use std::ptr;
use std::slice;

/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
enum Entry {
    // Mimicking types from proc-macro.
    Group(Group, TokenBuffer),
    Ident(Ident),
    Punct(Punct),
    Literal(Literal),
    // End entries contain a raw pointer to the entry from the containing
    // token tree, or null if this is the outermost level.
    End(*const Entry),
}

/// A buffer that can be efficiently traversed multiple times, unlike
/// `TokenStream` which requires a deep copy in order to traverse more than
/// once.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct TokenBuffer {
    // NOTE: Do not implement clone on this - there are raw pointers inside
    // these entries which will be messed up. Moving the `TokenBuffer` itself is
    // safe as the data pointed to won't be moved.
    ptr: *const Entry,
    len: usize,
}

impl Drop for TokenBuffer {
    fn drop(&mut self) {
        unsafe {
            let slice = slice::from_raw_parts_mut(self.ptr as *mut Entry, self.len);
            let _ = Box::from_raw(slice);
        }
    }
}

impl TokenBuffer {
    // NOTE: Do not mutate the Vec returned from this function once it returns;
    // the address of its backing memory must remain stable.
    fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut groups = Vec::new();
        for tt in stream {
            match tt {
                TokenTree::Ident(sym) => {
                    entries.push(Entry::Ident(sym));
                }
                TokenTree::Punct(op) => {
                    entries.push(Entry::Punct(op));
                }
                TokenTree::Literal(l) => {
                    entries.push(Entry::Literal(l));
                }
                TokenTree::Group(g) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarily.
                    groups.push((entries.len(), g));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));

        // NOTE: This is done to ensure that we don't accidentally modify the
        // length of the backing buffer. The backing buffer must remain at a
        // constant address after this point, as we are going to store a raw
        // pointer into it.
        let mut entries = entries.into_boxed_slice();
        for (idx, group) in groups {
            // We know that this index refers to one of the temporary
            // `End(null)` entries, and we know that the last entry is
            // `End(up)`, so the next index is also valid.
            let group_up = unsafe { entries.as_ptr().add(idx + 1) };

            // The end entry stored at the end of this Entry::Group should
            // point to the Entry which follows the Group in the list.
            let inner = Self::inner_new(group.stream(), group_up);
            entries[idx] = Entry::Group(group, inner);
        }

        let len = entries.len();
        let ptr = Box::into_raw(entries);
        TokenBuffer {
            ptr: ptr as *const Entry,
            len,
        }
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `TokenStream`.
    ///
    /// *This method is available only if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
        feature = "proc-macro"
    ))]
    pub fn new(stream: pm::TokenStream) -> TokenBuffer {
        Self::new2(stream.into())
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `TokenStream`.
    pub fn new2(stream: TokenStream) -> TokenBuffer {
        Self::inner_new(stream, ptr::null())
    }

    /// Creates a cursor referencing the first token in the buffer and able to
    /// traverse until the end of the buffer.
    pub fn begin(&self) -> Cursor {
        unsafe { Cursor::create(self.ptr, self.ptr.add(self.len - 1)) }
    }
}

/// A cheaply copyable cursor into a `TokenBuffer`.
///
/// This cursor holds a shared reference into the immutable data which is used
/// internally to represent a `TokenStream`, and can be efficiently manipulated
/// and copied around.
///
/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer`
/// object and get a cursor to its first token with `begin()`.
///
/// Two cursors are equal if they have the same location in the same input
/// stream, and have the same scope.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct Cursor<'a> {
    // The current entry which the `Cursor` is pointing at.
    ptr: *const Entry,
    // This is the only `Entry::End(..)` object which this cursor is allowed to
    // point at. All other `End` objects are skipped over in `Cursor::create`.
    scope: *const Entry,
    // Cursor is covariant in 'a. This field ensures that our pointers are still
    // valid.
    marker: PhantomData<&'a Entry>,
}

impl<'a> Cursor<'a> {
    /// Creates a cursor referencing a static empty TokenStream.
    pub fn empty() -> Self {
        // It's safe in this situation for us to put an `Entry` object in global
        // storage, despite it not actually being safe to send across threads
        // (`Ident` is a reference into a thread-local table). This is because
        // this entry never includes a `Ident` object.
        //
        // This wrapper struct allows us to break the rules and put a `Sync`
        // object in global storage.
        struct UnsafeSyncEntry(Entry);
        unsafe impl Sync for UnsafeSyncEntry {}
        static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry));

        Cursor {
            ptr: &EMPTY_ENTRY.0,
            scope: &EMPTY_ENTRY.0,
            marker: PhantomData,
        }
    }

    /// This create method intelligently exits non-explicitly-entered
    /// `None`-delimited scopes when the cursor reaches the end of them,
    /// allowing for them to be treated transparently.
    unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self {
        // NOTE: If we're looking at a `End(..)`, we want to advance the cursor
        // past it, unless `ptr == scope`, which means that we're at the edge of
        // our cursor's scope. We should only have `ptr != scope` at the exit
        // from None-delimited groups entered with `ignore_none`.
        while let Entry::End(exit) = *ptr {
            if ptr == scope {
                break;
            }
            ptr = exit;
        }

        Cursor {
            ptr,
            scope,
            marker: PhantomData,
        }
    }

    /// Get the current entry.
    fn entry(self) -> &'a Entry {
        unsafe { &*self.ptr }
    }

    /// Bump the cursor to point at the next token after the current one. This
    /// is undefined behavior if the cursor is currently looking at an
    /// `Entry::End`.
    unsafe fn bump(self) -> Cursor<'a> {
        Cursor::create(self.ptr.offset(1), self.scope)
    }

    /// While the cursor is looking at a `None`-delimited group, move it to look
    /// at the first token inside instead. If the group is empty, this will move
    /// the cursor past the `None`-delimited group.
    ///
    /// WARNING: This mutates its argument.
    fn ignore_none(&mut self) {
        while let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == Delimiter::None {
                // NOTE: We call `Cursor::create` here to make sure that
                // situations where we should immediately exit the span after
                // entering it are handled correctly.
                unsafe {
                    *self = Cursor::create(buf.ptr, self.scope);
                }
            } else {
                break;
            }
        }
    }

    /// Checks whether the cursor is currently pointing at the end of its valid
    /// scope.
    pub fn eof(self) -> bool {
        // We're at eof if we're at the end of our scope.
        self.ptr == self.scope
    }

    /// If the cursor is pointing at a `Group` with the given delimiter, returns
    /// a cursor into that group and one pointing to the next `TokenTree`.
    pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> {
        // If we're not trying to enter a none-delimited group, we want to
        // ignore them. We have to make sure to _not_ ignore them when we want
        // to enter them, of course. For obvious reasons.
        if delim != Delimiter::None {
            self.ignore_none();
        }

        if let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == delim {
                return Some((buf.begin(), group.span(), unsafe { self.bump() }));
            }
        }

        None
    }

    /// If the cursor is pointing at a `Ident`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at an `Punct`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() != '\'' => Some((op.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Literal`, return it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Literal(lit) => Some((lit.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Lifetime`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.ident() {
                    Some((ident, rest)) => {
                        let lifetime = Lifetime {
                            apostrophe: op.span(),
                            ident,
                        };
                        Some((lifetime, rest))
                    }
                    None => None,
                }
            }
            _ => None,
        }
    }

    /// Copies all remaining tokens visible from this cursor into a
    /// `TokenStream`.
    pub fn token_stream(self) -> TokenStream {
        let mut tts = Vec::new();
        let mut cursor = self;
        while let Some((tt, rest)) = cursor.token_tree() {
            tts.push(tt);
            cursor = rest;
        }
        tts.into_iter().collect()
    }

    /// If the cursor is pointing at a `TokenTree`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    ///
    /// Returns `None` if the cursor has reached the end of its stream.
    ///
    /// This method does not treat `None`-delimited groups as transparent, and
    /// will return a `Group(None, ..)` if the cursor is looking at one.
    pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> {
        let tree = match self.entry() {
            Entry::Group(group, _) => group.clone().into(),
            Entry::Literal(lit) => lit.clone().into(),
            Entry::Ident(ident) => ident.clone().into(),
            Entry::Punct(op) => op.clone().into(),
            Entry::End(..) => {
                return None;
            }
        };

        Some((tree, unsafe { self.bump() }))
    }

    /// Returns the `Span` of the current token, or `Span::call_site()` if this
    /// cursor points to eof.
    pub fn span(self) -> Span {
        match self.entry() {
            Entry::Group(group, _) => group.span(),
            Entry::Literal(l) => l.span(),
            Entry::Ident(t) => t.span(),
            Entry::Punct(o) => o.span(),
            Entry::End(..) => Span::call_site(),
        }
    }

    /// Skip over the next token without cloning it. Returns `None` if this
    /// cursor points to eof.
    ///
    /// This method treats `'lifetimes` as a single token.
    pub(crate) fn skip(self) -> Option<Cursor<'a>> {
        match self.entry() {
            Entry::End(..) => None,

            // Treat lifetimes as a single tt for the purposes of 'skip'.
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.entry() {
                    Entry::Ident(_) => Some(unsafe { next.bump() }),
                    _ => Some(next),
                }
            }
            _ => Some(unsafe { self.bump() }),
        }
    }
}

impl<'a> Copy for Cursor<'a> {}

impl<'a> Clone for Cursor<'a> {
    fn clone(&self) -> Self {
        *self
    }
}

impl<'a> Eq for Cursor<'a> {}

impl<'a> PartialEq for Cursor<'a> {
    fn eq(&self, other: &Self) -> bool {
        let Cursor { ptr, scope, marker } = self;
        let _ = marker;
        *ptr == other.ptr && *scope == other.scope
    }
}

pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool {
    a.scope == b.scope
}

pub(crate) fn open_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_open(),
        _ => cursor.span(),
    }
}

pub(crate) fn close_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_close(),
        _ => cursor.span(),
    }
}
<commit_msg>Touch up some formatting in Cursor::token_tree<commit_after>//! A stably addressed token buffer supporting efficient traversal based on a
//! cheaply copyable cursor.
//!
//! *This module is available only if Syn is built with the `"parsing"` feature.*

// This module is heavily commented as it contains most of the unsafe code in
// Syn, and caution should be used when editing it. The public-facing interface
// is 100% safe but the implementation is fragile internally.

#[cfg(all(
    not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
    feature = "proc-macro"
))]
use crate::proc_macro as pm;
use crate::Lifetime;
use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::marker::PhantomData;
use std::ptr;
use std::slice;

/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
enum Entry {
    // Mimicking types from proc-macro.
    Group(Group, TokenBuffer),
    Ident(Ident),
    Punct(Punct),
    Literal(Literal),
    // End entries contain a raw pointer to the entry from the containing
    // token tree, or null if this is the outermost level.
    End(*const Entry),
}

/// A buffer that can be efficiently traversed multiple times, unlike
/// `TokenStream` which requires a deep copy in order to traverse more than
/// once.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct TokenBuffer {
    // NOTE: Do not implement clone on this - there are raw pointers inside
    // these entries which will be messed up. Moving the `TokenBuffer` itself is
    // safe as the data pointed to won't be moved.
    ptr: *const Entry,
    len: usize,
}

impl Drop for TokenBuffer {
    fn drop(&mut self) {
        unsafe {
            let slice = slice::from_raw_parts_mut(self.ptr as *mut Entry, self.len);
            let _ = Box::from_raw(slice);
        }
    }
}

impl TokenBuffer {
    // NOTE: Do not mutate the Vec returned from this function once it returns;
    // the address of its backing memory must remain stable.
    fn inner_new(stream: TokenStream, up: *const Entry) -> TokenBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut groups = Vec::new();
        for tt in stream {
            match tt {
                TokenTree::Ident(sym) => {
                    entries.push(Entry::Ident(sym));
                }
                TokenTree::Punct(op) => {
                    entries.push(Entry::Punct(op));
                }
                TokenTree::Literal(l) => {
                    entries.push(Entry::Literal(l));
                }
                TokenTree::Group(g) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarily.
                    groups.push((entries.len(), g));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));

        // NOTE: This is done to ensure that we don't accidentally modify the
        // length of the backing buffer. The backing buffer must remain at a
        // constant address after this point, as we are going to store a raw
        // pointer into it.
        let mut entries = entries.into_boxed_slice();
        for (idx, group) in groups {
            // We know that this index refers to one of the temporary
            // `End(null)` entries, and we know that the last entry is
            // `End(up)`, so the next index is also valid.
            let group_up = unsafe { entries.as_ptr().add(idx + 1) };

            // The end entry stored at the end of this Entry::Group should
            // point to the Entry which follows the Group in the list.
            let inner = Self::inner_new(group.stream(), group_up);
            entries[idx] = Entry::Group(group, inner);
        }

        let len = entries.len();
        let ptr = Box::into_raw(entries);
        TokenBuffer {
            ptr: ptr as *const Entry,
            len,
        }
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `TokenStream`.
    ///
    /// *This method is available only if Syn is built with both the `"parsing"` and
    /// `"proc-macro"` features.*
    #[cfg(all(
        not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
        feature = "proc-macro"
    ))]
    pub fn new(stream: pm::TokenStream) -> TokenBuffer {
        Self::new2(stream.into())
    }

    /// Creates a `TokenBuffer` containing all the tokens from the input
    /// `TokenStream`.
    pub fn new2(stream: TokenStream) -> TokenBuffer {
        Self::inner_new(stream, ptr::null())
    }

    /// Creates a cursor referencing the first token in the buffer and able to
    /// traverse until the end of the buffer.
    pub fn begin(&self) -> Cursor {
        unsafe { Cursor::create(self.ptr, self.ptr.add(self.len - 1)) }
    }
}

/// A cheaply copyable cursor into a `TokenBuffer`.
///
/// This cursor holds a shared reference into the immutable data which is used
/// internally to represent a `TokenStream`, and can be efficiently manipulated
/// and copied around.
///
/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer`
/// object and get a cursor to its first token with `begin()`.
///
/// Two cursors are equal if they have the same location in the same input
/// stream, and have the same scope.
///
/// *This type is available only if Syn is built with the `"parsing"` feature.*
pub struct Cursor<'a> {
    // The current entry which the `Cursor` is pointing at.
    ptr: *const Entry,
    // This is the only `Entry::End(..)` object which this cursor is allowed to
    // point at. All other `End` objects are skipped over in `Cursor::create`.
    scope: *const Entry,
    // Cursor is covariant in 'a. This field ensures that our pointers are still
    // valid.
    marker: PhantomData<&'a Entry>,
}

impl<'a> Cursor<'a> {
    /// Creates a cursor referencing a static empty TokenStream.
    pub fn empty() -> Self {
        // It's safe in this situation for us to put an `Entry` object in global
        // storage, despite it not actually being safe to send across threads
        // (`Ident` is a reference into a thread-local table). This is because
        // this entry never includes a `Ident` object.
        //
        // This wrapper struct allows us to break the rules and put a `Sync`
        // object in global storage.
        struct UnsafeSyncEntry(Entry);
        unsafe impl Sync for UnsafeSyncEntry {}
        static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0 as *const Entry));

        Cursor {
            ptr: &EMPTY_ENTRY.0,
            scope: &EMPTY_ENTRY.0,
            marker: PhantomData,
        }
    }

    /// This create method intelligently exits non-explicitly-entered
    /// `None`-delimited scopes when the cursor reaches the end of them,
    /// allowing for them to be treated transparently.
    unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self {
        // NOTE: If we're looking at a `End(..)`, we want to advance the cursor
        // past it, unless `ptr == scope`, which means that we're at the edge of
        // our cursor's scope. We should only have `ptr != scope` at the exit
        // from None-delimited groups entered with `ignore_none`.
        while let Entry::End(exit) = *ptr {
            if ptr == scope {
                break;
            }
            ptr = exit;
        }

        Cursor {
            ptr,
            scope,
            marker: PhantomData,
        }
    }

    /// Get the current entry.
    fn entry(self) -> &'a Entry {
        unsafe { &*self.ptr }
    }

    /// Bump the cursor to point at the next token after the current one. This
    /// is undefined behavior if the cursor is currently looking at an
    /// `Entry::End`.
    unsafe fn bump(self) -> Cursor<'a> {
        Cursor::create(self.ptr.offset(1), self.scope)
    }

    /// While the cursor is looking at a `None`-delimited group, move it to look
    /// at the first token inside instead. If the group is empty, this will move
    /// the cursor past the `None`-delimited group.
    ///
    /// WARNING: This mutates its argument.
    fn ignore_none(&mut self) {
        while let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == Delimiter::None {
                // NOTE: We call `Cursor::create` here to make sure that
                // situations where we should immediately exit the span after
                // entering it are handled correctly.
                unsafe {
                    *self = Cursor::create(buf.ptr, self.scope);
                }
            } else {
                break;
            }
        }
    }

    /// Checks whether the cursor is currently pointing at the end of its valid
    /// scope.
    pub fn eof(self) -> bool {
        // We're at eof if we're at the end of our scope.
        self.ptr == self.scope
    }

    /// If the cursor is pointing at a `Group` with the given delimiter, returns
    /// a cursor into that group and one pointing to the next `TokenTree`.
    pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> {
        // If we're not trying to enter a none-delimited group, we want to
        // ignore them. We have to make sure to _not_ ignore them when we want
        // to enter them, of course. For obvious reasons.
        if delim != Delimiter::None {
            self.ignore_none();
        }

        if let Entry::Group(group, buf) = self.entry() {
            if group.delimiter() == delim {
                return Some((buf.begin(), group.span(), unsafe { self.bump() }));
            }
        }

        None
    }

    /// If the cursor is pointing at a `Ident`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at an `Punct`, returns it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() != '\'' => Some((op.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Literal`, return it along with a cursor
    /// pointing at the next `TokenTree`.
    pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Literal(lit) => Some((lit.clone(), unsafe { self.bump() })),
            _ => None,
        }
    }

    /// If the cursor is pointing at a `Lifetime`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> {
        self.ignore_none();
        match self.entry() {
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.ident() {
                    Some((ident, rest)) => {
                        let lifetime = Lifetime {
                            apostrophe: op.span(),
                            ident,
                        };
                        Some((lifetime, rest))
                    }
                    None => None,
                }
            }
            _ => None,
        }
    }

    /// Copies all remaining tokens visible from this cursor into a
    /// `TokenStream`.
    pub fn token_stream(self) -> TokenStream {
        let mut tts = Vec::new();
        let mut cursor = self;
        while let Some((tt, rest)) = cursor.token_tree() {
            tts.push(tt);
            cursor = rest;
        }
        tts.into_iter().collect()
    }

    /// If the cursor is pointing at a `TokenTree`, returns it along with a
    /// cursor pointing at the next `TokenTree`.
    ///
    /// Returns `None` if the cursor has reached the end of its stream.
    ///
    /// This method does not treat `None`-delimited groups as transparent, and
    /// will return a `Group(None, ..)` if the cursor is looking at one.
    pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> {
        let tree = match self.entry() {
            Entry::Group(group, _) => group.clone().into(),
            Entry::Literal(lit) => lit.clone().into(),
            Entry::Ident(ident) => ident.clone().into(),
            Entry::Punct(op) => op.clone().into(),
            Entry::End(..) => return None,
        };

        Some((tree, unsafe { self.bump() }))
    }

    /// Returns the `Span` of the current token, or `Span::call_site()` if this
    /// cursor points to eof.
    pub fn span(self) -> Span {
        match self.entry() {
            Entry::Group(group, _) => group.span(),
            Entry::Literal(l) => l.span(),
            Entry::Ident(t) => t.span(),
            Entry::Punct(o) => o.span(),
            Entry::End(..) => Span::call_site(),
        }
    }

    /// Skip over the next token without cloning it. Returns `None` if this
    /// cursor points to eof.
    ///
    /// This method treats `'lifetimes` as a single token.
    pub(crate) fn skip(self) -> Option<Cursor<'a>> {
        match self.entry() {
            Entry::End(..) => None,

            // Treat lifetimes as a single tt for the purposes of 'skip'.
            Entry::Punct(op) if op.as_char() == '\'' && op.spacing() == Spacing::Joint => {
                let next = unsafe { self.bump() };
                match next.entry() {
                    Entry::Ident(_) => Some(unsafe { next.bump() }),
                    _ => Some(next),
                }
            }
            _ => Some(unsafe { self.bump() }),
        }
    }
}

impl<'a> Copy for Cursor<'a> {}

impl<'a> Clone for Cursor<'a> {
    fn clone(&self) -> Self {
        *self
    }
}

impl<'a> Eq for Cursor<'a> {}

impl<'a> PartialEq for Cursor<'a> {
    fn eq(&self, other: &Self) -> bool {
        let Cursor { ptr, scope, marker } = self;
        let _ = marker;
        *ptr == other.ptr && *scope == other.scope
    }
}

pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool {
    a.scope == b.scope
}

pub(crate) fn open_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_open(),
        _ => cursor.span(),
    }
}

pub(crate) fn close_span_of_group(cursor: Cursor) -> Span {
    match cursor.entry() {
        Entry::Group(group, _) => group.span_close(),
        _ => cursor.span(),
    }
}
<|endoftext|>


<commit_before>extern crate tensorflow_sys as tf;

use libc;
use libc::c_void;
use libc::size_t;
use std::borrow::Borrow;
use std::borrow::BorrowMut;
use std::cmp;
use std::ffi::CStr;
use std::marker::PhantomData;
use std::mem;
use std::ops::Deref;
use std::ops::DerefMut;
use std::ops::Index;
use std::ops::IndexMut;
use std::ops::Range;
use std::ops::RangeFrom;
use std::ops::RangeFull;
use std::ops::RangeTo;
use std::os::raw::c_void as std_c_void;
use std::ptr;
use std::slice;
use super::BufferTrait;
use super::TensorType;

/// Fixed-length heap-allocated vector.
/// This is basically a `Box<[T]>`, except that that type can't actually be constructed.
/// Furthermore, `[T; N]` can't be constructed if N is not a compile-time constant.
#[derive(Debug)]
pub struct Buffer<T: TensorType> {
    inner: *mut tf::TF_Buffer,
    owned: bool,
    phantom: PhantomData<T>,
}

impl<T: TensorType> Buffer<T> {
    /// Creates a new buffer initialized to zeros.
    ///
    /// `len` is the number of elements.
    pub fn new(len: usize) -> Self {
        let mut b = unsafe { Buffer::new_uninitialized(len) };
        // TODO: Use libc::memset for primitives once we have impl specialization and
        // memset is included
        for i in 0..len {
            b[i] = T::default();
        }
        b
    }

    /// Creates a new uninitialized buffer.
    ///
    /// `len` is the number of elements.
    /// The caller is responsible for initializing the data.
    pub unsafe fn new_uninitialized(len: usize) -> Self {
        let inner = tf::TF_NewBuffer();
        let elem_size = mem::size_of::<T>();
        let alloc_size = len * elem_size;
        let align = cmp::max(mem::align_of::<T>(), mem::size_of::<*const c_void>());
        // posix_memalign requires the alignment to be at least sizeof(void*).
        // TODO: Use alloc::heap::allocate once it's stable, or at least
        // libc::aligned_alloc once it exists
        let mut ptr = ptr::null::<c_void>() as *mut c_void;
        let err = libc::posix_memalign(&mut ptr, align, alloc_size);
        if err != 0 {
            let c_msg = libc::strerror(err);
            let msg = CStr::from_ptr(c_msg);
            panic!("Failed to allocate: {}", msg.to_str().unwrap());
        }
        (*inner).data = ptr as *mut std_c_void;
        (*inner).length = len;
        (*inner).data_deallocator = Some(deallocator);
        Buffer {
            inner: inner,
            owned: true,
            phantom: PhantomData,
        }
    }

    /// Creates a buffer from data owned by the C API.
    ///
    /// `len` is the number of elements.
    /// The underlying data is *not* freed when the buffer is destroyed.
    pub unsafe fn from_ptr(ptr: *mut T, len: usize) -> Self {
        let inner = tf::TF_NewBuffer();
        (*inner).data = ptr as *const std_c_void;
        (*inner).length = len;
        Buffer {
            inner: inner,
            owned: false,
            phantom: PhantomData,
        }
    }

    /// Consumes the buffer and returns the data.
    ///
    /// The caller is responsible for freeing the data.
    pub unsafe fn into_ptr(mut self) -> (*mut T, usize) {
        // TODO: remove
        // This flag is used by drop.
        self.owned = false;
        (self.data_mut(), self.length())
    }

    /// Returns a buffer with a null pointer.
    pub unsafe fn null() -> Self {
        Buffer {
            inner: ptr::null::<tf::TF_Buffer>() as *mut _,
            owned: false,
            phantom: PhantomData,
        }
    }

    #[inline]
    fn data(&self) -> *const T {
        unsafe { (*self.inner).data as *const T }
    }

    #[inline]
    fn data_mut(&mut self) -> *mut T {
        unsafe { (*self.inner).data as *mut T }
    }

    #[inline]
    fn length(&self) -> usize {
        unsafe { (*self.inner).length }
    }

    /// Creates a buffer from data owned by the C API.
    ///
    /// `len` is the number of elements.
    /// The underlying data is freed when the buffer is destroyed if `owned` is true.
    pub unsafe fn from_c(buf: *mut tf::TF_Buffer, owned: bool) -> Self {
        Buffer {
            inner: buf,
            owned: owned,
            phantom: PhantomData,
        }
    }
}

impl<T: TensorType> BufferTrait for Buffer<T> {
    fn is_owned(&self) -> bool {
        self.owned
    }

    fn set_owned(&mut self, owned: bool) {
        self.owned = owned;
    }

    fn inner(&self) -> *const tf::TF_Buffer {
        self.inner
    }

    fn inner_mut(&mut self) -> *mut tf::TF_Buffer {
        self.inner
    }
}

unsafe extern "C" fn deallocator(data: *mut std_c_void, _length: size_t) {
    libc::free(data as *mut c_void);
}

impl<T: TensorType> Drop for Buffer<T> {
    fn drop(&mut self) {
        if self.owned {
            unsafe {
                tf::TF_DeleteBuffer(self.inner);
            }
        }
    }
}

impl<T: TensorType> AsRef<[T]> for Buffer<T> {
    #[inline]
    fn as_ref(&self) -> &[T] {
        unsafe { slice::from_raw_parts(self.data(), (*self.inner).length) }
    }
}

impl<T: TensorType> AsMut<[T]> for Buffer<T> {
    #[inline]
    fn as_mut(&mut self) -> &mut [T] {
        unsafe { slice::from_raw_parts_mut(self.data_mut(), (*self.inner).length) }
    }
}

impl<T: TensorType> Deref for Buffer<T> {
    type Target = [T];

    #[inline]
    fn deref(&self) -> &[T] {
        self.as_ref()
    }
}

impl<T: TensorType> DerefMut for Buffer<T> {
    #[inline]
    fn deref_mut(&mut self) -> &mut [T] {
        self.as_mut()
    }
}

impl<T: TensorType> Borrow<[T]> for Buffer<T> {
    #[inline]
    fn borrow(&self) -> &[T] {
        self.as_ref()
    }
}

impl<T: TensorType> BorrowMut<[T]> for Buffer<T> {
    #[inline]
    fn borrow_mut(&mut self) -> &mut [T] {
        self.as_mut()
    }
}

impl<T: TensorType> Clone for Buffer<T>
    where T: Clone
{
    #[inline]
    fn clone(&self) -> Buffer<T> {
        let mut b = unsafe { Buffer::new_uninitialized((*self.inner).length) };
        // TODO: Use std::ptr::copy for primitives once we have impl specialization
        for i in 0..self.length() {
            b[i] = self[i].clone();
        }
        b
    }

    #[inline]
    fn clone_from(&mut self, other: &Buffer<T>) {
        assert!(self.length() == other.length(),
                "self.length() = {}, other.length() = {}",
                self.length(),
                other.length());
        // TODO: Use std::ptr::copy for primitives once we have impl specialization
        for i in 0..self.length() {
            self[i] = other[i].clone();
        }
    }
}

impl<T: TensorType> Index<usize> for Buffer<T> {
    type Output = T;

    #[inline]
    fn index(&self, index: usize) -> &T {
        assert!(index < self.length(),
                "index = {}, length = {}",
                index,
                self.length());
        unsafe { &*self.data().offset(index as isize) }
    }
}

impl<T: TensorType> IndexMut<usize> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: usize) -> &mut T {
        assert!(index < self.length(),
                "index = {}, length = {}",
                index,
                self.length());
        unsafe { &mut *self.data_mut().offset(index as isize) }
    }
}

impl<T: TensorType> Index<Range<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: Range<usize>) -> &[T] {
        assert!(index.start <= index.end,
                "index.start = {}, index.end = {}",
                index.start,
                index.end);
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts(&*self.data().offset(index.start as isize), index.len()) }
    }
}

impl<T: TensorType> IndexMut<Range<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: Range<usize>) -> &mut [T] {
        assert!(index.start <= index.end,
                "index.start = {}, index.end = {}",
                index.start,
                index.end);
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe {
            slice::from_raw_parts_mut(&mut *self.data_mut().offset(index.start as isize),
                                      index.len())
        }
    }
}

impl<T: TensorType> Index<RangeTo<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: RangeTo<usize>) -> &[T] {
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts(&*self.data(), index.end) }
    }
}

impl<T: TensorType> IndexMut<RangeTo<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: RangeTo<usize>) -> &mut [T] {
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts_mut(&mut *self.data_mut(), index.end) }
    }
}

impl<T: TensorType> Index<RangeFrom<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: RangeFrom<usize>) -> &[T] {
        assert!(index.start <= self.length(),
                "index.start = {}, length = {}",
                index.start,
                self.length());
        unsafe {
            slice::from_raw_parts(&*self.data().offset(index.start as isize),
                                  self.length() - index.start)
        }
    }
}

impl<T: TensorType> IndexMut<RangeFrom<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: RangeFrom<usize>) -> &mut [T] {
        assert!(index.start <= self.length(),
                "index.start = {}, length = {}",
                index.start,
                self.length());
        unsafe {
            slice::from_raw_parts_mut(&mut *self.data_mut().offset(index.start as isize),
                                      self.length() - index.start)
        }
    }
}

impl<T: TensorType> Index<RangeFull> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, _: RangeFull) -> &[T] {
        unsafe { slice::from_raw_parts(&*self.data(), self.length()) }
    }
}

impl<T: TensorType> IndexMut<RangeFull> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, _: RangeFull) -> &mut [T] {
        unsafe { slice::from_raw_parts_mut(&mut *self.data_mut(), self.length()) }
    }
}

impl<'a, T: TensorType> From<&'a [T]> for Buffer<T> {
    fn from(data: &'a [T]) -> Buffer<T> {
        let mut buffer = Buffer::new(data.len());
        buffer.clone_from_slice(data);
        buffer
    }
}

impl<'a, T: TensorType> From<&'a Vec<T>> for Buffer<T> {
    #[allow(trivial_casts)]
    fn from(data: &'a Vec<T>) -> Buffer<T> {
        Buffer::from(data as &[T])
    }
}

impl<T: TensorType> Into<Vec<T>> for Buffer<T> {
    fn into(self) -> Vec<T> {
        let mut vec = Vec::with_capacity(self.len());
        vec.extend_from_slice(&self);
        vec
    }
}

////////////////////////

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn basic() {
        let mut buf = Buffer::new(10);
        assert_eq!(buf.len(), 10);
        buf[0] = 1;
        assert_eq!(buf[0], 1);
    }
}
<commit_msg>Fix a memory leak in Buffer.<commit_after>extern crate tensorflow_sys as tf;

use libc;
use libc::c_void;
use libc::size_t;
use std::borrow::Borrow;
use std::borrow::BorrowMut;
use std::cmp;
use std::ffi::CStr;
use std::marker::PhantomData;
use std::mem;
use std::ops::Deref;
use std::ops::DerefMut;
use std::ops::Index;
use std::ops::IndexMut;
use std::ops::Range;
use std::ops::RangeFrom;
use std::ops::RangeFull;
use std::ops::RangeTo;
use std::os::raw::c_void as std_c_void;
use std::ptr;
use std::slice;
use super::BufferTrait;
use super::TensorType;

/// Fixed-length heap-allocated vector.
/// This is basically a `Box<[T]>`, except that that type can't actually be constructed.
/// Furthermore, `[T; N]` can't be constructed if N is not a compile-time constant.
#[derive(Debug)]
pub struct Buffer<T: TensorType> {
    inner: *mut tf::TF_Buffer,
    owned: bool,
    phantom: PhantomData<T>,
}

impl<T: TensorType> Buffer<T> {
    /// Creates a new buffer initialized to zeros.
    ///
    /// `len` is the number of elements.
    pub fn new(len: usize) -> Self {
        let mut b = unsafe { Buffer::new_uninitialized(len) };
        // TODO: Use libc::memset for primitives once we have impl specialization and
        // memset is included
        for i in 0..len {
            b[i] = T::default();
        }
        b
    }

    /// Creates a new uninitialized buffer.
    ///
    /// `len` is the number of elements.
    /// The caller is responsible for initializing the data.
    pub unsafe fn new_uninitialized(len: usize) -> Self {
        let inner = tf::TF_NewBuffer();
        let elem_size = mem::size_of::<T>();
        let alloc_size = len * elem_size;
        let align = cmp::max(mem::align_of::<T>(), mem::size_of::<*const c_void>());
        // posix_memalign requires the alignment to be at least sizeof(void*).
        // TODO: Use alloc::heap::allocate once it's stable, or at least
        // libc::aligned_alloc once it exists
        let mut ptr = ptr::null::<c_void>() as *mut c_void;
        let err = libc::posix_memalign(&mut ptr, align, alloc_size);
        if err != 0 {
            let c_msg = libc::strerror(err);
            let msg = CStr::from_ptr(c_msg);
            panic!("Failed to allocate: {}", msg.to_str().unwrap());
        }
        (*inner).data = ptr as *mut std_c_void;
        (*inner).length = len;
        (*inner).data_deallocator = Some(deallocator);
        Buffer {
            inner: inner,
            owned: true,
            phantom: PhantomData,
        }
    }

    /// Creates a buffer from data owned by the C API.
    ///
    /// `len` is the number of elements.
    /// The underlying data is *not* freed when the buffer is destroyed.
    pub unsafe fn from_ptr(ptr: *mut T, len: usize) -> Self {
        let inner = tf::TF_NewBuffer();
        (*inner).data = ptr as *const std_c_void;
        (*inner).length = len;
        Buffer {
            inner: inner,
            owned: true,
            phantom: PhantomData,
        }
    }

    /// Consumes the buffer and returns the data.
    ///
    /// The caller is responsible for freeing the data.
    pub unsafe fn into_ptr(mut self) -> (*mut T, usize) {
        // TODO: remove
        (*self.inner).data_deallocator = None;
        (self.data_mut(), self.length())
    }

    /// Returns a buffer with a null pointer.
    pub unsafe fn null() -> Self {
        Buffer {
            inner: ptr::null::<tf::TF_Buffer>() as *mut _,
            owned: false,
            phantom: PhantomData,
        }
    }

    #[inline]
    fn data(&self) -> *const T {
        unsafe { (*self.inner).data as *const T }
    }

    #[inline]
    fn data_mut(&mut self) -> *mut T {
        unsafe { (*self.inner).data as *mut T }
    }

    #[inline]
    fn length(&self) -> usize {
        unsafe { (*self.inner).length }
    }

    /// Creates a buffer from data owned by the C API.
    ///
    /// `len` is the number of elements.
    /// The underlying data is freed when the buffer is destroyed if `owned`
    /// is true and the `buf` has a data deallocator.
    pub unsafe fn from_c(buf: *mut tf::TF_Buffer, owned: bool) -> Self {
        Buffer {
            inner: buf,
            owned: owned,
            phantom: PhantomData,
        }
    }
}

impl<T: TensorType> BufferTrait for Buffer<T> {
    fn is_owned(&self) -> bool {
        self.owned
    }

    fn set_owned(&mut self, owned: bool) {
        self.owned = owned;
    }

    fn inner(&self) -> *const tf::TF_Buffer {
        self.inner
    }

    fn inner_mut(&mut self) -> *mut tf::TF_Buffer {
        self.inner
    }
}

unsafe extern "C" fn deallocator(data: *mut std_c_void, _length: size_t) {
    libc::free(data as *mut c_void);
}

impl<T: TensorType> Drop for Buffer<T> {
    fn drop(&mut self) {
        if self.owned {
            unsafe {
                tf::TF_DeleteBuffer(self.inner);
            }
        }
    }
}

impl<T: TensorType> AsRef<[T]> for Buffer<T> {
    #[inline]
    fn as_ref(&self) -> &[T] {
        unsafe { slice::from_raw_parts(self.data(), (*self.inner).length) }
    }
}

impl<T: TensorType> AsMut<[T]> for Buffer<T> {
    #[inline]
    fn as_mut(&mut self) -> &mut [T] {
        unsafe { slice::from_raw_parts_mut(self.data_mut(), (*self.inner).length) }
    }
}

impl<T: TensorType> Deref for Buffer<T> {
    type Target = [T];

    #[inline]
    fn deref(&self) -> &[T] {
        self.as_ref()
    }
}

impl<T: TensorType> DerefMut for Buffer<T> {
    #[inline]
    fn deref_mut(&mut self) -> &mut [T] {
        self.as_mut()
    }
}

impl<T: TensorType> Borrow<[T]> for Buffer<T> {
    #[inline]
    fn borrow(&self) -> &[T] {
        self.as_ref()
    }
}

impl<T: TensorType> BorrowMut<[T]> for Buffer<T> {
    #[inline]
    fn borrow_mut(&mut self) -> &mut [T] {
        self.as_mut()
    }
}

impl<T: TensorType> Clone for Buffer<T>
    where T: Clone
{
    #[inline]
    fn clone(&self) -> Buffer<T> {
        let mut b = unsafe { Buffer::new_uninitialized((*self.inner).length) };
        // TODO: Use std::ptr::copy for primitives once we have impl specialization
        for i in 0..self.length() {
            b[i] = self[i].clone();
        }
        b
    }

    #[inline]
    fn clone_from(&mut self, other: &Buffer<T>) {
        assert!(self.length() == other.length(),
                "self.length() = {}, other.length() = {}",
                self.length(),
                other.length());
        // TODO: Use std::ptr::copy for primitives once we have impl specialization
        for i in 0..self.length() {
            self[i] = other[i].clone();
        }
    }
}

impl<T: TensorType> Index<usize> for Buffer<T> {
    type Output = T;

    #[inline]
    fn index(&self, index: usize) -> &T {
        assert!(index < self.length(),
                "index = {}, length = {}",
                index,
                self.length());
        unsafe { &*self.data().offset(index as isize) }
    }
}

impl<T: TensorType> IndexMut<usize> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: usize) -> &mut T {
        assert!(index < self.length(),
                "index = {}, length = {}",
                index,
                self.length());
        unsafe { &mut *self.data_mut().offset(index as isize) }
    }
}

impl<T: TensorType> Index<Range<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: Range<usize>) -> &[T] {
        assert!(index.start <= index.end,
                "index.start = {}, index.end = {}",
                index.start,
                index.end);
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts(&*self.data().offset(index.start as isize), index.len()) }
    }
}

impl<T: TensorType> IndexMut<Range<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: Range<usize>) -> &mut [T] {
        assert!(index.start <= index.end,
                "index.start = {}, index.end = {}",
                index.start,
                index.end);
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe {
            slice::from_raw_parts_mut(&mut *self.data_mut().offset(index.start as isize),
                                      index.len())
        }
    }
}

impl<T: TensorType> Index<RangeTo<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: RangeTo<usize>) -> &[T] {
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts(&*self.data(), index.end) }
    }
}

impl<T: TensorType> IndexMut<RangeTo<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: RangeTo<usize>) -> &mut [T] {
        assert!(index.end <= self.length(),
                "index.end = {}, length = {}",
                index.end,
                self.length());
        unsafe { slice::from_raw_parts_mut(&mut *self.data_mut(), index.end) }
    }
}

impl<T: TensorType> Index<RangeFrom<usize>> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, index: RangeFrom<usize>) -> &[T] {
        assert!(index.start <= self.length(),
                "index.start = {}, length = {}",
                index.start,
                self.length());
        unsafe {
            slice::from_raw_parts(&*self.data().offset(index.start as isize),
                                  self.length() - index.start)
        }
    }
}

impl<T: TensorType> IndexMut<RangeFrom<usize>> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, index: RangeFrom<usize>) -> &mut [T] {
        assert!(index.start <= self.length(),
                "index.start = {}, length = {}",
                index.start,
                self.length());
        unsafe {
            slice::from_raw_parts_mut(&mut *self.data_mut().offset(index.start as isize),
                                      self.length() - index.start)
        }
    }
}

impl<T: TensorType> Index<RangeFull> for Buffer<T> {
    type Output = [T];

    #[inline]
    fn index(&self, _: RangeFull) -> &[T] {
        unsafe { slice::from_raw_parts(&*self.data(), self.length()) }
    }
}

impl<T: TensorType> IndexMut<RangeFull> for Buffer<T> {
    #[inline]
    fn index_mut(&mut self, _: RangeFull) -> &mut [T] {
        unsafe { slice::from_raw_parts_mut(&mut *self.data_mut(), self.length()) }
    }
}

impl<'a, T: TensorType> From<&'a [T]> for Buffer<T> {
    fn from(data: &'a [T]) -> Buffer<T> {
        let mut buffer = Buffer::new(data.len());
        buffer.clone_from_slice(data);
        buffer
    }
}

impl<'a, T: TensorType> From<&'a Vec<T>> for Buffer<T> {
    #[allow(trivial_casts)]
    fn from(data: &'a Vec<T>) -> Buffer<T> {
        Buffer::from(data as &[T])
    }
}

impl<T: TensorType> Into<Vec<T>> for Buffer<T> {
    fn into(self) -> Vec<T> {
        let mut vec = Vec::with_capacity(self.len());
        vec.extend_from_slice(&self);
        vec
    }
}

////////////////////////

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn basic() {
        let mut buf = Buffer::new(10);
        assert_eq!(buf.len(), 10);
        buf[0] = 1;
        assert_eq!(buf[0], 1);
    }
}
<|endoftext|>


<commit_before>use std::io::{BufRead, BufReader, Write};
use std::net::{Shutdown, TcpStream};
use std::sync::{Arc, Mutex};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

// use command::Command;
use message::Message;
use server::Server;

// use command::parse_command;

pub fn run(stream: TcpStream, server: &Arc<Mutex<Server>>) {
    println!("New client");
    Server::welcome_user(clone_stream(&stream));
    let mut client = Client::new(clone_stream(&stream));
    choose_username(&mut client, server);
    Server::display_instructions(clone_stream(&stream));
    chat(client, server);
}

pub struct Client {
    username: String,
    stream: TcpStream,
    sndr: Sender<String>,
    rcvr: Receiver<String>,
}

impl Client {
    pub fn new(stream: TcpStream) -> Self {
        let (sndr, rcvr) = channel();
        Client {
            username: "".to_string(),
            stream: stream,
            sndr: sndr,
            rcvr: rcvr,
        }
    }
}

fn choose_username(client: &mut Client, server: &Arc<Mutex<Server>>) {
    let mut reader = BufReader::new(clone_stream(&client.stream));
    let mut username = "".to_string();

    loop {
        match reader.read_line(&mut username) {
            Ok(_) => {
                username = username.trim().to_string().to_lowercase();

                if server.lock().unwrap().add_user(username.to_string()) {
                    client.username = username.to_string();
                    break
                }
                else {
                    let mut stream = clone_stream(&client.stream);
                    let invalid_msg = "Invalid username. Please try again.\n".to_string();
                    stream.write(&invalid_msg.into_bytes())
                          .expect("Error writing to stream");
                    username = "".to_string();
                }
            },
            Err(e) => println!("{}", e)
        }
    }
}

fn choose_chatroom(client: &Client, server: &Arc<Mutex<Server>>) -> usize {
    // Display rooms and give option to create new room
    let mut stream = clone_stream(&client.stream);
    let msg = "Available rooms:\n".to_string();
    stream.write(&msg.into_bytes()).expect("Error writing to stream");

    server.lock().unwrap().display_rooms(clone_stream(&stream));

    let msg = "\nEnter the room number of the room you wish to join, \
                or type \"new\".\n".to_string();
    stream.write(&msg.into_bytes()).expect("Error writing to stream");

    // If create new room, call server.create_room
    let mut reader = BufReader::new(clone_stream(&stream));
    let mut choice = "".to_string();
    loop {
        match reader.read_line(&mut choice) {
            Ok(_) => {
                choice = choice.trim().to_string().to_lowercase();

                if choice == "new".to_string() {
                    let msg = "Enter the name of your room: ".to_string();
                    stream.write(&msg.into_bytes()).expect("Error writing to stream");
                    let mut room_name = "".to_string();
                    if let Ok(_) = reader.read_line(&mut room_name) {
                        let mut server = server.lock().unwrap();
                        let room_num = server.create_room(room_name.trim().to_string());
                        server.join_room(room_num, client.username.to_string(),
                                         client.sndr.clone());
                        return room_num;
                    }
                }
                else if let Ok(room_num) = choice.parse() {
                    if server.lock().unwrap()
                            .join_room(room_num, client.username.to_string(),
                                       client.sndr.clone()) {
                        return room_num;
                    }
                }
                let msg = "Try again\n".to_string();
                choice = "".to_string();
                stream.write(&msg.into_bytes()).expect("Error writing to stream");
            },
            Err(e) => println!("{}", e)
        }
    }
}

fn chat(client: Client, server: &Arc<Mutex<Server>>) {

    let room_num = choose_chatroom(&client, server);

    // Send messages
    {
        let username = client.username.to_string();
        let reader = BufReader::new(clone_stream(&client.stream));
        let server = server.clone();

        thread::spawn(move|| {
            let mut lines = reader.lines();
            while let Some(Ok(line)) = lines.next() {
                if line.len() > 0 {
                    let msg = Message::new(username.to_string(),
                                           room_num,
                                           line.to_string());
                    server.lock().unwrap().send_message(msg);
                }
            }
        });
    }

    // Receive Messages
    {
        thread::spawn(move|| {
            let mut stream = clone_stream(&client.stream);
            loop {
                match client.rcvr.recv() {
                    Ok(msg) => {
                        stream.write(&msg.into_bytes())
                              .expect("Error writing to stream");
                    },

                    Err(e) => {
                        println!("Error receiving message {}", e);
                        break;
                    }
                }
            }
        });
    }
}

fn clone_stream(stream: &TcpStream) -> TcpStream {
    stream.try_clone().expect("Error cloning stream")
}

#[cfg(test)]
mod client_tests {

}
<commit_msg>Move functions into Client impl block<commit_after>use std::io::{BufRead, BufReader, Write};
use std::net::{Shutdown, TcpStream};
use std::sync::{Arc, Mutex};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

// use command::Command;
use message::Message;
use server::Server;

// use command::parse_command;

pub fn run(stream: TcpStream, server: &Arc<Mutex<Server>>) {
    println!("New client");
    Server::welcome_user(clone_stream(&stream));
    let mut client = Client::new(clone_stream(&stream));
    client.choose_username(server);
    Server::display_instructions(clone_stream(&stream));
    client.chat(server);
}

struct Client {
    username: String,
    stream: TcpStream,
    sndr: Sender<String>,
    rcvr: Receiver<String>,
}

impl Client {
    fn new(stream: TcpStream) -> Self {
        let (sndr, rcvr) = channel();
        Client {
            username: "".to_string(),
            stream: stream,
            sndr: sndr,
            rcvr: rcvr,
        }
    }

    fn choose_username(&mut self, server: &Arc<Mutex<Server>>) {
        let mut reader = BufReader::new(clone_stream(&self.stream));
        let mut username = "".to_string();

        while self.username.len() == 0 {
            match reader.read_line(&mut username) {
                Ok(_) => {
                    username = username.trim().to_string().to_lowercase();

                    if server.lock().unwrap().add_user(username.to_string()) {
                        self.username = username.to_string();
                    }
                    else {
                        let mut stream = clone_stream(&self.stream);
                        let invalid_msg = "Invalid username. Please try again.\n".to_string();
                        stream.write(&invalid_msg.into_bytes())
                              .expect("Error writing to stream");
                        username = "".to_string();
                    }
                },
                Err(e) => println!("{}", e)
            }
        }
    }

    fn choose_chatroom(&self, server: &Arc<Mutex<Server>>) -> usize {
        let mut stream = clone_stream(&self.stream);
        let msg = "Available rooms:\n".to_string();
        stream.write(&msg.into_bytes()).expect("Error writing to stream");

        server.lock().unwrap().display_rooms(clone_stream(&stream));

        let msg = "\nEnter the room number of the room you wish to join, \
                    or type \"new\".\n".to_string();
        stream.write(&msg.into_bytes()).expect("Error writing to stream");

        let mut reader = BufReader::new(clone_stream(&stream));
        let mut choice = "".to_string();
        loop {
            match reader.read_line(&mut choice) {
                Ok(_) => {
                    choice = choice.trim().to_string().to_lowercase();

                    if choice == "new".to_string() {
                        let msg = "Enter the name of your room: ".to_string();
                        stream.write(&msg.into_bytes()).expect("Error writing to stream");
                        let mut room_name = "".to_string();
                        if let Ok(_) = reader.read_line(&mut room_name) {
                            let mut server = server.lock().unwrap();
                            let room_num = server.create_room(room_name.trim().to_string());
                            server.join_room(room_num, self.username.to_string(),
                                             self.sndr.clone());
                            return room_num;
                        }
                    }
                    else if let Ok(room_num) = choice.parse() {
                        if server.lock().unwrap()
                                .join_room(room_num, self.username.to_string(),
                                           self.sndr.clone()) {
                            return room_num;
                        }
                    }
                    let msg = "Try again\n".to_string();
                    choice = "".to_string();
                    stream.write(&msg.into_bytes()).expect("Error writing to stream");
                },
                Err(e) => println!("{}", e)
            }
        }
    }

    fn chat(self, server: &Arc<Mutex<Server>>) {

        let room_num = self.choose_chatroom(server);

        // Send messages
        {
            let username = self.username.to_string();
            let reader = BufReader::new(clone_stream(&self.stream));
            let server = server.clone();

            thread::spawn(move|| {
                let mut lines = reader.lines();
                while let Some(Ok(line)) = lines.next() {
                    if line.len() > 0 {
                        let msg = Message::new(username.to_string(),
                                               room_num,
                                               line.to_string());
                        server.lock().unwrap().send_message(msg);
                    }
                }
            });
        }

        // Receive Messages
        {
            thread::spawn(move|| {
                let mut stream = clone_stream(&self.stream);
                loop {
                    match self.rcvr.recv() {
                        Ok(msg) => {
                            stream.write(&msg.into_bytes())
                                  .expect("Error writing to stream");
                        },

                        Err(e) => {
                            println!("Error receiving message {}", e);
                            break;
                        }
                    }
                }
            });
        }
    }

}

fn clone_stream(stream: &TcpStream) -> TcpStream {
    stream.try_clone().expect("Error cloning stream")
}

#[cfg(test)]
mod client_tests {

}
<|endoftext|>


<commit_before>#[feature(macro_rules)];

use std::io::{file_writer, Create, Truncate};
use std::os;

pub mod branchify;
pub mod status;
pub mod read_method;

fn main() {
    let args = os::args();
    match args.len() {
        0 => {
            println("usage: codegen [read_method|status].rs");
            os::set_exit_status(1);
        },
        3 => {
            let output_dir = from_str(args[2]).unwrap();
            // TODO: maybe not 0777?
            os::make_dir(&output_dir, 0b111_111_111);

            match args[1] {
                ~"read_method.rs" => read_method::generate(&output_dir),
                ~"status.rs" => status::generate(&output_dir),
                s => {
                    println!("unknown thing-to-generate '{}'", s);
                    os::set_exit_status(1);
                }
            }
        },
        _ => {
            println!("usage: {} [read_method|status].rs", args[0]);
            os::set_exit_status(1);
        }
    }
}

pub fn get_writer(output_dir: &Path, filename: &str) -> @Writer {
    let mut output_dir = output_dir.clone();
    output_dir.push(filename);
    match file_writer(&output_dir, [Create, Truncate]) {
        Ok(writer) => writer,
        Err(msg) => fail!("Unable to write file: %s", msg),
    }
}
<commit_msg>Switch from old-style fail! to new-style. (I'll leave it named fail2! until next snapshot).<commit_after>#[feature(macro_rules)];

use std::io::{file_writer, Create, Truncate};
use std::os;

pub mod branchify;
pub mod status;
pub mod read_method;

fn main() {
    let args = os::args();
    match args.len() {
        0 => {
            println("usage: codegen [read_method|status].rs");
            os::set_exit_status(1);
        },
        3 => {
            let output_dir = from_str(args[2]).unwrap();
            // TODO: maybe not 0777?
            os::make_dir(&output_dir, 0b111_111_111);

            match args[1] {
                ~"read_method.rs" => read_method::generate(&output_dir),
                ~"status.rs" => status::generate(&output_dir),
                s => {
                    println!("unknown thing-to-generate '{}'", s);
                    os::set_exit_status(1);
                }
            }
        },
        _ => {
            println!("usage: {} [read_method|status].rs", args[0]);
            os::set_exit_status(1);
        }
    }
}

pub fn get_writer(output_dir: &Path, filename: &str) -> @Writer {
    let mut output_dir = output_dir.clone();
    output_dir.push(filename);
    match file_writer(&output_dir, [Create, Truncate]) {
        Ok(writer) => writer,
        Err(msg) => fail2!("Unable to write file: {}", msg),
    }
}
<|endoftext|>


<commit_before>// In heavy WIP
use std::fs::File;
use std::io::Read;
use std::path::Path;
use toml;

use error::PackageError;

/// Represents the config file.
#[derive(Deserialize)]
struct Config {
    buffer_size: Option<u64>,
    config_path: Option<String>,
    download_path: Option<String>,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            buffer_size: Some(65536),
            config_path: Some(String::from("~/.wiz/config/")),
            download_path: Some(String::from("~/.wiz/downloads/")),
        }
    }
}

impl Config {
    /// Read the config file from the config path specified in `path`.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from(path: &Path) -> Result<Self, PackageError> {
        // Read the config file into a string
        let mut config = File::open(path)?;
        let mut content = String::new();
        config.read_to_string(&mut content)?;

        // Try parsing the string, and convert it into a `Config`.
        let config = content.parse::<toml::Value>()?;
        let config = config.try_into::<Self>()?;
        Ok(config)
    }

    /// Read the config file from the default config path.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from_default() -> Result<Self, PackageError> {
        if let Self { config_path: Some(path), .. } = Self::default() {
            Self::read_from(Path::new(&path))
        } else {
            // This should never, ever, happen.
            Err(PackageError::Parsing(
                String::from("Unable to get config_path from the default cfg.")
            ))
        }
    }

}<commit_msg>Making the config fields public<commit_after>// In heavy WIP
use std::fs::File;
use std::io::Read;
use std::path::Path;
use toml;

use error::PackageError;

/// Represents the config file.
#[derive(Deserialize)]
pub struct Config {
    pub buffer_size: Option<u64>,
    pub config_path: Option<String>,
    pub download_path: Option<String>,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            buffer_size: Some(65536),
            config_path: Some(String::from("~/.wiz/config/")),
            download_path: Some(String::from("~/.wiz/downloads/")),
        }
    }
}

impl Config {
    /// Read the config file from the config path specified in `path`.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from(path: &Path) -> Result<Self, PackageError> {
        // Read the config file into a string
        let mut config = File::open(path)?;
        let mut content = String::new();
        config.read_to_string(&mut content)?;

        // Try parsing the string, and convert it into a `Config`.
        let config = content.parse::<toml::Value>()?;
        let config = config.try_into::<Self>()?;
        Ok(config)
    }

    /// Read the config file from the default config path.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from_default() -> Result<Self, PackageError> {
        if let Self { config_path: Some(path), .. } = Self::default() {
            Self::read_from(Path::new(&path))
        } else {
            // This should never, ever, happen.
            Err(PackageError::Parsing(
                String::from("Unable to get config_path from the default cfg.")
            ))
        }
    }

}<|endoftext|>


<commit_before>use std::path::{Path, PathBuf};
use std::fs::File;
use std::io::Read;
use toml::{self, Value};
use toml::value::Table;
use serde::{Deserialize, Deserializer};

use errors::*;

/// The overall configuration object for MDBook.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Config {
    /// Metadata about the book.
    pub book: BookConfig,
    rest: Table,
}

impl Config {
    /// Load a `Config` from some string.
    pub fn from_str(src: &str) -> Result<Config> {
        toml::from_str(src).chain_err(|| Error::from("Invalid configuration file"))
    }

    /// Load the configuration file from disk.
    pub fn from_disk<P: AsRef<Path>>(config_file: P) -> Result<Config> {
        let mut buffer = String::new();
        File::open(config_file).chain_err(|| "Unable to open the configuration file")?
                               .read_to_string(&mut buffer)
                               .chain_err(|| "Couldn't read the file")?;

        Config::from_str(&buffer)
    }

    /// Fetch an arbitrary item from the `Config` as a `toml::Value`.
    ///
    /// You can use dotted indices to access nested items (e.g.
    /// `output.html.playpen` will fetch the "playpen" out of the html output
    /// table).
    pub fn get(&self, key: &str) -> Option<&Value> {
        let pieces: Vec<_> = key.split(".").collect();
        recursive_get(&pieces, &self.rest)
    }

    /// Fetch a value from the `Config` so you can mutate it.
    pub fn get_mut<'a>(&'a mut self, key: &str) -> Option<&'a mut Value> {
        let pieces: Vec<_> = key.split(".").collect();
        recursive_get_mut(&pieces, &mut self.rest)
    }

    /// Convenience method for getting the html renderer's configuration.
    ///
    /// # Note
    ///
    /// This is for compatibility only. It will be removed completely once the
    /// rendering and plugin system is established.
    pub fn html_config(&self) -> Option<HtmlConfig> {
        self.get_deserialized("output.html").ok()
    }

    /// Convenience function to fetch a value from the config and deserialize it
    /// into some arbitrary type.
    pub fn get_deserialized<'de, T: Deserialize<'de>, S: AsRef<str>>(&self, name: S) -> Result<T> {
        let name = name.as_ref();

        if let Some(value) = self.get(name) {
            value.clone()
                 .try_into()
                 .chain_err(|| "Couldn't deserialize the value")
        } else {
            bail!("Key not found, {:?}", name)
        }
    }

    fn from_legacy(mut table: Table) -> Config {
        let mut cfg = Config::default();

        // we use a macro here instead of a normal loop because the $out
        // variable can be different types. This way we can make type inference
        // figure out what try_into() deserializes to.
        macro_rules! get_and_insert {
            ($table:expr, $key:expr => $out:expr) => {
                if let Some(value) = $table.remove($key).and_then(|v| v.try_into().ok()) {
                    $out = value;
                }
            };
        }

        get_and_insert!(table, "title" => cfg.book.title);
        get_and_insert!(table, "authors" => cfg.book.authors);
        get_and_insert!(table, "source" => cfg.book.src);
        get_and_insert!(table, "description" => cfg.book.description);

        // This complicated chain of and_then's is so we can move
        // "output.html.destination" to "book.build_dir" and parse it into a
        // PathBuf.
        let destination: Option<PathBuf> = table.get_mut("output")
            .and_then(|output| output.as_table_mut())
            .and_then(|output| output.get_mut("html"))
            .and_then(|html| html.as_table_mut())
            .and_then(|html| html.remove("destination"))
            .and_then(|dest| dest.try_into().ok());

        if let Some(dest) = destination {
            cfg.book.build_dir = dest;
        }

        cfg.rest = table;
        cfg
    }
}

fn recursive_get<'a>(key: &[&str], table: &'a Table) -> Option<&'a Value> {
    if key.is_empty() {
        return None;
    } else if key.len() == 1 {
        return table.get(key[0]);
    }

    let first = key[0];
    let rest = &key[1..];

    if let Some(&Value::Table(ref nested)) = table.get(first) {
        recursive_get(rest, nested)
    } else {
        None
    }
}

fn recursive_get_mut<'a>(key: &[&str], table: &'a mut Table) -> Option<&'a mut Value> {
    // TODO: Figure out how to abstract over mutability to reduce copy-pasta
    if key.is_empty() {
        return None;
    } else if key.len() == 1 {
        return table.get_mut(key[0]);
    }

    let first = key[0];
    let rest = &key[1..];

    if let Some(&mut Value::Table(ref mut nested)) = table.get_mut(first) {
        recursive_get_mut(rest, nested)
    } else {
        None
    }
}

impl<'de> Deserialize<'de> for Config {
    fn deserialize<D: Deserializer<'de>>(de: D) -> ::std::result::Result<Self, D::Error> {
        let raw = Value::deserialize(de)?;

        let mut table = match raw {
            Value::Table(t) => t,
            _ => {
                use serde::de::Error;
                return Err(D::Error::custom(
                    "A config file should always be a toml table",
                ));
            }
        };

        if is_legacy_format(&table) {
            warn!("It looks like you are using the legacy book.toml format.");
            warn!("We'll parse it for now, but you should probably convert to the new format.");
            warn!("See the mdbook documentation for more details");
            warn!("http://azerupi.github.io/mdBook/format/config.html");
            return Ok(Config::from_legacy(table));
        }

        let book: BookConfig = table.remove("book")
                                    .and_then(|value| value.try_into().ok())
                                    .unwrap_or_default();
        Ok(Config {
            book: book,
            rest: table,
        })
    }
}

fn is_legacy_format(table: &Table) -> bool {
    let top_level_items = ["title", "author", "authors"];

    top_level_items.iter().any(|key| table.contains_key(&key.to_string()))
}

/// Configuration options which are specific to the book and required for
/// loading it from disk.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "kebab-case")]
pub struct BookConfig {
    /// The book's title.
    pub title: Option<String>,
    /// The book's authors.
    pub authors: Vec<String>,
    /// An optional description for the book.
    pub description: Option<String>,
    /// Location of the book source relative to the book's root directory.
    pub src: PathBuf,
    /// Where to put built artefacts relative to the book's root directory.
    pub build_dir: PathBuf,
    /// Does this book support more than one language?
    pub multilingual: bool,
}

impl Default for BookConfig {
    fn default() -> BookConfig {
        BookConfig {
            title: None,
            authors: Vec::new(),
            description: None,
            src: PathBuf::from("src"),
            build_dir: PathBuf::from("book"),
            multilingual: false,
        }
    }
}

#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "kebab-case")]
pub struct HtmlConfig {
    pub theme: Option<PathBuf>,
    pub curly_quotes: bool,
    pub mathjax_support: bool,
    pub google_analytics: Option<String>,
    pub additional_css: Vec<PathBuf>,
    pub additional_js: Vec<PathBuf>,
    pub playpen: Playpen,
}

/// Configuration for tweaking how the the HTML renderer handles the playpen.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Playpen {
    pub editor: PathBuf,
    pub editable: bool,
}

#[cfg(test)]
mod tests {
    use super::*;

    const COMPLEX_CONFIG: &'static str = r#"
        [book]
        title = "Some Book"
        authors = ["Michael-F-Bryan <michaelfbryan@gmail.com>"]
        description = "A completely useless book"
        multilingual = true
        src = "source"
        build-dir = "outputs"

        [output.html]
        theme = "./themedir"
        curly-quotes = true
        google-analytics = "123456"
        additional-css = ["./foo/bar/baz.css"]

        [output.html.playpen]
        editable = true
        editor = "ace"
        "#;

    #[test]
    fn load_a_complex_config_file() {
        let src = COMPLEX_CONFIG;

        let book_should_be = BookConfig {
            title: Some(String::from("Some Book")),
            authors: vec![String::from("Michael-F-Bryan <michaelfbryan@gmail.com>")],
            description: Some(String::from("A completely useless book")),
            multilingual: true,
            src: PathBuf::from("source"),
            build_dir: PathBuf::from("outputs"),
            ..Default::default()
        };
        let playpen_should_be = Playpen {
            editable: true,
            editor: PathBuf::from("ace"),
        };
        let html_should_be = HtmlConfig {
            curly_quotes: true,
            google_analytics: Some(String::from("123456")),
            additional_css: vec![PathBuf::from("./foo/bar/baz.css")],
            theme: Some(PathBuf::from("./themedir")),
            playpen: playpen_should_be,
            ..Default::default()
        };

        let got = Config::from_str(src).unwrap();

        assert_eq!(got.book, book_should_be);
        assert_eq!(got.html_config().unwrap(), html_should_be);
    }

    #[test]
    fn load_arbitrary_output_type() {
        #[derive(Debug, Deserialize, PartialEq)]
        struct RandomOutput {
            foo: u32,
            bar: String,
            baz: Vec<bool>,
        }

        let src = r#"
        [output.random]
        foo = 5
        bar = "Hello World"
        baz = [true, true, false]
        "#;

        let should_be = RandomOutput {
            foo: 5,
            bar: String::from("Hello World"),
            baz: vec![true, true, false],
        };

        let cfg = Config::from_str(src).unwrap();
        let got: RandomOutput = cfg.get_deserialized("output.random").unwrap();

        assert_eq!(got, should_be);

        let baz: Vec<bool> = cfg.get_deserialized("output.random.baz").unwrap();
        let baz_should_be = vec![true, true, false];

        assert_eq!(baz, baz_should_be);
    }

    #[test]
    fn mutate_some_stuff() {
        // really this is just a sanity check to make sure the borrow checker
        // is happy...
        let src = COMPLEX_CONFIG;
        let mut config = Config::from_str(src).unwrap();
        let key = "output.html.playpen.editable";

        assert_eq!(config.get(key).unwrap(), &Value::Boolean(true));
        *config.get_mut(key).unwrap() = Value::Boolean(false);
        assert_eq!(config.get(key).unwrap(), &Value::Boolean(false));
    }

    /// The config file format has slightly changed (metadata stuff is now under
    /// the `book` table instead of being at the top level) so we're adding a
    /// **temporary** compatibility check. You should be able to still load the
    /// old format, emitting a warning.
    #[test]
    fn can_still_load_the_previous_format() {
        let src = r#"
        title = "mdBook Documentation"
        description = "Create book from markdown files. Like Gitbook but implemented in Rust"
        authors = ["Mathieu David"]
        source = "./source"

        [output.html]
        destination = "my-book" # the output files will be generated in `root/my-book` instead of `root/book`
        theme = "my-theme"
        curly-quotes = true
        google-analytics = "123456"
        additional-css = ["custom.css", "custom2.css"]
        additional-js = ["custom.js"]
        "#;

        let book_should_be = BookConfig {
            title: Some(String::from("mdBook Documentation")),
            description: Some(String::from(
                "Create book from markdown files. Like Gitbook but implemented in Rust",
            )),
            authors: vec![String::from("Mathieu David")],
            build_dir: PathBuf::from("my-book"),
            src: PathBuf::from("./source"),
            ..Default::default()
        };

        let html_should_be = HtmlConfig {
            theme: Some(PathBuf::from("my-theme")),
            curly_quotes: true,
            google_analytics: Some(String::from("123456")),
            additional_css: vec![PathBuf::from("custom.css"), PathBuf::from("custom2.css")],
            additional_js: vec![PathBuf::from("custom.js")],
            ..Default::default()
        };

        let got = Config::from_str(src).unwrap();
        assert_eq!(got.book, book_should_be);
        assert_eq!(got.html_config().unwrap(), html_should_be);
    }
}
<commit_msg>Updated the warning to give some basic migration instructions<commit_after>use std::path::{Path, PathBuf};
use std::fs::File;
use std::io::Read;
use toml::{self, Value};
use toml::value::Table;
use serde::{Deserialize, Deserializer};

use errors::*;

/// The overall configuration object for MDBook.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Config {
    /// Metadata about the book.
    pub book: BookConfig,
    rest: Table,
}

impl Config {
    /// Load a `Config` from some string.
    pub fn from_str(src: &str) -> Result<Config> {
        toml::from_str(src).chain_err(|| Error::from("Invalid configuration file"))
    }

    /// Load the configuration file from disk.
    pub fn from_disk<P: AsRef<Path>>(config_file: P) -> Result<Config> {
        let mut buffer = String::new();
        File::open(config_file).chain_err(|| "Unable to open the configuration file")?
                               .read_to_string(&mut buffer)
                               .chain_err(|| "Couldn't read the file")?;

        Config::from_str(&buffer)
    }

    /// Fetch an arbitrary item from the `Config` as a `toml::Value`.
    ///
    /// You can use dotted indices to access nested items (e.g.
    /// `output.html.playpen` will fetch the "playpen" out of the html output
    /// table).
    pub fn get(&self, key: &str) -> Option<&Value> {
        let pieces: Vec<_> = key.split(".").collect();
        recursive_get(&pieces, &self.rest)
    }

    /// Fetch a value from the `Config` so you can mutate it.
    pub fn get_mut<'a>(&'a mut self, key: &str) -> Option<&'a mut Value> {
        let pieces: Vec<_> = key.split(".").collect();
        recursive_get_mut(&pieces, &mut self.rest)
    }

    /// Convenience method for getting the html renderer's configuration.
    ///
    /// # Note
    ///
    /// This is for compatibility only. It will be removed completely once the
    /// rendering and plugin system is established.
    pub fn html_config(&self) -> Option<HtmlConfig> {
        self.get_deserialized("output.html").ok()
    }

    /// Convenience function to fetch a value from the config and deserialize it
    /// into some arbitrary type.
    pub fn get_deserialized<'de, T: Deserialize<'de>, S: AsRef<str>>(&self, name: S) -> Result<T> {
        let name = name.as_ref();

        if let Some(value) = self.get(name) {
            value.clone()
                 .try_into()
                 .chain_err(|| "Couldn't deserialize the value")
        } else {
            bail!("Key not found, {:?}", name)
        }
    }

    fn from_legacy(mut table: Table) -> Config {
        let mut cfg = Config::default();

        // we use a macro here instead of a normal loop because the $out
        // variable can be different types. This way we can make type inference
        // figure out what try_into() deserializes to.
        macro_rules! get_and_insert {
            ($table:expr, $key:expr => $out:expr) => {
                if let Some(value) = $table.remove($key).and_then(|v| v.try_into().ok()) {
                    $out = value;
                }
            };
        }

        get_and_insert!(table, "title" => cfg.book.title);
        get_and_insert!(table, "authors" => cfg.book.authors);
        get_and_insert!(table, "source" => cfg.book.src);
        get_and_insert!(table, "description" => cfg.book.description);

        // This complicated chain of and_then's is so we can move
        // "output.html.destination" to "book.build_dir" and parse it into a
        // PathBuf.
        let destination: Option<PathBuf> = table.get_mut("output")
            .and_then(|output| output.as_table_mut())
            .and_then(|output| output.get_mut("html"))
            .and_then(|html| html.as_table_mut())
            .and_then(|html| html.remove("destination"))
            .and_then(|dest| dest.try_into().ok());

        if let Some(dest) = destination {
            cfg.book.build_dir = dest;
        }

        cfg.rest = table;
        cfg
    }
}

fn recursive_get<'a>(key: &[&str], table: &'a Table) -> Option<&'a Value> {
    if key.is_empty() {
        return None;
    } else if key.len() == 1 {
        return table.get(key[0]);
    }

    let first = key[0];
    let rest = &key[1..];

    if let Some(&Value::Table(ref nested)) = table.get(first) {
        recursive_get(rest, nested)
    } else {
        None
    }
}

fn recursive_get_mut<'a>(key: &[&str], table: &'a mut Table) -> Option<&'a mut Value> {
    // TODO: Figure out how to abstract over mutability to reduce copy-pasta
    if key.is_empty() {
        return None;
    } else if key.len() == 1 {
        return table.get_mut(key[0]);
    }

    let first = key[0];
    let rest = &key[1..];

    if let Some(&mut Value::Table(ref mut nested)) = table.get_mut(first) {
        recursive_get_mut(rest, nested)
    } else {
        None
    }
}

impl<'de> Deserialize<'de> for Config {
    fn deserialize<D: Deserializer<'de>>(de: D) -> ::std::result::Result<Self, D::Error> {
        let raw = Value::deserialize(de)?;

        let mut table = match raw {
            Value::Table(t) => t,
            _ => {
                use serde::de::Error;
                return Err(D::Error::custom(
                    "A config file should always be a toml table",
                ));
            }
        };

        if is_legacy_format(&table) {
            warn!("It looks like you are using the legacy book.toml format.");
            warn!("We'll parse it for now, but you should probably convert to the new format.");
            warn!("See the mdbook documentation for more details, although as a rule of thumb");
            warn!("just move all top level configuration entries like `title`, `author` and ");
            warn!("`description` under a table called `[book]` and it should all work.");
            warn!("Documentation: http://rust-lang-nursery.github.io/mdBook/format/config.html");
            return Ok(Config::from_legacy(table));
        }

        let book: BookConfig = table.remove("book")
                                    .and_then(|value| value.try_into().ok())
                                    .unwrap_or_default();
        Ok(Config {
            book: book,
            rest: table,
        })
    }
}

fn is_legacy_format(table: &Table) -> bool {
    let top_level_items = ["title", "author", "authors"];

    top_level_items.iter().any(|key| table.contains_key(&key.to_string()))
}

/// Configuration options which are specific to the book and required for
/// loading it from disk.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "kebab-case")]
pub struct BookConfig {
    /// The book's title.
    pub title: Option<String>,
    /// The book's authors.
    pub authors: Vec<String>,
    /// An optional description for the book.
    pub description: Option<String>,
    /// Location of the book source relative to the book's root directory.
    pub src: PathBuf,
    /// Where to put built artefacts relative to the book's root directory.
    pub build_dir: PathBuf,
    /// Does this book support more than one language?
    pub multilingual: bool,
}

impl Default for BookConfig {
    fn default() -> BookConfig {
        BookConfig {
            title: None,
            authors: Vec::new(),
            description: None,
            src: PathBuf::from("src"),
            build_dir: PathBuf::from("book"),
            multilingual: false,
        }
    }
}

#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "kebab-case")]
pub struct HtmlConfig {
    pub theme: Option<PathBuf>,
    pub curly_quotes: bool,
    pub mathjax_support: bool,
    pub google_analytics: Option<String>,
    pub additional_css: Vec<PathBuf>,
    pub additional_js: Vec<PathBuf>,
    pub playpen: Playpen,
}

/// Configuration for tweaking how the the HTML renderer handles the playpen.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Playpen {
    pub editor: PathBuf,
    pub editable: bool,
}

#[cfg(test)]
mod tests {
    use super::*;

    const COMPLEX_CONFIG: &'static str = r#"
        [book]
        title = "Some Book"
        authors = ["Michael-F-Bryan <michaelfbryan@gmail.com>"]
        description = "A completely useless book"
        multilingual = true
        src = "source"
        build-dir = "outputs"

        [output.html]
        theme = "./themedir"
        curly-quotes = true
        google-analytics = "123456"
        additional-css = ["./foo/bar/baz.css"]

        [output.html.playpen]
        editable = true
        editor = "ace"
        "#;

    #[test]
    fn load_a_complex_config_file() {
        let src = COMPLEX_CONFIG;

        let book_should_be = BookConfig {
            title: Some(String::from("Some Book")),
            authors: vec![String::from("Michael-F-Bryan <michaelfbryan@gmail.com>")],
            description: Some(String::from("A completely useless book")),
            multilingual: true,
            src: PathBuf::from("source"),
            build_dir: PathBuf::from("outputs"),
            ..Default::default()
        };
        let playpen_should_be = Playpen {
            editable: true,
            editor: PathBuf::from("ace"),
        };
        let html_should_be = HtmlConfig {
            curly_quotes: true,
            google_analytics: Some(String::from("123456")),
            additional_css: vec![PathBuf::from("./foo/bar/baz.css")],
            theme: Some(PathBuf::from("./themedir")),
            playpen: playpen_should_be,
            ..Default::default()
        };

        let got = Config::from_str(src).unwrap();

        assert_eq!(got.book, book_should_be);
        assert_eq!(got.html_config().unwrap(), html_should_be);
    }

    #[test]
    fn load_arbitrary_output_type() {
        #[derive(Debug, Deserialize, PartialEq)]
        struct RandomOutput {
            foo: u32,
            bar: String,
            baz: Vec<bool>,
        }

        let src = r#"
        [output.random]
        foo = 5
        bar = "Hello World"
        baz = [true, true, false]
        "#;

        let should_be = RandomOutput {
            foo: 5,
            bar: String::from("Hello World"),
            baz: vec![true, true, false],
        };

        let cfg = Config::from_str(src).unwrap();
        let got: RandomOutput = cfg.get_deserialized("output.random").unwrap();

        assert_eq!(got, should_be);

        let baz: Vec<bool> = cfg.get_deserialized("output.random.baz").unwrap();
        let baz_should_be = vec![true, true, false];

        assert_eq!(baz, baz_should_be);
    }

    #[test]
    fn mutate_some_stuff() {
        // really this is just a sanity check to make sure the borrow checker
        // is happy...
        let src = COMPLEX_CONFIG;
        let mut config = Config::from_str(src).unwrap();
        let key = "output.html.playpen.editable";

        assert_eq!(config.get(key).unwrap(), &Value::Boolean(true));
        *config.get_mut(key).unwrap() = Value::Boolean(false);
        assert_eq!(config.get(key).unwrap(), &Value::Boolean(false));
    }

    /// The config file format has slightly changed (metadata stuff is now under
    /// the `book` table instead of being at the top level) so we're adding a
    /// **temporary** compatibility check. You should be able to still load the
    /// old format, emitting a warning.
    #[test]
    fn can_still_load_the_previous_format() {
        let src = r#"
        title = "mdBook Documentation"
        description = "Create book from markdown files. Like Gitbook but implemented in Rust"
        authors = ["Mathieu David"]
        source = "./source"

        [output.html]
        destination = "my-book" # the output files will be generated in `root/my-book` instead of `root/book`
        theme = "my-theme"
        curly-quotes = true
        google-analytics = "123456"
        additional-css = ["custom.css", "custom2.css"]
        additional-js = ["custom.js"]
        "#;

        let book_should_be = BookConfig {
            title: Some(String::from("mdBook Documentation")),
            description: Some(String::from(
                "Create book from markdown files. Like Gitbook but implemented in Rust",
            )),
            authors: vec![String::from("Mathieu David")],
            build_dir: PathBuf::from("my-book"),
            src: PathBuf::from("./source"),
            ..Default::default()
        };

        let html_should_be = HtmlConfig {
            theme: Some(PathBuf::from("my-theme")),
            curly_quotes: true,
            google_analytics: Some(String::from("123456")),
            additional_css: vec![PathBuf::from("custom.css"), PathBuf::from("custom2.css")],
            additional_js: vec![PathBuf::from("custom.js")],
            ..Default::default()
        };

        let got = Config::from_str(src).unwrap();
        assert_eq!(got.book, book_should_be);
        assert_eq!(got.html_config().unwrap(), html_should_be);
    }
}
<|endoftext|>