<commit_before>//! The global allocator.
//!
//! This contains primitives for the cross-thread allocator.

use prelude::*;

use core::{mem, ops};

use bookkeeper::{self, Allocator, Bookkeeper};
use {brk, sync};

use shim::config;

#[cfg(feature = "tls")]
use tls;

/// Alias for the wrapper type of the thread-local variable holding the local
/// allocator.
#[cfg(feature = "tls")]
type ThreadLocalAllocator =
    MoveCell<Option<LazyInit<fn() -> LocalAllocator, LocalAllocator>>>;

/// The global default allocator.
// TODO: Remove these filthy function pointers.
static GLOBAL_ALLOCATOR: sync::Mutex<
    LazyInit<fn() -> GlobalAllocator, GlobalAllocator>,
> = sync::Mutex::new(LazyInit::new(GlobalAllocator::init));
#[cfg(feature = "tls")]
tls! {
    /// The thread-local allocator.
    static THREAD_ALLOCATOR: ThreadLocalAllocator = MoveCell::new(Some(LazyInit::new(LocalAllocator::init)));
}

/// Temporarily get the allocator.
///
/// This is simply to avoid repeating ourself, so we let this take care of the
/// hairy stuff:
///
/// 1. Initialize the allocator if needed.
/// 2. If the allocator is not yet initialized, fallback to the global
/// allocator. 3. Unlock/move temporarily out of reference.
///
/// This is a macro due to the lack of generic closure, which makes it
/// impossible to have one closure for both cases (global and local).
// TODO: Instead of falling back to the global allocator, the thread dtor
// should be set such that it run after the TLS keys that might be declared.
macro_rules! get_allocator {
    (| $v:ident | $b:expr) => {{
        // Get the thread allocator, if TLS is enabled
        #[cfg(feature = "tls")]
        {
            THREAD_ALLOCATOR.with(|thread_alloc| {
                if let Some(mut thread_alloc_original) = thread_alloc.replace(None) {
                    let res = {
                        // Call the closure involved.
                        let $v = thread_alloc_original.get();
                        $b
                    };

                    // Put back the original allocator.
                    thread_alloc.replace(Some(thread_alloc_original));

                    res
                } else {
                    // The local allocator seems to have been deinitialized, for this reason we
                    // fallback to the global allocator.
                    log!(
                        WARNING,
                        "Accessing the allocator after deinitialization of the local allocator."
                    );

                    // Lock the global allocator.
                    let mut guard = GLOBAL_ALLOCATOR.lock();

                    // Call the block in question.
                    let $v = guard.get();
                    $b
                }
            })
        }

        // TLS is disabled, just use the global allocator.
        #[cfg(not(feature = "tls"))]
        {
            // Lock the global allocator.
            let mut guard = GLOBAL_ALLOCATOR.lock();

            // Call the block in question.
            let $v = guard.get();
            $b
        }
    }};
}

/// Derives `Deref` and `DerefMut` to the `inner` field.
///
/// This requires importing `core::ops`.
macro_rules! derive_deref {
    ($imp:ty, $target:ty) => {
        impl ops::Deref for $imp {
            type Target = $target;

            fn deref(&self) -> &$target {
                &self.inner
            }
        }

        impl ops::DerefMut for $imp {
            fn deref_mut(&mut self) -> &mut $target {
                &mut self.inner
            }
        }
    };
}

/// Global SBRK-based allocator.
///
/// This will extend the data segment whenever new memory is needed. Since this
/// includes leaving userspace, this shouldn't be used when other allocators
/// are available (i.e. the bookkeeper is local).
struct GlobalAllocator {
    // The inner bookkeeper.
    inner: Bookkeeper,
}

impl GlobalAllocator {
    /// Initialize the global allocator.
    fn init() -> GlobalAllocator {
        /// Logging...
        log!(NOTE, "Initializing the global allocator.");

        // The initial acquired segment.
        let (aligner, initial_segment, excessive) = brk::lock().canonical_brk(
            4 * bookkeeper::EXTRA_ELEMENTS * mem::size_of::<Block>(),
            mem::align_of::<Block>(),
        );

        // Initialize the new allocator.
        let mut res = GlobalAllocator {
            inner: Bookkeeper::new(unsafe {
                // LAST AUDIT: 2016-08-21 (Ticki).

                Vec::from_raw_parts(initial_segment, 0)
            }),
        };

        // Free the secondary space.
        res.push(aligner);
        res.push(excessive);

        res
    }
}

derive_deref!(GlobalAllocator, Bookkeeper);

impl Allocator for GlobalAllocator {
    #[inline]
    fn alloc_fresh(&mut self, size: usize, align: usize) -> Block {
        // Obtain what you need.
        let (alignment_block, res, excessive) =
            brk::lock().canonical_brk(size, align);

        // Add it to the list. This will not change the order, since the
        // pointer is higher than all the previous blocks (BRK extends
        // the data segment). Although, it is worth noting that the
        // stack is higher than the program break.
        self.push(alignment_block);
        self.push(excessive);

        res
    }

    fn on_new_memory(&mut self) {
        if self.total_bytes() > config::OS_MEMTRIM_LIMIT {
            // memtrim the fack outta 'em.

            // Pop the last block.
            let block = self
                .pop()
                .expect("The byte count on the global allocator is invalid.");

            // Check if the memtrim is worth it.
            if block.size() >= config::OS_MEMTRIM_WORTHY {
                /// Logging...
                log!(NOTE, "Memtrimming the global allocator.");

                // Release the block to the OS.
                if let Err(block) = brk::lock().release(block) {
                    // It failed, put the block back.
                    // TODO: This can be done faster.
                    self.push(block);
                }

            // Note that this block is the only block next to the program
            // break, due to the segments being as long as
            // possible. For that reason, repeating to push and
            // release would fail.
            } else {
                /// Logging...
                log!(WARNING, "Memtrimming for the global allocator failed.");

                // Push the block back.
                // TODO: This can be done faster.
                self.push(block);
            }
        }
    }
}

/// A local allocator.
///
/// This acquires memory from the upstream (global) allocator, which is
/// protected by a `Mutex`.
#[cfg(feature = "tls")]
pub struct LocalAllocator {
    // The inner bookkeeper.
    inner: Bookkeeper,
}

#[cfg(feature = "tls")]
impl LocalAllocator {
    /// Initialize the local allocator.
    #[cfg(feature = "tls")]
    fn init() -> LocalAllocator {
        /// The destructor of the local allocator.
        ///
        /// This will simply free everything to the global allocator.
        extern "C" fn dtor(alloc: &ThreadLocalAllocator) {
            /// Logging...
            log!(NOTE, "Deinitializing and freeing the local allocator.");

            // This is important! The thread destructors guarantee no other,
            // and thus one could use the allocator _after_ this
            // destructor have been finished. In fact, this is a
            // real problem, and happens when using `Arc` and
            // terminating the main thread, for this reason we place
            // `None` as a permanent marker indicating that the allocator is
            // deinitialized. After such a state is in place, all
            // allocation calls will be redirected to the global
            // allocator, which is of course still usable at this
            // moment.
            let alloc = alloc
                .replace(None)
                .expect("Thread-local allocator is already freed.");

            // Lock the global allocator.
            let mut global_alloc = GLOBAL_ALLOCATOR.lock();
            let global_alloc = global_alloc.get();

            // TODO: we know this is sorted, so we could abuse that fact to
            // faster insertion in the global allocator.

            alloc
                .into_inner()
                .inner
                .for_each(move |block| global_alloc.free(block));
        }

        /// Logging...
        log!(NOTE, "Initializing the local allocator.");

        // The initial acquired segment.
        let initial_segment = GLOBAL_ALLOCATOR.lock().get().alloc(
            4 * bookkeeper::EXTRA_ELEMENTS * mem::size_of::<Block>(),
            mem::align_of::<Block>(),
        );

        unsafe {
            // LAST AUDIT: 2016-08-21 (Ticki).

            // Register the thread destructor on the current thread.
            THREAD_ALLOCATOR.register_thread_destructor(dtor);

            LocalAllocator {
                inner: Bookkeeper::new(Vec::from_raw_parts(initial_segment, 0)),
            }
        }
    }
}

#[cfg(feature = "tls")]
derive_deref!(LocalAllocator, Bookkeeper);

#[cfg(feature = "tls")]
impl Allocator for LocalAllocator {
    #[inline]
    fn alloc_fresh(&mut self, size: usize, align: usize) -> Block {
        // Get the block from the global allocator. Please note that we cannot
        // canonicalize `size`, due to freeing excessive blocks would change
        // the order.
        GLOBAL_ALLOCATOR.lock().get().alloc(size, align)
    }

    #[inline]
    fn on_new_memory(&mut self) {
        // The idea is to free memory to the global allocator to unify small
        // stubs and avoid fragmentation and thread accumulation.
        if self.total_bytes() < config::FRAGMENTATION_SCALE * self.len()
            || self.total_bytes() > config::LOCAL_MEMTRIM_LIMIT
        {
            // Log stuff.
            log!(NOTE, "Memtrimming the local allocator.");

            // Lock the global allocator.
            let mut global_alloc = GLOBAL_ALLOCATOR.lock();
            let global_alloc = global_alloc.get();

            while let Some(block) = self.pop() {
                // Pop'n'free.
                global_alloc.free(block);

                // Memtrim 'till we won't memtrim anymore.
                if self.total_bytes() < config::LOCAL_MEMTRIM_STOP {
                    break;
                }
            }
        }
    }
}

/// Allocate a block of memory.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
#[inline]
pub fn alloc(size: usize, align: usize) -> *mut u8 {
    log!(
        CALL,
        "Allocating buffer of size {} (align {}).",
        size,
        align
    );

    get_allocator!(|alloc| Pointer::from(alloc.alloc(size, align)).get())
}

/// Free a buffer.
///
/// Note that this do not have to be a buffer allocated through ralloc. The
/// only requirement is that it is not used after the free.
///
/// # Important!
///
/// You should only allocate buffers allocated through `ralloc`. Anything else
/// is considered invalid.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
///
/// # Safety
///
/// Rust assume that the allocation symbols returns correct values. For this
/// reason, freeing invalid pointers might introduce memory unsafety.
///
/// Secondly, freeing an used buffer can introduce use-after-free.
#[inline]
pub unsafe fn free(ptr: *mut u8, size: usize) {
    log!(CALL, "Freeing buffer of size {}.", size);

    get_allocator!(
        |alloc| alloc.free(Block::from_raw_parts(Pointer::new(ptr), size))
    )
}

/// Reallocate memory.
///
/// Reallocate the buffer starting at `ptr` with size `old_size`, to a buffer
/// starting at the returned pointer with size `size`.
///
/// # Important!
///
/// You should only reallocate buffers allocated through `ralloc`. Anything
/// else is considered invalid.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
///
/// # Safety
///
/// Due to being able to potentially memcpy an arbitrary buffer, as well as
/// shrinking a buffer, this is marked unsafe.
#[inline]
pub unsafe fn realloc(
    ptr: *mut u8,
    old_size: usize,
    size: usize,
    align: usize,
) -> *mut u8 {
    log!(
        CALL,
        "Reallocating buffer of size {} to new size {}.",
        old_size,
        size
    );

    get_allocator!(|alloc| Pointer::from(alloc.realloc(
        Block::from_raw_parts(Pointer::new(ptr), old_size),
        size,
        align
    )).get())
}

/// Try to reallocate the buffer _inplace_.
///
/// In case of success, return the new buffer's size. On failure, return the
/// old size.
///
/// This can be used to shrink (truncate) a buffer as well.
///
/// # Safety
///
/// Due to being able to shrink (and thus free) the buffer, this is marked
/// unsafe.
#[inline]
pub unsafe fn realloc_inplace(
    ptr: *mut u8,
    old_size: usize,
    size: usize,
) -> Result<(), ()> {
    log!(
        CALL,
        "Inplace reallocating buffer of size {} to new size {}.",
        old_size,
        size
    );

    get_allocator!(|alloc| {
        if alloc
            .realloc_inplace(
                Block::from_raw_parts(Pointer::new(ptr), old_size),
                size,
            )
            .is_ok()
        {
            Ok(())
        } else {
            Err(())
        }
    })
}
<commit_msg>Fixed occasional failing of 'partial_realloc'. This is because our bookkeeper's vec gets too cramped and should (theoretically) need resizing<commit_after>//! The global allocator.
//!
//! This contains primitives for the cross-thread allocator.

use prelude::*;

use core::{mem, ops};

use bookkeeper::{self, Allocator, Bookkeeper};
use {brk, sync};

use shim::config;

#[cfg(feature = "tls")]
use tls;

/// Alias for the wrapper type of the thread-local variable holding the local
/// allocator.
#[cfg(feature = "tls")]
type ThreadLocalAllocator =
    MoveCell<Option<LazyInit<fn() -> LocalAllocator, LocalAllocator>>>;

/// The global default allocator.
// TODO: Remove these filthy function pointers.
static GLOBAL_ALLOCATOR: sync::Mutex<
    LazyInit<fn() -> GlobalAllocator, GlobalAllocator>,
> = sync::Mutex::new(LazyInit::new(GlobalAllocator::init));
#[cfg(feature = "tls")]
tls! {
    /// The thread-local allocator.
    static THREAD_ALLOCATOR: ThreadLocalAllocator = MoveCell::new(Some(LazyInit::new(LocalAllocator::init)));
}

/// Temporarily get the allocator.
///
/// This is simply to avoid repeating ourself, so we let this take care of the
/// hairy stuff:
///
/// 1. Initialize the allocator if needed.
/// 2. If the allocator is not yet initialized, fallback to the global
/// allocator. 3. Unlock/move temporarily out of reference.
///
/// This is a macro due to the lack of generic closure, which makes it
/// impossible to have one closure for both cases (global and local).
// TODO: Instead of falling back to the global allocator, the thread dtor
// should be set such that it run after the TLS keys that might be declared.
macro_rules! get_allocator {
    (| $v:ident | $b:expr) => {{
        // Get the thread allocator, if TLS is enabled
        #[cfg(feature = "tls")]
        {
            THREAD_ALLOCATOR.with(|thread_alloc| {
                if let Some(mut thread_alloc_original) = thread_alloc.replace(None) {
                    let res = {
                        // Call the closure involved.
                        let $v = thread_alloc_original.get();
                        $b
                    };

                    // Put back the original allocator.
                    thread_alloc.replace(Some(thread_alloc_original));

                    res
                } else {
                    // The local allocator seems to have been deinitialized, for this reason we
                    // fallback to the global allocator.
                    log!(
                        WARNING,
                        "Accessing the allocator after deinitialization of the local allocator."
                    );

                    // Lock the global allocator.
                    let mut guard = GLOBAL_ALLOCATOR.lock();

                    // Call the block in question.
                    let $v = guard.get();
                    $b
                }
            })
        }

        // TLS is disabled, just use the global allocator.
        #[cfg(not(feature = "tls"))]
        {
            // Lock the global allocator.
            let mut guard = GLOBAL_ALLOCATOR.lock();

            // Call the block in question.
            let $v = guard.get();
            $b
        }
    }};
}

/// Derives `Deref` and `DerefMut` to the `inner` field.
///
/// This requires importing `core::ops`.
macro_rules! derive_deref {
    ($imp:ty, $target:ty) => {
        impl ops::Deref for $imp {
            type Target = $target;

            fn deref(&self) -> &$target {
                &self.inner
            }
        }

        impl ops::DerefMut for $imp {
            fn deref_mut(&mut self) -> &mut $target {
                &mut self.inner
            }
        }
    };
}

/// Global SBRK-based allocator.
///
/// This will extend the data segment whenever new memory is needed. Since this
/// includes leaving userspace, this shouldn't be used when other allocators
/// are available (i.e. the bookkeeper is local).
struct GlobalAllocator {
    // The inner bookkeeper.
    inner: Bookkeeper,
}

impl GlobalAllocator {
    /// Initialize the global allocator.
    fn init() -> GlobalAllocator {
        /// Logging...
        log!(NOTE, "Initializing the global allocator.");

        // The initial acquired segment.
        let (aligner, initial_segment, excessive) = brk::lock().canonical_brk(
            8 * bookkeeper::EXTRA_ELEMENTS * mem::size_of::<Block>(),
            mem::align_of::<Block>(),
        );

        // Initialize the new allocator.
        let mut res = GlobalAllocator {
            inner: Bookkeeper::new(unsafe {
                // LAST AUDIT: 2016-08-21 (Ticki).

                Vec::from_raw_parts(initial_segment, 0)
            }),
        };

        // Free the secondary space.
        res.push(aligner);
        res.push(excessive);

        res
    }
}

derive_deref!(GlobalAllocator, Bookkeeper);

impl Allocator for GlobalAllocator {
    #[inline]
    fn alloc_fresh(&mut self, size: usize, align: usize) -> Block {
        // Obtain what you need.
        let (alignment_block, res, excessive) =
            brk::lock().canonical_brk(size, align);

        // Add it to the list. This will not change the order, since the
        // pointer is higher than all the previous blocks (BRK extends
        // the data segment). Although, it is worth noting that the
        // stack is higher than the program break.
        self.push(alignment_block);
        self.push(excessive);

        res
    }

    fn on_new_memory(&mut self) {
        if self.total_bytes() > config::OS_MEMTRIM_LIMIT {
            // memtrim the fack outta 'em.

            // Pop the last block.
            let block = self
                .pop()
                .expect("The byte count on the global allocator is invalid.");

            // Check if the memtrim is worth it.
            if block.size() >= config::OS_MEMTRIM_WORTHY {
                /// Logging...
                log!(NOTE, "Memtrimming the global allocator.");

                // Release the block to the OS.
                if let Err(block) = brk::lock().release(block) {
                    // It failed, put the block back.
                    // TODO: This can be done faster.
                    self.push(block);
                }

            // Note that this block is the only block next to the program
            // break, due to the segments being as long as
            // possible. For that reason, repeating to push and
            // release would fail.
            } else {
                /// Logging...
                log!(WARNING, "Memtrimming for the global allocator failed.");

                // Push the block back.
                // TODO: This can be done faster.
                self.push(block);
            }
        }
    }
}

/// A local allocator.
///
/// This acquires memory from the upstream (global) allocator, which is
/// protected by a `Mutex`.
#[cfg(feature = "tls")]
pub struct LocalAllocator {
    // The inner bookkeeper.
    inner: Bookkeeper,
}

#[cfg(feature = "tls")]
impl LocalAllocator {
    /// Initialize the local allocator.
    #[cfg(feature = "tls")]
    fn init() -> LocalAllocator {
        /// The destructor of the local allocator.
        ///
        /// This will simply free everything to the global allocator.
        extern "C" fn dtor(alloc: &ThreadLocalAllocator) {
            /// Logging...
            log!(NOTE, "Deinitializing and freeing the local allocator.");

            // This is important! The thread destructors guarantee no other,
            // and thus one could use the allocator _after_ this
            // destructor have been finished. In fact, this is a
            // real problem, and happens when using `Arc` and
            // terminating the main thread, for this reason we place
            // `None` as a permanent marker indicating that the allocator is
            // deinitialized. After such a state is in place, all
            // allocation calls will be redirected to the global
            // allocator, which is of course still usable at this
            // moment.
            let alloc = alloc
                .replace(None)
                .expect("Thread-local allocator is already freed.");

            // Lock the global allocator.
            let mut global_alloc = GLOBAL_ALLOCATOR.lock();
            let global_alloc = global_alloc.get();

            // TODO: we know this is sorted, so we could abuse that fact to
            // faster insertion in the global allocator.

            alloc
                .into_inner()
                .inner
                .for_each(move |block| global_alloc.free(block));
        }

        /// Logging...
        log!(NOTE, "Initializing the local allocator.");

        // The initial acquired segment.
        let initial_segment = GLOBAL_ALLOCATOR.lock().get().alloc(
            8 * bookkeeper::EXTRA_ELEMENTS * mem::size_of::<Block>(),
            mem::align_of::<Block>(),
        );

        unsafe {
            // LAST AUDIT: 2016-08-21 (Ticki).

            // Register the thread destructor on the current thread.
            THREAD_ALLOCATOR.register_thread_destructor(dtor);

            LocalAllocator {
                inner: Bookkeeper::new(Vec::from_raw_parts(initial_segment, 0)),
            }
        }
    }
}

#[cfg(feature = "tls")]
derive_deref!(LocalAllocator, Bookkeeper);

#[cfg(feature = "tls")]
impl Allocator for LocalAllocator {
    #[inline]
    fn alloc_fresh(&mut self, size: usize, align: usize) -> Block {
        // Get the block from the global allocator. Please note that we cannot
        // canonicalize `size`, due to freeing excessive blocks would change
        // the order.
        GLOBAL_ALLOCATOR.lock().get().alloc(size, align)
    }

    #[inline]
    fn on_new_memory(&mut self) {
        // The idea is to free memory to the global allocator to unify small
        // stubs and avoid fragmentation and thread accumulation.
        if self.total_bytes() < config::FRAGMENTATION_SCALE * self.len()
            || self.total_bytes() > config::LOCAL_MEMTRIM_LIMIT
        {
            // Log stuff.
            log!(NOTE, "Memtrimming the local allocator.");

            // Lock the global allocator.
            let mut global_alloc = GLOBAL_ALLOCATOR.lock();
            let global_alloc = global_alloc.get();

            while let Some(block) = self.pop() {
                // Pop'n'free.
                global_alloc.free(block);

                // Memtrim 'till we won't memtrim anymore.
                if self.total_bytes() < config::LOCAL_MEMTRIM_STOP {
                    break;
                }
            }
        }
    }
}

/// Allocate a block of memory.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
#[inline]
pub fn alloc(size: usize, align: usize) -> *mut u8 {
    log!(
        CALL,
        "Allocating buffer of size {} (align {}).",
        size,
        align
    );

    get_allocator!(|alloc| Pointer::from(alloc.alloc(size, align)).get())
}

/// Free a buffer.
///
/// Note that this do not have to be a buffer allocated through ralloc. The
/// only requirement is that it is not used after the free.
///
/// # Important!
///
/// You should only allocate buffers allocated through `ralloc`. Anything else
/// is considered invalid.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
///
/// # Safety
///
/// Rust assume that the allocation symbols returns correct values. For this
/// reason, freeing invalid pointers might introduce memory unsafety.
///
/// Secondly, freeing an used buffer can introduce use-after-free.
#[inline]
pub unsafe fn free(ptr: *mut u8, size: usize) {
    log!(CALL, "Freeing buffer of size {}.", size);

    get_allocator!(
        |alloc| alloc.free(Block::from_raw_parts(Pointer::new(ptr), size))
    )
}

/// Reallocate memory.
///
/// Reallocate the buffer starting at `ptr` with size `old_size`, to a buffer
/// starting at the returned pointer with size `size`.
///
/// # Important!
///
/// You should only reallocate buffers allocated through `ralloc`. Anything
/// else is considered invalid.
///
/// # Errors
///
/// The OOM handler handles out-of-memory conditions.
///
/// # Safety
///
/// Due to being able to potentially memcpy an arbitrary buffer, as well as
/// shrinking a buffer, this is marked unsafe.
#[inline]
pub unsafe fn realloc(
    ptr: *mut u8,
    old_size: usize,
    size: usize,
    align: usize,
) -> *mut u8 {
    log!(
        CALL,
        "Reallocating buffer of size {} to new size {}.",
        old_size,
        size
    );

    get_allocator!(|alloc| Pointer::from(alloc.realloc(
        Block::from_raw_parts(Pointer::new(ptr), old_size),
        size,
        align
    )).get())
}

/// Try to reallocate the buffer _inplace_.
///
/// In case of success, return the new buffer's size. On failure, return the
/// old size.
///
/// This can be used to shrink (truncate) a buffer as well.
///
/// # Safety
///
/// Due to being able to shrink (and thus free) the buffer, this is marked
/// unsafe.
#[inline]
pub unsafe fn realloc_inplace(
    ptr: *mut u8,
    old_size: usize,
    size: usize,
) -> Result<(), ()> {
    log!(
        CALL,
        "Inplace reallocating buffer of size {} to new size {}.",
        old_size,
        size
    );

    get_allocator!(|alloc| {
        if alloc
            .realloc_inplace(
                Block::from_raw_parts(Pointer::new(ptr), old_size),
                size,
            )
            .is_ok()
        {
            Ok(())
        } else {
            Err(())
        }
    })
}
<|endoftext|>


<commit_before>//! Storage Engine trait and several implementations
//!
//!

pub mod flatfile;

use self::flatfile::FlatFile;

use super::parse::ast::SqlType;

use std::mem;

use std::io;
use std::io::prelude::*;

use std::fs;
use std::fs::{OpenOptions, create_dir};

use std::convert::From;

use byteorder::Error;
use byteorder::{WriteBytesExt, ReadBytesExt, BigEndian};

use bincode::SizeLimit;
use bincode::rustc_serialize::{EncodingError,
    DecodingError, encode_into, decode_from};

/// constants
 const MAGIC_NUMBER: u64 = 0x6561742073686974; // secret
 const VERSION_NO: u8 = 1;

/// A database table
///
/// Through this type, you can retreive certain meta information about the
/// table (like column names, column types, storage engine, ...). It's `access`
/// method locks the table globally and returns a storage engine to access
/// the table data.
#[derive(Debug)]
pub enum DatabaseError {
    Io(io::Error),
    BinEn(EncodingError),
    BinDe(DecodingError),
    Byteorder(::byteorder::Error),
    WrongMagicNmbr,
    Engine, //cur not used
    LoadDataBase,
    RemoveColumn,
    AddColumn,
}

impl From<io::Error> for DatabaseError {
    fn from(err: io::Error) -> DatabaseError {
        DatabaseError::Io(err)
    }
}

impl  From<EncodingError> for DatabaseError {
    fn from(err: EncodingError) -> DatabaseError {
        DatabaseError::BinEn(err)
    }
}

impl From<DecodingError> for DatabaseError {
    fn from(err: DecodingError) -> DatabaseError {
        DatabaseError::BinDe(err)
    }
}

impl From< ::byteorder::Error> for DatabaseError {
    fn from(err: ::byteorder::Error) -> DatabaseError {
        DatabaseError::Byteorder(err)
    }
}

//---------------------------------------------------------------
// DataType
//---------------------------------------------------------------
/// A Enum for Datatypes (will be removed later)
#[repr(u8)]
#[derive(Clone,Copy,Debug,RustcDecodable, RustcEncodable)]
pub enum DataType { Integer = 1, Float = 2, }

impl DataType {
    pub fn value(&self) -> u8 {
       *self as u8
    }
}

//---------------------------------------------------------------
// Database
//---------------------------------------------------------------
#[derive(Debug)]
pub struct Database {
    name: String,
}

impl Database {
    /// Starts the process of creating a new Database
    /// Returns database or on fail DatabaseError
    pub fn create(name: &str) -> Result<Database, DatabaseError> {
        let d = Database{ name: name.to_string() };
        try!(d.save());
        info!("created new database {:?}",d);
        Ok(d)
    }

    /// Loads already existing Database
    /// returns DataBase Error when database does not exist else the loaded DB
    pub fn load(name: &str) -> Result<Database, DatabaseError> {
        if try!(fs::metadata(name)).is_dir() {
            info!("loaded Database {:?}", name.to_string());
            Ok(Database{ name: name.to_string() })
        } else {
            warn!("could not load database {:?}", name.to_string());
            return Err(DatabaseError::LoadDataBase)
        }
    }

    /// Creates a folder for the database
    fn save(&self) -> Result<(), DatabaseError> {
        info!("trying to create dir!");
        try!(create_dir(&self.name));
        info!("created dir");
        Ok(())
    }

    /// Creates a new table in the DB folder
    /// Returns with DatabaseError on fail else Table
    pub fn create_table(&self, name: &str, columns: Vec<Column>, engine_id: u8)
        -> Result<Table, DatabaseError>
    {

        let t = Table::new(&self, name, columns, engine_id);
        try!(t.save());
        info!("created new table {:?}", t);
        Ok(t)
    }

    /// calls load for table with the database path
    /// Returns with DatabaseError on fail else Table
    pub fn load_table(&self, name: &str) -> Result<Table, DatabaseError> {
        Table::load(&self, name)
    }
}

//---------------------------------------------------------------
// TableMetaData
//---------------------------------------------------------------

#[derive(Debug, RustcDecodable, RustcEncodable)]
pub struct  TableMetaData {
    version_nmbr: u8,
    engine_id: u8,
    columns: Vec<Column>,
}

//---------------------------------------------------------------
// Table
//---------------------------------------------------------------

/// Table struct that contains the table information
#[derive(Debug)]
pub struct Table<'a> {
    database: &'a Database,
    name: String,
    meta_data: TableMetaData,
}

impl<'a> Table<'a> {
    /// Creates new table object
    /// Returns Table
    pub fn new<'b>(database: &'b Database, name: &str,
                   columns: Vec<Column>, engine_id: u8)
        -> Table<'b>
    {

        let meta_data = TableMetaData {
            version_nmbr: VERSION_NO,
            engine_id: engine_id,
            columns: columns,
        };
        info!("created meta data: {:?}", meta_data);

        Table {
            name: name.to_string(),
            database: database,
            meta_data: meta_data,
        }
    }

    /// Loads the table from the DB
    /// Returns with DatabaseError on fail else Table
    fn load<'b>(database: &'b Database, name: &str)
        -> Result<Table<'b>, DatabaseError>
    {
        // TODO: Read the .tbl file from disk and parse it

        let path_to_table = Table::get_path(&database.name, name, "tbl");
        info!("getting path and opening file: {:?}", path_to_table);
        let mut file = try!(OpenOptions::new()
            .read(true)
            .open(path_to_table));
        info!("reading file: {:?}", file);
        let ma_nmbr = try!(file.read_uint::<BigEndian>(mem::size_of_val(&MAGIC_NUMBER)));

        info!("checking magic number: {:?}",ma_nmbr);
        if ma_nmbr != MAGIC_NUMBER {
            println!("Magic Number not correct");
            return Err(DatabaseError::WrongMagicNmbr)
        }
        let meta_data: TableMetaData = try!(decode_from(&mut file, SizeLimit::Infinite));
        info!("getting meta data{:?}", meta_data);

        let table = Table::new(database, name, meta_data.columns, meta_data.engine_id);
        info!("returning table: {:?}", table);
        Ok(table)
    }

    /// Saves the table with a identification number in table file
    /// Returns DatabaseError on fail else Nothing
    pub fn save(&self) -> Result<(), DatabaseError> {
        // call for open file
        info!("opening file to write",);
        let mut file = try!(OpenOptions::new()
            .write(true)
            .create(true)
            .open(self.get_table_metadata_path()));
        info!("writing magic number in file: {:?}", file);
        try!(file.write_u64::<BigEndian>(MAGIC_NUMBER));//MAGIC_NUMBER
        info!("writing meta data in file: {:?}",file);
        try!(encode_into(&self.meta_data, &mut file,SizeLimit::Infinite));

        // debug message all okay
        info!("I Wrote my File");
        Ok(())
    }

    /// Returns columns of table as array
    pub fn columns(&self) -> &[Column] {
        &self.meta_data.columns
    }

    /// Adds a column to the tabel
    /// Returns name of Column or on fail DatabaseError
    pub fn add_column(&mut self, name: &str, sql_type: SqlType) -> Result<(), DatabaseError> {
        match self.meta_data.columns.iter().find(|x| x.name == name) {
            Some(_) => {
                warn!("Column {:?} already exists", name);
                return Err(DatabaseError::AddColumn)
            },
            None => {
                info!("Column {:?} was added", name);
            },
        }
        self.meta_data.columns.push(Column::create_new(name, sql_type));
        Ok(())
    }

    /// Removes a column from the table
    /// Returns name of Column or on fail DatabaseError
    pub fn remove_column(&mut self, name: &str) -> Result<(), DatabaseError> {
        let index = match self.meta_data.columns.iter().position(|x| x.name == name) {
            Some(x) => {
                info!("Column {:?} was removed" , self.name);
                x
            },
            None => {
                warn!("Column {:?} could not be found", self.name);
                return Err(DatabaseError::RemoveColumn)
            },
        };
        self.meta_data.columns.swap_remove(index);
        Ok(())
    }

    /// Creates an engine for Table
    /// Returns Box<Engine>
    pub fn create_engine(self) -> Box<Engine + 'a> {
        Box::new(FlatFile::new(self))
    }

    /// Returns the path for the metadata files
    fn get_table_metadata_path(&self) -> String {
        Self::get_path(&self.database.name, &self.name, "tbl")
    }

    /// Returns the path for the data files
    fn get_table_data_path(&self) -> String {
        Self::get_path(&self.database.name, &self.name, "dat")
    }

    /// Returns the path of the table
    fn get_path(database: &str, name: &str, ext: &str) -> String {
         format!("{}/{}.{}", database, name, ext)
    }
}

//---------------------------------------------------------------
// Column
//---------------------------------------------------------------

/// A table column. Has a name, a type, ...
#[derive(Debug,RustcDecodable, RustcEncodable,Clone)]
pub struct Column {
    pub name: String, //name of column
    pub sql_type: SqlType, //name of the data type that is contained in this column
}

impl Column {
    /// Creates a new column object
    /// Returns with Column
    pub fn create_new(name: &str, sql_type: SqlType) -> Column {
        Column {
            name: name.to_string(),
            sql_type: sql_type.clone()
        }
    }
}

//---------------------------------------------------------------
// Engine
//---------------------------------------------------------------

/// Storage Engine Interface.
///
/// A storage engine, like MyISAM and InnoDB, is responsible for reading and
/// writing data to disk, maintain and use indices, check for data corruption
/// and repair corrupt files.
///
/// Each table in a database may use a different storage engine.
pub trait Engine {
    fn create_table(&mut self) -> Result<(), DatabaseError>;
}

// # Some information for the `storage` working group:
//
// You work at the very bottom of the database: The thing that actually
// writes the data disk. Everything in this module is used only by the
// query execution module.
//
// The file layout may look like this:
// We have some `db_dir` where everything lives. In that directory, there are
// subdirectories for every database. In each of those subdirs is optionally
// a file `db.meta` that contains information about the database (such as
// permissions). The tables of each database are saved in *.tbl files that
// live inside the database directory.
//
// Your task is to provide types and method to:
// - read a specific table from a specific
//   database from file
// - create a new table in a database
// - query meta information about a table (columns for example)
// - lock a table for reading/writing it's data through a storage engine
//
// The other main task is to:
// - specify the storage engine interface
// - implement a simple storage engine (heap/flatfiles)
//
<commit_msg>Add delete function for Table and Enum containing Engines<commit_after>//! Storage Engine trait and several implementations
//!
//!

pub mod flatfile;

use self::flatfile::FlatFile;

use super::parse::ast::SqlType;

use std::mem;

use std::io;
use std::io::prelude::*;

use std::fs;
use std::fs::{OpenOptions, create_dir};

use std::convert::From;

use byteorder::Error;
use byteorder::{WriteBytesExt, ReadBytesExt, BigEndian};

use bincode::SizeLimit;
use bincode::rustc_serialize::{EncodingError,
    DecodingError, encode_into, decode_from};

/// constants
 const MAGIC_NUMBER: u64 = 0x6561742073686974; // secret
 const VERSION_NO: u8 = 1;

/// A database table
///
/// Through this type, you can retreive certain meta information about the
/// table (like column names, column types, storage engine, ...). It's `access`
/// method locks the table globally and returns a storage engine to access
/// the table data.
#[derive(Debug)]
pub enum DatabaseError {
    Io(io::Error),
    BinEn(EncodingError),
    BinDe(DecodingError),
    Byteorder(::byteorder::Error),
    WrongMagicNmbr,
    Engine, //cur not used
    LoadDataBase,
    RemoveColumn,
    AddColumn,
}

impl From<io::Error> for DatabaseError {
    fn from(err: io::Error) -> DatabaseError {
        DatabaseError::Io(err)
    }
}

impl  From<EncodingError> for DatabaseError {
    fn from(err: EncodingError) -> DatabaseError {
        DatabaseError::BinEn(err)
    }
}

impl From<DecodingError> for DatabaseError {
    fn from(err: DecodingError) -> DatabaseError {
        DatabaseError::BinDe(err)
    }
}

impl From< ::byteorder::Error> for DatabaseError {
    fn from(err: ::byteorder::Error) -> DatabaseError {
        DatabaseError::Byteorder(err)
    }
}

//---------------------------------------------------------------
// DataType
//---------------------------------------------------------------
/// A Enum for Datatypes (will be removed later)
#[repr(u8)]
#[derive(Clone,Copy,Debug,RustcDecodable, RustcEncodable)]
pub enum DataType { Integer = 1, Float = 2, }

impl DataType {
    pub fn value(&self) -> u8 {
       *self as u8
    }
}

//---------------------------------------------------------------
// Database
//---------------------------------------------------------------
#[derive(Debug)]
pub struct Database {
    name: String,
}

impl Database {
    /// Starts the process of creating a new Database
    /// Returns database or on fail DatabaseError
    pub fn create(name: &str) -> Result<Database, DatabaseError> {
        let d = Database{ name: name.to_string() };
        try!(d.save());
        info!("created new database {:?}",d);
        Ok(d)
    }

    /// Loads already existing Database
    /// returns DataBase Error when database does not exist else the loaded DB
    pub fn load(name: &str) -> Result<Database, DatabaseError> {
        if try!(fs::metadata(name)).is_dir() {
            info!("loaded Database {:?}", name.to_string());
            Ok(Database{ name: name.to_string() })
        } else {
            warn!("could not load database {:?}", name.to_string());
            return Err(DatabaseError::LoadDataBase)
        }
    }

    /// Creates a folder for the database
    fn save(&self) -> Result<(), DatabaseError> {
        info!("trying to create dir!");
        try!(create_dir(&self.name));
        info!("created dir");
        Ok(())
    }

    /// Creates a new table in the DB folder
    /// Returns with DatabaseError on fail else Table
    pub fn create_table(&self, name: &str, columns: Vec<Column>, engine_id: u8)
        -> Result<Table, DatabaseError>
    {

        let t = Table::new(&self, name, columns, engine_id);
        try!(t.save());
        info!("created new table {:?}", t);
        Ok(t)
    }

    /// calls load for table with the database path
    /// Returns with DatabaseError on fail else Table
    pub fn load_table(&self, name: &str) -> Result<Table, DatabaseError> {
        Table::load(&self, name)
    }
}

//---------------------------------------------------------------
// TableMetaData
//---------------------------------------------------------------

#[derive(Debug, RustcDecodable, RustcEncodable)]
pub struct  TableMetaData {
    version_nmbr: u8,
    engine_id: u8,
    columns: Vec<Column>,
}

//---------------------------------------------------------------
// Table
//---------------------------------------------------------------

/// Table struct that contains the table information
#[derive(Debug)]
pub struct Table<'a> {
    database: &'a Database,
    name: String,
    meta_data: TableMetaData,
}

impl<'a> Table<'a> {
    /// Creates new table object
    /// Returns Table
    pub fn new<'b>(database: &'b Database, name: &str,
                   columns: Vec<Column>, engine_id: u8)
        -> Table<'b>
    {

        let meta_data = TableMetaData {
            version_nmbr: VERSION_NO,
            engine_id: engine_id,
            columns: columns,
        };
        info!("created meta data: {:?}", meta_data);

        Table {
            name: name.to_string(),
            database: database,
            meta_data: meta_data,
        }
    }

    /// Loads the table from the DB
    /// Returns with DatabaseError on fail else Table
    fn load<'b>(database: &'b Database, name: &str)
        -> Result<Table<'b>, DatabaseError>
    {
        // TODO: Read the .tbl file from disk and parse it

        let path_to_table = Table::get_path(&database.name, name, "tbl");
        info!("getting path and opening file: {:?}", path_to_table);
        let mut file = try!(OpenOptions::new()
            .read(true)
            .open(path_to_table));
        info!("reading file: {:?}", file);
        let ma_nmbr = try!(file.read_uint::<BigEndian>(mem::size_of_val(&MAGIC_NUMBER)));

        info!("checking magic number: {:?}",ma_nmbr);
        if ma_nmbr != MAGIC_NUMBER {
            println!("Magic Number not correct");
            return Err(DatabaseError::WrongMagicNmbr)
        }
        let meta_data: TableMetaData = try!(decode_from(&mut file, SizeLimit::Infinite));
        info!("getting meta data{:?}", meta_data);

        let table = Table::new(database, name, meta_data.columns, meta_data.engine_id);
        info!("returning table: {:?}", table);
        Ok(table)
    }

    /// Saves the table with a identification number in table file
    /// Returns DatabaseError on fail else Nothing
    pub fn save(&self) -> Result<(), DatabaseError> {
        // call for open file
        info!("opening file to write",);
        let mut file = try!(OpenOptions::new()
            .write(true)
            .create(true)
            .open(self.get_table_metadata_path()));
        info!("writing magic number in file: {:?}", file);
        try!(file.write_u64::<BigEndian>(MAGIC_NUMBER));//MAGIC_NUMBER
        info!("writing meta data in file: {:?}",file);
        try!(encode_into(&self.meta_data, &mut file,SizeLimit::Infinite));

        // debug message all okay
        info!("I Wrote my File");
        Ok(())
    }

    /// Deletes the .tbl files
    /// Returns DatabaseError on fail if path points to a directory,
    /// if the user lacks permissions to remove the file,
    /// or if some other filesystem-level error occurs.
    pub fn delete(&self) -> Result<(), DatabaseError>{

        info!("remove meta file: {:?}", self.get_table_metadata_path());
        try!(fs::remove_file(self.get_table_metadata_path()));

        info!("remove data file: {:?}",self.get_table_data_path());
        try!(fs::remove_file(self.get_table_data_path()));

        Ok(())
    }

    /// Returns columns of table as array
    pub fn columns(&self) -> &[Column] {
        &self.meta_data.columns
    }

    /// Adds a column to the tabel
    /// Returns name of Column or on fail DatabaseError
    pub fn add_column(&mut self, name: &str, sql_type: SqlType) -> Result<(), DatabaseError> {
        match self.meta_data.columns.iter().find(|x| x.name == name) {
            Some(_) => {
                warn!("Column {:?} already exists", name);
                return Err(DatabaseError::AddColumn)
            },
            None => {
                info!("Column {:?} was added", name);
            },
        }
        self.meta_data.columns.push(Column::create_new(name, sql_type));
        Ok(())
    }

    /// Removes a column from the table
    /// Returns name of Column or on fail DatabaseError
    pub fn remove_column(&mut self, name: &str) -> Result<(), DatabaseError> {
        let index = match self.meta_data.columns.iter().position(|x| x.name == name) {
            Some(x) => {
                info!("Column {:?} was removed" , self.name);
                x
            },
            None => {
                warn!("Column {:?} could not be found", self.name);
                return Err(DatabaseError::RemoveColumn)
            },
        };
        self.meta_data.columns.swap_remove(index);
        Ok(())
    }

    /// Creates an engine for Table
    /// Returns Box<Engine>
    pub fn create_engine(self) -> Box<Engine + 'a> {
        Box::new(FlatFile::new(self))
    }

    /// Returns the path for the metadata files
    fn get_table_metadata_path(&self) -> String {
        Self::get_path(&self.database.name, &self.name, "tbl")
    }

    /// Returns the path for the data files
    fn get_table_data_path(&self) -> String {
        Self::get_path(&self.database.name, &self.name, "dat")
    }

    /// Returns the path of the table
    fn get_path(database: &str, name: &str, ext: &str) -> String {
         format!("{}/{}.{}", database, name, ext)
    }
}

//---------------------------------------------------------------
// Column
//---------------------------------------------------------------

/// A table column. Has a name, a type, ...
#[derive(Debug,RustcDecodable, RustcEncodable,Clone)]
pub struct Column {
    pub name: String, //name of column
    pub sql_type: SqlType, //name of the data type that is contained in this column
}

impl Column {
    /// Creates a new column object
    /// Returns with Column
    pub fn create_new(name: &str, sql_type: SqlType) -> Column {
        Column {
            name: name.to_string(),
            sql_type: sql_type.clone()
        }
    }
}

//---------------------------------------------------------------
// Engine
//---------------------------------------------------------------

/// Storage Engine Interface.
///
/// A storage engine, like MyISAM and InnoDB, is responsible for reading and
/// writing data to disk, maintain and use indices, check for data corruption
/// and repair corrupt files.
///
/// Each table in a database may use a different storage engine.
pub trait Engine {
    fn create_table(&mut self) -> Result<(), DatabaseError>;
}

#[repr(u8)]
#[derive(Clone,Copy,Debug,RustcDecodable, RustcEncodable)]
enum EngineID {
    FlatFile = 1,
    InvertedIndex = 2,
}

// # Some information for the `storage` working group:
//
// You work at the very bottom of the database: The thing that actually
// writes the data disk. Everything in this module is used only by the
// query execution module.
//
// The file layout may look like this:
// We have some `db_dir` where everything lives. In that directory, there are
// subdirectories for every database. In each of those subdirs is optionally
// a file `db.meta` that contains information about the database (such as
// permissions). The tables of each database are saved in *.tbl files that
// live inside the database directory.
//
// Your task is to provide types and method to:
// - read a specific table from a specific
//   database from file
// - create a new table in a database
// - query meta information about a table (columns for example)
// - lock a table for reading/writing it's data through a storage engine
//
// The other main task is to:
// - specify the storage engine interface
// - implement a simple storage engine (heap/flatfiles)
//
<|endoftext|>


<commit_before>extern crate libc;
extern crate rustc_serialize;

use rosxmlrpc;
use self::rustc_serialize::Encodable;
use std;
use std::sync::mpsc::{Sender, Receiver};
use std::sync::mpsc;
use std::sync::Mutex;
use std::error::Error as ErrorTrait;
use self::libc::getpid;

pub struct Slave {
    server: rosxmlrpc::Server,
    req: Mutex<Receiver<(String, rosxmlrpc::serde::Decoder)>>,
    res: Mutex<Sender<Vec<u8>>>,
    subscriptions: Vec<(String, String)>,
    publications: Vec<(String, String)>,
}

struct SlaveHandler {
    req: Mutex<Sender<(String, rosxmlrpc::serde::Decoder)>>,
    res: Mutex<Receiver<Vec<u8>>>,
}

type SerdeResult<T> = Result<T, Error>;

impl Slave {
    pub fn new(server_uri: &str) -> Result<Slave, Error> {
        let (tx_req, rx_req) = mpsc::channel();
        let (tx_res, rx_res) = mpsc::channel();
        let server = try!(rosxmlrpc::Server::new(server_uri,
                                                 SlaveHandler {
                                                     req: Mutex::new(tx_req),
                                                     res: Mutex::new(rx_res),
                                                 }));
        Ok(Slave {
            server: server,
            subscriptions: vec![],
            publications: vec![],
            req: Mutex::new(rx_req),
            res: Mutex::new(tx_res),
        })
    }

    pub fn uri(&self) -> &str {
        return &self.server.uri;
    }

    fn encode_response<T: Encodable>(&self, response: SerdeResult<T>, message: &str) {
        let mut body = Vec::<u8>::new();
        {
            let mut res = rosxmlrpc::serde::Encoder::new(&mut body);
            res.start_response().unwrap();
            match response {
                    Ok(value) => (1i32, message, value).encode(&mut res),
                    Err(err) => (-1i32, err.description(), 0).encode(&mut res),
                }
                .unwrap_or_else(|err| {
                    println!("Encoding error: {}", err);
                });
            res.end_response().unwrap();
        }
        self.res.lock().unwrap().send(body).unwrap();
    }

    fn param_update(&self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        let key = try!(req.decode_request_parameter::<String>());
        let value = try!(req.decode_request_parameter::<String>());
        if caller_id != "" && key != "" && value != "" {
            println!("{} {} {}", caller_id, key, value);
            Ok(0)
        } else {
            Err(Error::Protocol("Emtpy strings given".to_owned()))
        }
    }

    fn get_pid(&self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(unsafe { getpid() })
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn shutdown(&mut self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            self.server.shutdown().unwrap();
            Ok(0)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn get_publications(&self,
                        req: &mut rosxmlrpc::serde::Decoder)
                        -> SerdeResult<&[(String, String)]> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(&self.publications)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn get_subscriptions(&self,
                         req: &mut rosxmlrpc::serde::Decoder)
                         -> SerdeResult<&[(String, String)]> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(&self.subscriptions)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    pub fn handle_calls(&mut self) {
        loop {
            let recv = self.req.lock().unwrap().try_recv();
            if let Ok((method_name, mut req)) = recv {
                println!("HANDLING METHOD: {}", method_name);
                match method_name.as_str() {
                    "getBusStats" => unimplemented!(),
                    "getBusInfo" => unimplemented!(),
                    "getMasterUri" => unimplemented!(),
                    "shutdown" => {
                        let data = self.shutdown(&mut req);
                        self.encode_response(data, "Shutdown");
                    }
                    "getPid" => {
                        self.encode_response(self.get_pid(&mut req), "PID");
                    }
                    "getSubscriptions" => {
                        self.encode_response(self.get_subscriptions(&mut req),
                                             "List of subscriptions");
                    }
                    "getPublications" => {
                        self.encode_response(self.get_publications(&mut req),
                                             "List of publications");
                    }
                    "paramUpdate" => {
                        self.encode_response(self.param_update(&mut req), "Parameter updated");
                    }
                    "publisherUpdate" => unimplemented!(),
                    "requestTopic" => unimplemented!(),
                    _ => {}
                }
            }
        }
    }
}

impl rosxmlrpc::server::XmlRpcServer for SlaveHandler {
    fn handle(&self, method_name: &str, _: usize, req: rosxmlrpc::serde::Decoder) -> Vec<u8> {
        println!("CALLED METHOD: {}", method_name);
        self.req.lock().unwrap().send((method_name.to_owned(), req)).unwrap();
        self.res.lock().unwrap().recv().unwrap()
    }
}

#[derive(Debug)]
pub enum Error {
    Deserialization(rosxmlrpc::serde::decoder::Error),
    Protocol(String),
    Serialization(rosxmlrpc::serde::encoder::Error),
    XmlRpc(rosxmlrpc::error::Error),
}

impl From<rosxmlrpc::serde::decoder::Error> for Error {
    fn from(err: rosxmlrpc::serde::decoder::Error) -> Error {
        Error::Deserialization(err)
    }
}

impl From<rosxmlrpc::serde::encoder::Error> for Error {
    fn from(err: rosxmlrpc::serde::encoder::Error) -> Error {
        Error::Serialization(err)
    }
}

impl From<rosxmlrpc::error::Error> for Error {
    fn from(err: rosxmlrpc::error::Error) -> Error {
        Error::XmlRpc(err)
    }
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        match *self {
            Error::Deserialization(ref err) => write!(f, "Deserialization error: {}", err),
            Error::Protocol(ref err) => write!(f, "Protocol error: {}", err),
            Error::Serialization(ref err) => write!(f, "Serialization error: {}", err),
            Error::XmlRpc(ref err) => write!(f, "XML RPC error: {}", err),
        }
    }
}

impl std::error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            Error::Deserialization(ref err) => err.description(),
            Error::Protocol(ref err) => &err,
            Error::Serialization(ref err) => err.description(),
            Error::XmlRpc(ref err) => err.description(),
        }
    }

    fn cause(&self) -> Option<&std::error::Error> {
        match *self {
            Error::Deserialization(ref err) => Some(err),
            Error::Protocol(..) => None,
            Error::Serialization(ref err) => Some(err),
            Error::XmlRpc(ref err) => Some(err),
        }
    }
}
<commit_msg>Stop consuming encoding errors<commit_after>extern crate libc;
extern crate rustc_serialize;

use rosxmlrpc;
use self::rustc_serialize::Encodable;
use std;
use std::sync::mpsc::{Sender, Receiver};
use std::sync::mpsc;
use std::sync::Mutex;
use std::error::Error as ErrorTrait;
use self::libc::getpid;

pub struct Slave {
    server: rosxmlrpc::Server,
    req: Mutex<Receiver<(String, rosxmlrpc::serde::Decoder)>>,
    res: Mutex<Sender<Vec<u8>>>,
    subscriptions: Vec<(String, String)>,
    publications: Vec<(String, String)>,
}

struct SlaveHandler {
    req: Mutex<Sender<(String, rosxmlrpc::serde::Decoder)>>,
    res: Mutex<Receiver<Vec<u8>>>,
}

type SerdeResult<T> = Result<T, Error>;

impl Slave {
    pub fn new(server_uri: &str) -> Result<Slave, Error> {
        let (tx_req, rx_req) = mpsc::channel();
        let (tx_res, rx_res) = mpsc::channel();
        let server = try!(rosxmlrpc::Server::new(server_uri,
                                                 SlaveHandler {
                                                     req: Mutex::new(tx_req),
                                                     res: Mutex::new(rx_res),
                                                 }));
        Ok(Slave {
            server: server,
            subscriptions: vec![],
            publications: vec![],
            req: Mutex::new(rx_req),
            res: Mutex::new(tx_res),
        })
    }

    pub fn uri(&self) -> &str {
        return &self.server.uri;
    }

    fn encode_response<T: Encodable>(&self, response: SerdeResult<T>, message: &str) {
        let mut body = Vec::<u8>::new();
        {
            let mut res = rosxmlrpc::serde::Encoder::new(&mut body);
            res.start_response().unwrap();
            match response {
                    Ok(value) => (1i32, message, value).encode(&mut res),
                    Err(err) => (-1i32, err.description(), 0).encode(&mut res),
                }
                .unwrap();
            res.end_response().unwrap();
        }
        self.res.lock().unwrap().send(body).unwrap();
    }

    fn param_update(&self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        let key = try!(req.decode_request_parameter::<String>());
        let value = try!(req.decode_request_parameter::<String>());
        if caller_id != "" && key != "" && value != "" {
            println!("{} {} {}", caller_id, key, value);
            Ok(0)
        } else {
            Err(Error::Protocol("Emtpy strings given".to_owned()))
        }
    }

    fn get_pid(&self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(unsafe { getpid() })
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn shutdown(&mut self, req: &mut rosxmlrpc::serde::Decoder) -> SerdeResult<i32> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            self.server.shutdown().unwrap();
            Ok(0)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn get_publications(&self,
                        req: &mut rosxmlrpc::serde::Decoder)
                        -> SerdeResult<&[(String, String)]> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(&self.publications)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    fn get_subscriptions(&self,
                         req: &mut rosxmlrpc::serde::Decoder)
                         -> SerdeResult<&[(String, String)]> {
        let caller_id = try!(req.decode_request_parameter::<String>());
        if caller_id != "" {
            Ok(&self.subscriptions)
        } else {
            Err(Error::Protocol("Empty strings given".to_owned()))
        }
    }

    pub fn handle_calls(&mut self) {
        loop {
            let recv = self.req.lock().unwrap().try_recv();
            if let Ok((method_name, mut req)) = recv {
                println!("HANDLING METHOD: {}", method_name);
                match method_name.as_str() {
                    "getBusStats" => unimplemented!(),
                    "getBusInfo" => unimplemented!(),
                    "getMasterUri" => unimplemented!(),
                    "shutdown" => {
                        let data = self.shutdown(&mut req);
                        self.encode_response(data, "Shutdown");
                    }
                    "getPid" => {
                        self.encode_response(self.get_pid(&mut req), "PID");
                    }
                    "getSubscriptions" => {
                        self.encode_response(self.get_subscriptions(&mut req),
                                             "List of subscriptions");
                    }
                    "getPublications" => {
                        self.encode_response(self.get_publications(&mut req),
                                             "List of publications");
                    }
                    "paramUpdate" => {
                        self.encode_response(self.param_update(&mut req), "Parameter updated");
                    }
                    "publisherUpdate" => unimplemented!(),
                    "requestTopic" => unimplemented!(),
                    _ => {}
                }
            }
        }
    }
}

impl rosxmlrpc::server::XmlRpcServer for SlaveHandler {
    fn handle(&self, method_name: &str, _: usize, req: rosxmlrpc::serde::Decoder) -> Vec<u8> {
        println!("CALLED METHOD: {}", method_name);
        self.req.lock().unwrap().send((method_name.to_owned(), req)).unwrap();
        self.res.lock().unwrap().recv().unwrap()
    }
}

#[derive(Debug)]
pub enum Error {
    Deserialization(rosxmlrpc::serde::decoder::Error),
    Protocol(String),
    Serialization(rosxmlrpc::serde::encoder::Error),
    XmlRpc(rosxmlrpc::error::Error),
}

impl From<rosxmlrpc::serde::decoder::Error> for Error {
    fn from(err: rosxmlrpc::serde::decoder::Error) -> Error {
        Error::Deserialization(err)
    }
}

impl From<rosxmlrpc::serde::encoder::Error> for Error {
    fn from(err: rosxmlrpc::serde::encoder::Error) -> Error {
        Error::Serialization(err)
    }
}

impl From<rosxmlrpc::error::Error> for Error {
    fn from(err: rosxmlrpc::error::Error) -> Error {
        Error::XmlRpc(err)
    }
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        match *self {
            Error::Deserialization(ref err) => write!(f, "Deserialization error: {}", err),
            Error::Protocol(ref err) => write!(f, "Protocol error: {}", err),
            Error::Serialization(ref err) => write!(f, "Serialization error: {}", err),
            Error::XmlRpc(ref err) => write!(f, "XML RPC error: {}", err),
        }
    }
}

impl std::error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            Error::Deserialization(ref err) => err.description(),
            Error::Protocol(ref err) => &err,
            Error::Serialization(ref err) => err.description(),
            Error::XmlRpc(ref err) => err.description(),
        }
    }

    fn cause(&self) -> Option<&std::error::Error> {
        match *self {
            Error::Deserialization(ref err) => Some(err),
            Error::Protocol(..) => None,
            Error::Serialization(ref err) => Some(err),
            Error::XmlRpc(ref err) => Some(err),
        }
    }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::mem::transmute;
use std::ptr::copy_nonoverlapping;

/// Encode `value` in little-endian and put in the first 4-bytes of `dst`.
pub fn encode_fixed_32(dst: &mut [u8], value: u32) {
  assert!(dst.len() >= 4);
  unsafe {
    let bytes = transmute::<_, [u8; 4]>(value.to_le());
    copy_nonoverlapping(bytes.as_ptr(), dst.as_mut_ptr(), 4);
  }
}

/// Encode `value` in little-endian and put in the first 8-bytes of `dst`.
pub fn encode_fixed_64(dst: &mut [u8], value: u64) {
  assert!(dst.len() >= 8);
  unsafe {
    let bytes = transmute::<_, [u8; 8]>(value.to_le());
    copy_nonoverlapping(bytes.as_ptr(), dst.as_mut_ptr(), 8);
  }
}

/// Decode the first 4-bytes of `src` in little-endian and return the result.
pub fn decode_fixed_32(src: &[u8]) -> u32 {
  assert!(src.len() >= 4);
  let mut data: u32 = 0;
  unsafe {
    copy_nonoverlapping(
      src.as_ptr(),
      &mut data as *mut u32 as *mut u8,
      4
    );
  }
  data.to_le()
}

/// Decode the first 8-bytes of `src` in little-endian and return the result.
pub fn decode_fixed_64(src: &[u8]) -> u64 {
  assert!(src.len() >= 8);
  let mut data: u64 = 0;
  unsafe {
    copy_nonoverlapping(
      src.as_ptr(),
      &mut data as *mut u64 as *mut u8,
      8
    );
  }
  data.to_le()
}

/// Encode the `value` in varint32 and put it in the first N-bytes of `dst`.
/// Return N.
/// This panics if `dst` doesn't have enough space to encode the value.
pub fn encode_varint_32(dst: &mut [u8], value: u32) -> usize {
  const B: u32 = 0x80;
  let mut idx = 0;
  if value < (1 << 7) {
    dst[idx] = value as u8;
    idx += 1;
  } else if value < (1 << 14) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = (value >> 7) as u8;
    idx += 2;
  } else if value < (1 << 21) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = (value >> 14) as u8;
    idx += 3;
  } else if value < (1 << 28) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = ((value >> 14) | B) as u8;
    dst[idx + 3] = (value >> 21) as u8;
    idx += 4;
  } else {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = ((value >> 14) | B) as u8;
    dst[idx + 3] = ((value >> 21) | B) as u8;
    dst[idx + 4] = (value >> 28) as u8;
    idx += 5;
  }
  idx
}

/// Encode the `value` in varint64 and put it in the first N-bytes of `dst`.
/// Return N.
/// This panics if `dst` doesn't have enough space to encode the value.
pub fn encode_varint_64(dst: &mut [u8], mut value: u64) -> usize {
  let mut idx = 0;
  while value & 0xFFFFFFFFFFFFFF80 != 0 {
    dst[idx] = ((value & 0x7F) | 0x80) as u8;
    idx += 1;
    value >>= 7;
  }
  dst[idx] = (value & 0x7F) as u8;
  idx + 1
}

/// Decode varint32 from `src`.
/// If `src` begins with a valid varint32, return a tuple
/// where the first element is the decoded value, and the second element is
/// the number of bytes for the varint32.
/// If `src` doesn't contain a valid varint32, return `None`.
pub fn decode_varint_32(src: &[u8]) -> Option<(u32, usize)> {
  decode_varint_32_limit(src, src.len())
}

/// Decode varint32 from the first `limit` bytes of `src`.
/// If the first `limit` bytes of `src` contains a valid varint32,
/// return a tuple where the first element is the decoded value, and the
/// second element is the actual number of bytes for the varint32.
/// Otherwise, if the first `limit` bytes of `src` doesn't contain a valid
/// varint32, return `None`.
pub fn decode_varint_32_limit(
  src: &[u8],
  limit: usize
) -> Option<(u32, usize)> {
  assert!(src.len() >= 4);
  let mut shift = 0;
  let mut idx = 0;
  let mut result: u32 = 0;
  while shift <= 28 && idx < limit {
    let byte = src[idx];
    idx += 1;
    result |= ((byte & 0x7F) as u32) << shift;
    shift += 7;
    if byte & 0x80 == 0 {
      return Some((result, idx))
    }
  }
  None
}

/// Decode varint64 from `src`.
/// If `src` begins with a valid varint64, return a tuple
/// where the first element is the decoded value, and the second element is
/// the number of bytes for the varint64.
/// If `src` doesn't contain a valid varint64, return `None`.
pub fn decode_varint_64(src: &[u8]) -> Option<(u64, usize)> {
  decode_varint_64_limit(src, src.len())
}

/// Decode varint64 from the first `limit` bytes of `src`.
/// If the first `limit` bytes of `src` contains a valid varint64,
/// return a tuple where the first element is the decoded value, and the
/// second element is the actual number of bytes for the varint64.
/// Otherwise, if the first `limit` bytes of `src` doesn't contain a valid
/// varint64, return `None`.
pub fn decode_varint_64_limit(
  src: &[u8],
  limit: usize
) -> Option<(u64, usize)> {
  assert!(src.len() >= 8);
  let mut shift = 0;
  let mut idx = 0;
  let mut result: u64 = 0;
  while shift <= 63 && idx < limit {
    let byte = src[idx];
    idx += 1;
    result |= ((byte & 0x7F) as u64) << shift;
    shift += 7;
    if byte & 0x80 == 0 {
      return Some((result, idx))
    }
  }
  None
}

/// Return the length of the varint32 or varint64 encoding of `v`
pub fn varint_length(mut v: u64) -> usize {
  let mut len = 1;
  while v >= 128 {
    v >>= 7;
    len += 1;
  }
  len
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  fn test_fixed_32() {
    const N: usize = 100000;
    let mut data: Vec<u8> = vec![0; N * 4];

    for i in 0..N {
      encode_fixed_32(&mut data[i*4..], i as u32);
    }

    for i in 0..N {
      let actual: u32 = decode_fixed_32(&data[i*4..]);
      assert_eq!(actual, i as u32);
    }
  }

  #[test]
  fn test_fixed_64() {
    let mut data: Vec<u8> = vec![0; 64 * 3 * 8];

    for power in 0..64 {
      let v = 1 << power as u64;
      encode_fixed_64(&mut data[(power*3+0)*8..], v - 1);
      encode_fixed_64(&mut data[(power*3+1)*8..], v + 0);
      encode_fixed_64(&mut data[(power*3+2)*8..], v + 1);
    }

    for power in 0..64 {
      let v = 1 << power as u64;
      let actual = decode_fixed_64(&data[(power*3+0)*8..]);
      assert_eq!(actual, v - 1);

      let actual = decode_fixed_64(&data[(power*3+1)*8..]);
      assert_eq!(actual, v + 0);

      let actual = decode_fixed_64(&data[(power*3+2)*8..]);
      assert_eq!(actual, v + 1);
    }
  }

  #[test]
  fn test_varint_32() {
    let mut idx = 0;
    let mut data: Vec<u8> = vec![0; 32 * 32 * 5];

    for i in 0..32*32 {
      let v: u32 = (i / 32) << (i % 32);
      idx += encode_varint_32(&mut data[idx..], v);
    }

    idx = 0;
    for i in 0..32*32 {
      let expected = (i / 32) << (i % 32);
      let result = decode_varint_32(&data[idx..]);
      assert!(result.is_some());
      let (actual, next_idx) = result.unwrap();
      assert_eq!(actual, expected);
      idx += next_idx;
    }
  }

  #[test]
  fn test_varint_64() {
    let mut values = Vec::new();
    values.push(0);
    values.push(100);
    values.push(!0u64);
    values.push(!0u64 - 1);
    for k in 0..64 {
      let power = 1u64 << k;
      values.push(power);
      values.push(power-1);
      values.push(power+1);
    }

    let mut data: Vec<u8> = vec![0; 196 * 10];
    let mut idx = 0;
    for v in &values {
      idx += encode_varint_64(&mut data[idx..], *v);
    }

    idx = 0;
    for v in &values {
      let result = decode_varint_64(&data[idx..]);
      assert!(result.is_some());
      let (actual, offset) = result.unwrap();
      assert_eq!(actual, *v);
      idx += offset;
    }
  }
}
<commit_msg>More test coverage for coding and improve docs<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use std::mem::transmute;
use std::ptr::copy_nonoverlapping;

use slice::Slice;

/// --------------------------------------------------------------------------------
/// Encoding & Decoding which deal with primitive Rust slices
/// --------------------------------------------------------------------------------

/// Encodes `value` in little-endian and puts it in the first 4-bytes of `dst`.
///
/// Panics if `dst.len()` is less than 4.
pub fn encode_fixed_32(dst: &mut [u8], value: u32) {
  assert!(dst.len() >= 4);
  unsafe {
    let bytes = transmute::<_, [u8; 4]>(value.to_le());
    copy_nonoverlapping(bytes.as_ptr(), dst.as_mut_ptr(), 4);
  }
}

/// Encodes `value` in little-endian and puts in the first 8-bytes of `dst`.
///
/// Panics if `dst.len()` is less than 8.
pub fn encode_fixed_64(dst: &mut [u8], value: u64) {
  assert!(dst.len() >= 8);
  unsafe {
    let bytes = transmute::<_, [u8; 8]>(value.to_le());
    copy_nonoverlapping(bytes.as_ptr(), dst.as_mut_ptr(), 8);
  }
}

/// Decodes the first 4-bytes of `src` in little-endian and returns the decoded value.
///
/// Panics if `src.len()` is less than 4.
pub fn decode_fixed_32(src: &[u8]) -> u32 {
  assert!(src.len() >= 4);
  let mut data: u32 = 0;
  unsafe {
    copy_nonoverlapping(
      src.as_ptr(),
      &mut data as *mut u32 as *mut u8,
      4
    );
  }
  data.to_le()
}

/// Decodes the first 8-bytes of `src` in little-endian and returns the decoded value.
///
/// Panics if `src.len()` is less than 8.
pub fn decode_fixed_64(src: &[u8]) -> u64 {
  assert!(src.len() >= 8);
  let mut data: u64 = 0;
  unsafe {
    copy_nonoverlapping(
      src.as_ptr(),
      &mut data as *mut u64 as *mut u8,
      8
    );
  }
  data.to_le()
}

/// Encode thes `value` in varint32 and puts it in the first `N`-bytes of `dst`.
/// Returns N.
///
/// Panics if `dst` doesn't have enough space to encode the value.
pub fn encode_varint_32(dst: &mut [u8], value: u32) -> usize {
  const B: u32 = 0x80;
  let mut idx = 0;
  if value < (1 << 7) {
    dst[idx] = value as u8;
    idx += 1;
  } else if value < (1 << 14) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = (value >> 7) as u8;
    idx += 2;
  } else if value < (1 << 21) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = (value >> 14) as u8;
    idx += 3;
  } else if value < (1 << 28) {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = ((value >> 14) | B) as u8;
    dst[idx + 3] = (value >> 21) as u8;
    idx += 4;
  } else {
    dst[idx] = (value | B) as u8;
    dst[idx + 1] = ((value >> 7) | B) as u8;
    dst[idx + 2] = ((value >> 14) | B) as u8;
    dst[idx + 3] = ((value >> 21) | B) as u8;
    dst[idx + 4] = (value >> 28) as u8;
    idx += 5;
  }
  idx
}

/// Encodes the `value` in varint64 and puts it in the first `N`-bytes of `dst`.
/// Returns N.
///
/// Panics if `dst` doesn't have enough space to encode the value.
pub fn encode_varint_64(dst: &mut [u8], mut value: u64) -> usize {
  let mut idx = 0;
  while value & 0xFFFFFFFFFFFFFF80 != 0 {
    dst[idx] = ((value & 0x7F) | 0x80) as u8;
    idx += 1;
    value >>= 7;
  }
  dst[idx] = (value & 0x7F) as u8;
  idx + 1
}

/// Decodes varint32 from `src`, and returns a tuple of which the first element is the
/// decoded value, and the second element is the number of bytes used to encode the result
/// value.
///
/// If `src` doesn't contain a valid varint32, returns `None`.
pub fn decode_varint_32(src: &[u8]) -> Option<(u32, usize)> {
  decode_varint_32_limit(src, src.len())
}

/// Decodes varint32 from the first `limit` bytes of `src`, and returns a tuple of which
/// the first is the decoded value, and the second element is the number of bytes used to
/// encode the result value.

/// If `src.len` is less than `limit`, or the first `limit` bytes of `src` doesn't contain
/// a valid varint32, returns `None`.
pub fn decode_varint_32_limit(src: &[u8], limit: usize) -> Option<(u32, usize)> {
  assert!(src.len() >= limit);
  let mut shift = 0;
  let mut idx = 0;
  let mut result: u32 = 0;
  while shift <= 28 && idx < limit {
    let byte = src[idx];
    idx += 1;
    result |= ((byte & 0x7F) as u32) << shift;
    shift += 7;
    if byte & 0x80 == 0 {
      return Some((result, idx))
    }
  }
  None
}

/// Decodes varint64 from `src`, and returns a tuple of which the first is the decoded
/// value, and the second element is the number of bytes used to encode the result value.
///
/// If `src` doesn't contain a valid varint64, returns `None`.
pub fn decode_varint_64(src: &[u8]) -> Option<(u64, usize)> {
  decode_varint_64_limit(src, src.len())
}

/// Decodes varint64 from the first `limit` bytes of `src`, and returns a tuple of which
/// the first is the decoded value, and the second element is the number of bytes used to
/// encode the result value.
///
/// If `src.len` is less than `limit`, or the first `limit` bytes of `src` doesn't contain
/// a valid varint64, returns `None`.
pub fn decode_varint_64_limit(src: &[u8], limit: usize) -> Option<(u64, usize)> {
  assert!(src.len() >= limit);
  let mut shift = 0;
  let mut idx = 0;
  let mut result: u64 = 0;
  while shift <= 63 && idx < limit {
    let byte = src[idx];
    idx += 1;
    result |= ((byte & 0x7F) as u64) << shift;
    shift += 7;
    if byte & 0x80 == 0 {
      return Some((result, idx))
    }
  }
  None
}

/// Returns the length of the varint32 or varint64 encoding of `v`
pub fn varint_length(mut v: u64) -> usize {
  let mut len = 1;
  while v >= 128 {
    v >>= 7;
    len += 1;
  }
  len
}

#[cfg(test)]
mod tests {
  use super::*;
  use util::random::Random;

  #[test]
  fn test_fixed_32() {
    const N: usize = 100000;
    let mut data: Vec<u8> = vec![0; N * 4];

    for i in 0..N {
      encode_fixed_32(&mut data[i*4..], i as u32);
    }

    for i in 0..N {
      let actual: u32 = decode_fixed_32(&data[i*4..]);
      assert_eq!(actual, i as u32);
    }
  }

  #[test]
  #[should_panic]
  fn test_fixed_32_panic() {
    let mut data = vec![0; 3];
    encode_fixed_32(&mut data, 100);
  }

  #[test]
  fn test_fixed_64() {
    let mut data: Vec<u8> = vec![0; 64 * 3 * 8];

    for power in 0..64 {
      let v = 1 << power as u64;
      encode_fixed_64(&mut data[(power*3+0)*8..], v - 1);
      encode_fixed_64(&mut data[(power*3+1)*8..], v + 0);
      encode_fixed_64(&mut data[(power*3+2)*8..], v + 1);
    }

    for power in 0..64 {
      let v = 1 << power as u64;
      let actual = decode_fixed_64(&data[(power*3+0)*8..]);
      assert_eq!(actual, v - 1);

      let actual = decode_fixed_64(&data[(power*3+1)*8..]);
      assert_eq!(actual, v + 0);

      let actual = decode_fixed_64(&data[(power*3+2)*8..]);
      assert_eq!(actual, v + 1);
    }
  }

  #[test]
  #[should_panic]
  fn test_fixed_64_panic() {
    let mut data = vec![0; 6];
    encode_fixed_64(&mut data, 100);
  }

  #[test]
  fn test_varint_32() {
    let mut idx = 0;
    let mut data: Vec<u8> = vec![0; 32 * 32 * 5];

    for i in 0..32*32 {
      let v: u32 = (i / 32) << (i % 32);
      idx += encode_varint_32(&mut data[idx..], v);
    }

    idx = 0;
    for i in 0..32*32 {
      let expected = (i / 32) << (i % 32);
      let result = decode_varint_32(&data[idx..]);
      assert!(result.is_some());
      let (actual, next_idx) = result.unwrap();
      assert_eq!(actual, expected);
      idx += next_idx;
    }
  }

  #[test]
  #[should_panic]
  fn test_varint_32_no_space() {
    let mut data = vec![0; 1];
    let _ = encode_varint_32(&mut data, 128);
  }

  #[test]
  fn test_varint_32_limit() {
    let mut data = vec![0; 5];
    for i in 0..32*32 {
      let v: u32 = (i / 32) << (i % 32);
      let limit = encode_varint_32(&mut data, v);
      assert!(limit <= 5);
      let result = decode_varint_32_limit(&mut data, limit);
      assert!(result.is_some());
      let (actual, len) = result.unwrap();
      assert_eq!(actual, v);
      assert_eq!(len, limit);
    }
  }

  #[test]
  fn test_varint_64() {
    let mut values = Vec::new();
    values.push(0);
    values.push(100);
    values.push(!0u64);
    values.push(!0u64 - 1);
    for k in 0..64 {
      let power = 1u64 << k;
      values.push(power);
      values.push(power-1);
      values.push(power+1);
    }

    let mut data: Vec<u8> = vec![0; 196 * 10];
    let mut idx = 0;
    for v in &values {
      idx += encode_varint_64(&mut data[idx..], *v);
    }

    idx = 0;
    for v in &values {
      let result = decode_varint_64(&data[idx..]);
      assert!(result.is_some());
      let (actual, offset) = result.unwrap();
      assert_eq!(actual, *v);
      idx += offset;
    }
  }

  #[test]
  fn test_varint_64_limit() {
    let mut data = vec![0; 10];
    for i in 0..64*64 {
      let v: u64 = (i / 64) << (i % 64);
      let limit = encode_varint_64(&mut data, v);
      assert!(limit <= 10);
      let result = decode_varint_64_limit(&mut data, limit);
      assert!(result.is_some());
      let (actual, len) = result.unwrap();
      assert_eq!(actual, v);
      assert_eq!(len, limit);
    }
  }

  #[test]
  #[should_panic]
  fn test_varint_64_no_space() {
    let mut data = vec![0; 4];
    let _ = encode_varint_64(&mut data, 2147483648u64);
  }

  #[test]
  fn test_varint_length() {
    let rand = Random::new(0xbaaaaaad);
    let mut data = vec![0; 5];
    for _ in 0..1000 {
      let v = rand.next();
      let len = encode_varint_32(&mut data, v);
      assert_eq!(varint_length(v as u64), len);
    }
  }
}
<|endoftext|>


<commit_before>//! Utilities to parse and validate Discord tokens.

use std::{fmt, str};

use crate::model::id::UserId;

/// Validates that a token is likely in a valid format.
///
/// This performs the following checks on a given token:
///
/// - Is not empty;
/// - Contains 3 parts (split by the period char `'.'`);
/// - The second part of the token is at least 6 characters long;
///
/// # Examples
///
/// Validate that a token is valid and that a number of malformed tokens are
/// actually invalid:
///
/// ```
/// use serenity::utils::token::validate;
///
/// // ensure a valid token is in fact a valid format:
/// assert!(validate("Mjg4NzYwMjQxMzYzODc3ODg4.C_ikow.j3VupLBuE1QWZng3TMGH0z_UAwg").is_ok());
///
/// assert!(validate("Mjg4NzYwMjQxMzYzODc3ODg4").is_err());
/// assert!(validate("").is_err());
/// ```
///
/// # Errors
///
/// Returns a [`InvalidToken`] when one of the above checks fail.
/// The type of failure is not specified.
pub fn validate(token: impl AsRef<str>) -> Result<(), InvalidToken> {
    parse(token).map(|_| ()).ok_or(InvalidToken)
}

/// Error that can be return by [`validate`].
#[derive(Debug)]
pub struct InvalidToken;

impl std::error::Error for InvalidToken {}

impl fmt::Display for InvalidToken {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str("The provided token was invalid")
    }
}

/// Verifies that the token adheres to the Discord token format and extracts the bot user ID and
/// the token generation unix timestamp.
pub fn parse(token: impl AsRef<str>) -> Option<(UserId, i64)> {
    // The token consists of three base64-encoded parts
    let mut parts = token.as_ref().split('.');

    // First part must be a base64-encoded stringified user ID
    let user_id = base64::decode_config(parts.next()?, base64::URL_SAFE).ok()?;
    let user_id = UserId(str::from_utf8(&user_id).ok()?.parse().ok()?);

    // Second part must be a base64-encoded token generation timestamp
    let timestamp = parts.next()?;
    // The base64-encoded timestamp must be at least 6 characters
    if timestamp.len() < 6 {
        return None;
    }
    let timestamp_bytes = base64::decode_config(timestamp, base64::URL_SAFE).ok()?;
    let mut timestamp = 0;
    for byte in timestamp_bytes {
        timestamp *= 256;
        timestamp += byte as i64;
    }
    // Some timestamps are based on the Discord epoch. Convert to Unix epoch
    if timestamp < 1_293_840_000 {
        timestamp += 1_293_840_000;
    }

    // Third part is a base64-encoded HMAC that's not interesting on its own
    base64::decode_config(parts.next()?, base64::URL_SAFE).ok()?;

    Some((user_id, timestamp))
}
<commit_msg>Strip leading "Bot " in parse_token (#1889)<commit_after>//! Utilities to parse and validate Discord tokens.

use std::{fmt, str};

use crate::model::id::UserId;

/// Validates that a token is likely in a valid format.
///
/// This performs the following checks on a given token:
///
/// - Is not empty;
/// - Contains 3 parts (split by the period char `'.'`);
/// - The second part of the token is at least 6 characters long;
///
/// # Examples
///
/// Validate that a token is valid and that a number of malformed tokens are
/// actually invalid:
///
/// ```
/// use serenity::utils::token::validate;
///
/// // ensure a valid token is in fact a valid format:
/// assert!(validate("Mjg4NzYwMjQxMzYzODc3ODg4.C_ikow.j3VupLBuE1QWZng3TMGH0z_UAwg").is_ok());
///
/// assert!(validate("Mjg4NzYwMjQxMzYzODc3ODg4").is_err());
/// assert!(validate("").is_err());
/// ```
///
/// # Errors
///
/// Returns a [`InvalidToken`] when one of the above checks fail.
/// The type of failure is not specified.
pub fn validate(token: impl AsRef<str>) -> Result<(), InvalidToken> {
    parse(token).map(|_| ()).ok_or(InvalidToken)
}

/// Error that can be return by [`validate`].
#[derive(Debug)]
pub struct InvalidToken;

impl std::error::Error for InvalidToken {}

impl fmt::Display for InvalidToken {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str("The provided token was invalid")
    }
}

/// Verifies that the token adheres to the Discord token format and extracts the bot user ID and
/// the token generation unix timestamp.
pub fn parse(token: impl AsRef<str>) -> Option<(UserId, i64)> {
    // The token consists of three base64-encoded parts
    // Tokens can be preceded by "Bot " (that's how the Discord API expects them)
    let mut parts = token.as_ref().trim_start_matches("Bot ").split('.');

    // First part must be a base64-encoded stringified user ID
    let user_id = base64::decode_config(parts.next()?, base64::URL_SAFE).ok()?;
    let user_id = UserId(str::from_utf8(&user_id).ok()?.parse().ok()?);

    // Second part must be a base64-encoded token generation timestamp
    let timestamp = parts.next()?;
    // The base64-encoded timestamp must be at least 6 characters
    if timestamp.len() < 6 {
        return None;
    }
    let timestamp_bytes = base64::decode_config(timestamp, base64::URL_SAFE).ok()?;
    let mut timestamp = 0;
    for byte in timestamp_bytes {
        timestamp *= 256;
        timestamp += byte as i64;
    }
    // Some timestamps are based on the Discord epoch. Convert to Unix epoch
    if timestamp < 1_293_840_000 {
        timestamp += 1_293_840_000;
    }

    // Third part is a base64-encoded HMAC that's not interesting on its own
    base64::decode_config(parts.next()?, base64::URL_SAFE).ok()?;

    Some((user_id, timestamp))
}
<|endoftext|>


<commit_before>#![feature(phase)]

extern crate cargo;
extern crate hammer;
extern crate serialize;
#[phase(plugin, link)]
extern crate log;

use hammer::{FlagConfig,FlagConfiguration};
use std::os;
use std::io::process::{Command,InheritFd,ExitStatus,ExitSignal};
use serialize::Encodable;
use cargo::{GlobalFlags, NoFlags, execute_main_without_stdin, handle_error, shell};
use cargo::core::MultiShell;
use cargo::util::important_paths::find_project;
use cargo::util::{CliError, CliResult, Require, config, human};

fn main() {
    execute();
}

#[deriving(Encodable)]
struct ProjectLocation {
    root: String
}

/**
  The top-level `cargo` command handles configuration and project location
  because they are fundamental (and intertwined). Other commands can rely
  on this top-level information.
*/
fn execute() {
    debug!("executing; cmd=cargo; args={}", os::args());

    let (cmd, args) = process(os::args());

    match cmd.as_slice() {
        "config-for-key" => {
            log!(4, "cmd == config-for-key");
            execute_main_without_stdin(config_for_key)
        },
        "config-list" => {
            log!(4, "cmd == config-list");
            execute_main_without_stdin(config_list)
        },
        "locate-project" => {
            log!(4, "cmd == locate-project");
            execute_main_without_stdin(locate_project)
        },
        "--help" | "-h" | "help" | "-?" => {
            println!("Commands:");
            println!("  build          # compile the current project");
            println!("  test           # run the tests\n");

            let (_, options) = hammer::usage::<GlobalFlags>(false);
            println!("Options (for all commands):\n\n{}", options);
        },
        _ => {
            let command = Command::new(format!("cargo-{}", cmd))
                .args(args.as_slice())
                .stdin(InheritFd(0))
                .stdout(InheritFd(1))
                .stderr(InheritFd(2))
                .status();

            match command {
                Ok(ExitStatus(0)) => (),
                Ok(ExitStatus(i)) | Ok(ExitSignal(i)) => {
                    handle_error(CliError::new("", i as uint), &mut shell(false))
                }
                Err(_) => handle_error(CliError::new("No such subcommand", 127), &mut shell(false))
            }
        }
    }
}

fn process(args: Vec<String>) -> (String, Vec<String>) {
    let mut args = Vec::from_slice(args.tail());
    let head = args.shift().unwrap_or("--help".to_str());

    (head, args)
}

#[deriving(Encodable)]
struct ConfigOut {
    values: std::collections::HashMap<String, config::ConfigValue>
}

#[deriving(Decodable)]
struct ConfigForKeyFlags {
    key: String,
    human: bool
}

impl FlagConfig for ConfigForKeyFlags {
    fn config(_: Option<ConfigForKeyFlags>,
              config: FlagConfiguration) -> FlagConfiguration {
        config.short("human", 'h')
    }
}

fn config_for_key(args: ConfigForKeyFlags, _: &mut MultiShell) -> CliResult<Option<ConfigOut>> {
    let value = try!(config::get_config(os::getcwd(),
                                        args.key.as_slice()).map_err(|_| {
        CliError::new("Couldn't load configuration",  1)
    }));

    if args.human {
        println!("{}", value);
        Ok(None)
    } else {
        let mut map = std::collections::HashMap::new();
        map.insert(args.key.clone(), value);
        Ok(Some(ConfigOut { values: map }))
    }
}

#[deriving(Decodable)]
struct ConfigListFlags {
    human: bool
}

impl FlagConfig for ConfigListFlags {
    fn config(_: Option<ConfigListFlags>,
              config: FlagConfiguration) -> FlagConfiguration {
        config.short("human", 'h')
    }
}

fn config_list(args: ConfigListFlags, _: &mut MultiShell) -> CliResult<Option<ConfigOut>> {
    let configs = try!(config::all_configs(os::getcwd()).map_err(|_|
        CliError::new("Couldn't load configuration", 1)));

    if args.human {
        for (key, value) in configs.iter() {
            println!("{} = {}", key, value);
        }
        Ok(None)
    } else {
        Ok(Some(ConfigOut { values: configs }))
    }
}

fn locate_project(_: NoFlags, _: &mut MultiShell) -> CliResult<Option<ProjectLocation>> {
    let root = try!(find_project(&os::getcwd(), "Cargo.toml").map_err(|e| {
        CliError::from_boxed(e, 1)
    }));

    let string = try!(root.as_str()
                      .require(|| human("Your project path contains characters \
                                         not representable in Unicode"))
                      .map_err(|e| CliError::from_boxed(e, 1)));

    Ok(Some(ProjectLocation { root: string.to_str() }))
}
<commit_msg>Enable execution without cargo in PATH<commit_after>#![feature(phase)]

extern crate cargo;
extern crate hammer;
extern crate serialize;
#[phase(plugin, link)]
extern crate log;

use hammer::{FlagConfig,FlagConfiguration};
use std::os;
use std::io::process::{Command,InheritFd,ExitStatus,ExitSignal};
use serialize::Encodable;
use cargo::{GlobalFlags, NoFlags, execute_main_without_stdin, handle_error, shell};
use cargo::core::MultiShell;
use cargo::util::important_paths::find_project;
use cargo::util::{CliError, CliResult, Require, config, human};

fn main() {
    execute();
}

#[deriving(Encodable)]
struct ProjectLocation {
    root: String
}

/**
  The top-level `cargo` command handles configuration and project location
  because they are fundamental (and intertwined). Other commands can rely
  on this top-level information.
*/
fn execute() {
    debug!("executing; cmd=cargo; args={}", os::args());

    let (cmd, args) = process(os::args());

    match cmd.as_slice() {
        "config-for-key" => {
            log!(4, "cmd == config-for-key");
            execute_main_without_stdin(config_for_key)
        },
        "config-list" => {
            log!(4, "cmd == config-list");
            execute_main_without_stdin(config_list)
        },
        "locate-project" => {
            log!(4, "cmd == locate-project");
            execute_main_without_stdin(locate_project)
        },
        "--help" | "-h" | "help" | "-?" => {
            println!("Commands:");
            println!("  build          # compile the current project");
            println!("  test           # run the tests\n");

            let (_, options) = hammer::usage::<GlobalFlags>(false);
            println!("Options (for all commands):\n\n{}", options);
        },
        _ => {
            let command = format!("cargo-{}", cmd);
            let mut command = match os::self_exe_path() {
                Some(path) => Command::new(path.join(command)),
                None => Command::new(command),
            };
            let command = command
                .args(args.as_slice())
                .stdin(InheritFd(0))
                .stdout(InheritFd(1))
                .stderr(InheritFd(2))
                .status();

            match command {
                Ok(ExitStatus(0)) => (),
                Ok(ExitStatus(i)) | Ok(ExitSignal(i)) => {
                    handle_error(CliError::new("", i as uint), &mut shell(false))
                }
                Err(_) => handle_error(CliError::new("No such subcommand", 127), &mut shell(false))
            }
        }
    }
}

fn process(args: Vec<String>) -> (String, Vec<String>) {
    let mut args = Vec::from_slice(args.tail());
    let head = args.shift().unwrap_or("--help".to_str());

    (head, args)
}

#[deriving(Encodable)]
struct ConfigOut {
    values: std::collections::HashMap<String, config::ConfigValue>
}

#[deriving(Decodable)]
struct ConfigForKeyFlags {
    key: String,
    human: bool
}

impl FlagConfig for ConfigForKeyFlags {
    fn config(_: Option<ConfigForKeyFlags>,
              config: FlagConfiguration) -> FlagConfiguration {
        config.short("human", 'h')
    }
}

fn config_for_key(args: ConfigForKeyFlags, _: &mut MultiShell) -> CliResult<Option<ConfigOut>> {
    let value = try!(config::get_config(os::getcwd(),
                                        args.key.as_slice()).map_err(|_| {
        CliError::new("Couldn't load configuration",  1)
    }));

    if args.human {
        println!("{}", value);
        Ok(None)
    } else {
        let mut map = std::collections::HashMap::new();
        map.insert(args.key.clone(), value);
        Ok(Some(ConfigOut { values: map }))
    }
}

#[deriving(Decodable)]
struct ConfigListFlags {
    human: bool
}

impl FlagConfig for ConfigListFlags {
    fn config(_: Option<ConfigListFlags>,
              config: FlagConfiguration) -> FlagConfiguration {
        config.short("human", 'h')
    }
}

fn config_list(args: ConfigListFlags, _: &mut MultiShell) -> CliResult<Option<ConfigOut>> {
    let configs = try!(config::all_configs(os::getcwd()).map_err(|_|
        CliError::new("Couldn't load configuration", 1)));

    if args.human {
        for (key, value) in configs.iter() {
            println!("{} = {}", key, value);
        }
        Ok(None)
    } else {
        Ok(Some(ConfigOut { values: configs }))
    }
}

fn locate_project(_: NoFlags, _: &mut MultiShell) -> CliResult<Option<ProjectLocation>> {
    let root = try!(find_project(&os::getcwd(), "Cargo.toml").map_err(|e| {
        CliError::from_boxed(e, 1)
    }));

    let string = try!(root.as_str()
                      .require(|| human("Your project path contains characters \
                                         not representable in Unicode"))
                      .map_err(|e| CliError::from_boxed(e, 1)));

    Ok(Some(ProjectLocation { root: string.to_str() }))
}
<|endoftext|>