<commit_before>//! Access secure credentials at runtime with multiple backends.
//!
//! For more information, see [the
//! homepage](https://github.com/emk/credentials).
//!
//! ```
//! use credentials;
//! use std::env;
//!
//! env::set_var("PASSWORD", "secret");
//! assert_eq!("secret", credentials::var("PASSWORD").unwrap());
//! ```

#[macro_use]
extern crate hyper;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
extern crate regex;
extern crate rustc_serialize;
#[cfg(test)] #[macro_use]
extern crate yup_hyper_mock as hyper_mock;

use backend::{Backend, BoxedError, err};
use std::convert::AsRef;
use std::ops::{Deref, DerefMut};
use std::error::{self, Error};
use std::fmt;
use std::sync::{Mutex, MutexGuard};

// Be very careful not to export any more of the Secretfile API than
// strictly necessary, because we don't want to stablize too much at this
// point.
pub use secretfile::Secretfile;

#[macro_use]
mod backend;
mod chained;
mod envvar;
mod secretfile;
mod vault;

lazy_static! {
    // Our shared global client, initialized by `lazy_static!` and
    // protected by a Mutex.  There's no reason why we couldn't create a
    // per-thread client API for performance, but this will do for now.
    static ref BACKEND: Mutex<Result<chained::Client, BoxedError>> =
        Mutex::new(chained::Client::new_default());
}

/// An error occurred accessing credentials.
#[derive(Debug)]
pub struct CredentialError {
    credential: String,
    original: Option<backend::BoxedError>,
}

impl error::Error for CredentialError {
    fn description(&self) -> &str { "can't access secure credential" }
    fn cause(&self) -> Option<&error::Error> {
        match self.original {
            None => None,
            Some(ref bx) => Some(bx.deref() as &std::error::Error),
        }
    }
}

impl fmt::Display for CredentialError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.original.is_none() {
            write!(f, "{} {}", self.description(), self.credential)
        } else {
            write!(f, "{} {}: {}", self.description(), self.credential,
                   self.cause().unwrap())
        }
    }
}

/// Helper function for `var`, below.
fn var_inner(key: &str) -> Result<String, BoxedError> {
    // This is a bit subtle: First we need to lock our Mutex, and then--if
    // our Mutex was poisoned by a panic in another thread--we want to
    // propagate the panic in this thread using `unwrap()`.  See
    // https://doc.rust-lang.org/std/sync/struct.Mutex.html for details.
    let mut backend_result: MutexGuard<_> = BACKEND.lock().unwrap();

    // Deref our MutexGuard as a mutable reference, and then check to see
    // if it was initialized correctly.  I had to tweak the `mut` bits for
    // a few minutes to get this past the borrow checker.
    match backend_result.deref_mut() {
        &mut Ok(ref mut backend) => backend.var(key),
        &mut Err(ref e) => Err(err!("Could not initialize: {}", e)),
    }
}

/// Fetch the value of a credential.
pub fn var<K: AsRef<str>>(key: K) -> Result<String, CredentialError> {
    let key_ref = key.as_ref();
    trace!("getting secure credential {}", key_ref);
    var_inner(key.as_ref()).map_err(|e| {
        let err = CredentialError {
            credential: key_ref.to_owned(),
            original: Some(e),
        };
        warn!("{}", err);
        err
    })
}
<commit_msg>Add a credentials::file function<commit_after>//! Access secure credentials at runtime with multiple backends.
//!
//! For more information, see [the
//! homepage](https://github.com/emk/credentials).
//!
//! ```
//! use credentials;
//! use std::env;
//!
//! env::set_var("PASSWORD", "secret");
//! assert_eq!("secret", credentials::var("PASSWORD").unwrap());
//! ```

#[macro_use]
extern crate hyper;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
extern crate regex;
extern crate rustc_serialize;
#[cfg(test)] #[macro_use]
extern crate yup_hyper_mock as hyper_mock;

use backend::{Backend, BoxedError, err};
use std::convert::AsRef;
use std::ops::{Deref, DerefMut};
use std::error::{self, Error};
use std::fmt;
use std::path::Path;
use std::sync::{Mutex, MutexGuard};

// Be very careful not to export any more of the Secretfile API than
// strictly necessary, because we don't want to stablize too much at this
// point.
pub use secretfile::Secretfile;

#[macro_use]
mod backend;
mod chained;
mod envvar;
mod secretfile;
mod vault;

lazy_static! {
    // Our shared global client, initialized by `lazy_static!` and
    // protected by a Mutex.  There's no reason why we couldn't create a
    // per-thread client API for performance, but this will do for now.
    static ref BACKEND: Mutex<Result<chained::Client, BoxedError>> =
        Mutex::new(chained::Client::new_default());
}

/// An error occurred accessing credentials.
#[derive(Debug)]
pub struct CredentialError {
    credential: String,
    original: Option<backend::BoxedError>,
}

impl error::Error for CredentialError {
    fn description(&self) -> &str { "can't access secure credential" }
    fn cause(&self) -> Option<&error::Error> {
        match self.original {
            None => None,
            Some(ref bx) => Some(bx.deref() as &std::error::Error),
        }
    }
}

impl fmt::Display for CredentialError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.original.is_none() {
            write!(f, "{} {}", self.description(), self.credential)
        } else {
            write!(f, "{} {}: {}", self.description(), self.credential,
                   self.cause().unwrap())
        }
    }
}

/// Helper function for `var`, below.
fn var_inner(key: &str) -> Result<String, BoxedError> {
    // This is a bit subtle: First we need to lock our Mutex, and then--if
    // our Mutex was poisoned by a panic in another thread--we want to
    // propagate the panic in this thread using `unwrap()`.  See
    // https://doc.rust-lang.org/std/sync/struct.Mutex.html for details.
    let mut backend_result: MutexGuard<_> = BACKEND.lock().unwrap();

    // Deref our MutexGuard as a mutable reference, and then check to see
    // if it was initialized correctly.  I had to tweak the `mut` bits for
    // a few minutes to get this past the borrow checker.
    match backend_result.deref_mut() {
        &mut Ok(ref mut backend) => backend.var(key),
        &mut Err(ref e) => Err(err!("Could not initialize: {}", e)),
    }
}

/// Helper function for `file`, below.
fn file_inner(key: &str) -> Result<String, BoxedError> {
    let mut backend_result: MutexGuard<_> = BACKEND.lock().unwrap();
    match backend_result.deref_mut() {
        &mut Ok(ref mut backend) => backend.file(key),
        &mut Err(ref e) => Err(err!("Could not initialize: {}", e)),
    }
}

/// Fetch the value of an environment-variable-style credential.
pub fn var<S: AsRef<str>>(name: S) -> Result<String, CredentialError> {
    let name_ref = name.as_ref();
    trace!("getting secure credential {}", name_ref);
    var_inner(name.as_ref()).map_err(|e| {
        let err = CredentialError {
            credential: name_ref.to_owned(),
            original: Some(e),
        };
        warn!("{}", err);
        err
    })
}

/// Fetch the value of an file-style credential.
pub fn file<S: AsRef<Path>>(path: S) -> Result<String, CredentialError> {
    let path_ref = path.as_ref();
    let path_str = try!(path_ref.to_str().ok_or_else(|| {
        CredentialError {
            credential: "(invalid path)".to_owned(),
            original: Some(err!("Path is not valid Unicode")),
        }
    }));
    trace!("getting secure credential {}", path_str);
    file_inner(path_str).map_err(|e| {
        let err = CredentialError {
            credential: path_str.to_owned(),
            original: Some(e),
        };
        warn!("{}", err);
        err
    })
}

#[cfg(test)]
mod test {
    use std::fs;
    use std::io::Read;
    use super::file;

    #[test]
    fn test_file() {
        // Some arbitrary file contents.
        let mut f = fs::File::open("Cargo.toml").unwrap();
        let mut expected = String::new();
        f.read_to_string(&mut expected).unwrap();

        assert_eq!(expected, file("Cargo.toml").unwrap());
        assert!(file("nosuchfile.txt").is_err());
    }
}
<|endoftext|>


<commit_before>extern crate time;

use std::iter::Iterator;

use time::{now_utc, Tm, Duration};

pub struct ProgressRecord {
    num: usize,
    iterating_for: Duration,

}

impl ProgressRecord {
    /// Returns a basic log message of where we are now. You can construct this yourself, but this
    /// is a helpful convience method.
    pub fn message(&self) -> String {
        format!("Have seen {} items and been iterating for {}", self.num_done(), self.iterating_for.num_seconds())
    }

    /// Duration since iteration started
    pub fn duration_since_start(&self) -> Duration {
        self.iterating_for
    }

    /// Number of items we've generated so far
    pub fn num_done(&self) -> usize {
        self.num
    }

    /// Prints a basic message
    pub fn print_message(&self) {
        println!("{}", self.message());
    }

    /// Number of items per second
    pub fn rate(&self) -> f32 {
        // number of items per second
        (self.num_done() as f32) / (self.duration_since_start().num_seconds() as f32)
    }

    /// None if we don't know how much we've done (as a fraction), otherwise a value form 0 to 1
    /// for what fraction along we are.
    pub fn fraction(&self) -> Option<f32> {
        None
    }

    /// None if we don't know how much we've done, otherwise value for 0 to 100 representing how
    /// far along as a percentage we are.
    pub fn percent(&self) -> Option<f32> {
        match self.fraction() {
            None => { None }
            Some(f) => { Some(f * 100.0) }
        }
    }

    /// If we want to print every `n` items, should we print now?
    pub fn should_print_every_items(&self, n: usize) -> bool {
        (self.num_done() - 1) % n == 0
    }

}

pub struct ProgressRecorderIter<I> {
    iter: I,
    count: usize,
    started_iterating: Tm,
}

impl<I> ProgressRecorderIter<I> {
    pub fn new(iter: I) -> ProgressRecorderIter<I> {
        ProgressRecorderIter{ iter: iter, count: 0, started_iterating: now_utc() }
    }

    fn generate_record(&mut self) -> ProgressRecord {
        self.count += 1;
        ProgressRecord{ num: self.count, iterating_for: now_utc() - self.started_iterating }
    }

}

pub trait ProgressableIter<I> {
    fn progress(self) -> ProgressRecorderIter<I>;
}

impl<I> ProgressableIter<I> for I where I: Iterator {
    fn progress(self) -> ProgressRecorderIter<I> {
        ProgressRecorderIter::new(self)
    }
}

impl<I> Iterator for ProgressRecorderIter<I> where I: Iterator {
    type Item = (ProgressRecord, <I as Iterator>::Item);

    #[inline]
    fn next(&mut self) -> Option<(ProgressRecord, <I as Iterator>::Item)> {
        self.iter.next().map(|a| {
            (self.generate_record(), a)
        })
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.iter.size_hint()
    }

    #[inline]
    fn count(self) -> usize {
        self.iter.count()
    }
}

mod test {
    #[test]
    fn test_simple() {
        use super::ProgressableIter;
        use std::thread::sleep_ms;
        use time::Duration;

        let vec: Vec<u8> = vec![0, 1, 2, 3, 4];
        let mut progressor = vec.iter().progress();

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 1 items and been iterating for 0");
        // It'll always print on the first one
        assert_eq!(state.should_print_every(2), true);
        assert_eq!(state.should_print_every(3), true);
        assert_eq!(state.should_print_every(5), true);
        assert_eq!(state.rate(), ::std::f32::INFINITY);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 2 items and been iterating for 1");
        assert_eq!(state.should_print_every(2), false);
        assert_eq!(state.should_print_every(3), false);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 2.);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 3 items and been iterating for 1");
        assert_eq!(state.should_print_every(2), true);
        assert_eq!(state.should_print_every(3), false);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 3.);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 4 items and been iterating for 2");
        assert_eq!(state.should_print_every(2), false);
        assert_eq!(state.should_print_every(3), true);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 2.);
    }
}

<commit_msg>feat: Add print_every method<commit_after>extern crate time;

use std::iter::Iterator;

use time::{now_utc, Tm, Duration};

pub struct ProgressRecord {
    num: usize,
    iterating_for: Duration,

}

impl ProgressRecord {
    /// Returns a basic log message of where we are now. You can construct this yourself, but this
    /// is a helpful convience method.
    pub fn message(&self) -> String {
        format!("Have seen {} items and been iterating for {}", self.num_done(), self.iterating_for.num_seconds())
    }

    /// Duration since iteration started
    pub fn duration_since_start(&self) -> Duration {
        self.iterating_for
    }

    /// Number of items we've generated so far
    pub fn num_done(&self) -> usize {
        self.num
    }

    /// Prints a basic message
    pub fn print_message(&self) {
        println!("{}", self.message());
    }

    /// Number of items per second
    pub fn rate(&self) -> f32 {
        // number of items per second
        (self.num_done() as f32) / (self.duration_since_start().num_seconds() as f32)
    }

    /// None if we don't know how much we've done (as a fraction), otherwise a value form 0 to 1
    /// for what fraction along we are.
    pub fn fraction(&self) -> Option<f32> {
        None
    }

    /// None if we don't know how much we've done, otherwise value for 0 to 100 representing how
    /// far along as a percentage we are.
    pub fn percent(&self) -> Option<f32> {
        match self.fraction() {
            None => { None }
            Some(f) => { Some(f * 100.0) }
        }
    }

    /// If we want to print every `n` items, should we print now?
    pub fn should_print_every_items(&self, n: usize) -> bool {
        (self.num_done() - 1) % n == 0
    }

    /// Print out `msg`, but only if there has been `n` items.
    /// Often you want to print out a debug message every 1,000 items or so. This function does
    /// that.
    pub fn print_every(&self, n: usize, msg: &str) {
        if should_print_every_items(n) {
            print!(msg);
        }
    }

}

pub struct ProgressRecorderIter<I> {
    iter: I,
    count: usize,
    started_iterating: Tm,
}

impl<I> ProgressRecorderIter<I> {
    pub fn new(iter: I) -> ProgressRecorderIter<I> {
        ProgressRecorderIter{ iter: iter, count: 0, started_iterating: now_utc() }
    }

    fn generate_record(&mut self) -> ProgressRecord {
        self.count += 1;
        ProgressRecord{ num: self.count, iterating_for: now_utc() - self.started_iterating }
    }

}

pub trait ProgressableIter<I> {
    fn progress(self) -> ProgressRecorderIter<I>;
}

impl<I> ProgressableIter<I> for I where I: Iterator {
    fn progress(self) -> ProgressRecorderIter<I> {
        ProgressRecorderIter::new(self)
    }
}

impl<I> Iterator for ProgressRecorderIter<I> where I: Iterator {
    type Item = (ProgressRecord, <I as Iterator>::Item);

    #[inline]
    fn next(&mut self) -> Option<(ProgressRecord, <I as Iterator>::Item)> {
        self.iter.next().map(|a| {
            (self.generate_record(), a)
        })
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.iter.size_hint()
    }

    #[inline]
    fn count(self) -> usize {
        self.iter.count()
    }
}

mod test {
    #[test]
    fn test_simple() {
        use super::ProgressableIter;
        use std::thread::sleep_ms;
        use time::Duration;

        let vec: Vec<u8> = vec![0, 1, 2, 3, 4];
        let mut progressor = vec.iter().progress();

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 1 items and been iterating for 0");
        // It'll always print on the first one
        assert_eq!(state.should_print_every(2), true);
        assert_eq!(state.should_print_every(3), true);
        assert_eq!(state.should_print_every(5), true);
        assert_eq!(state.rate(), ::std::f32::INFINITY);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 2 items and been iterating for 1");
        assert_eq!(state.should_print_every(2), false);
        assert_eq!(state.should_print_every(3), false);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 2.);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 3 items and been iterating for 1");
        assert_eq!(state.should_print_every(2), true);
        assert_eq!(state.should_print_every(3), false);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 3.);

        sleep_ms(500);
        let (state, _) = progressor.next().unwrap();
        assert_eq!(state.message(), "Have seen 4 items and been iterating for 2");
        assert_eq!(state.should_print_every(2), false);
        assert_eq!(state.should_print_every(3), true);
        assert_eq!(state.should_print_every(5), false);
        assert_eq!(state.rate(), 2.);
    }
}

<|endoftext|>


<commit_before>#![crate_name = "neovim"]
#![crate_type = "lib"]
#![crate_type = "rlib"]
#![allow(raw_pointer_deriving)]

extern crate libc;

use std::c_str::CString;
use std::fmt;
use std::string::String;
use std::vec::Vec;

#[cfg(target_os="macos")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    #[link(name = "iconv")]
    extern{}
}

#[cfg(target_os="linux")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    extern{}
}

#[cfg(target_os="windows")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    extern{}
}

mod ffi;

pub enum Object {
    Buffer(ffi::C_Buffer),
    Window(ffi::C_Window),
    Tabpage(ffi::C_Tabpage),
    Boolean(ffi::C_Boolean),
    Integer(ffi::C_Integer),
    Float(ffi::C_Float),
    String(String),
    Array(ffi::C_Array),
    Dictionary(ffi::C_Dictionary),
}

impl fmt::Show for Object {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Object::Buffer(ref obj) => write!(f, "Buffer({})", obj),
            Object::Window(ref obj) => write!(f, "Window({})", obj),
            Object::Tabpage(ref obj) => write!(f, "Tabpage({})", obj),
            Object::Boolean(ref obj) => write!(f, "Boolean({})", obj),
            Object::Integer(ref obj) => write!(f, "Integer({})", obj),
            Object::Float(ref obj) => write!(f, "Float({})", obj),
            Object::String(ref obj) => write!(f, "String({})", obj),
            Object::Array(ref obj) => {
                write!(f, "Array(").ok();
                for i in range(0, obj.size) {
                    let inner_obj_opt = unsafe { c_object_to_object(obj.items.offset(i as int)) };
                    if let Some(inner_obj) = inner_obj_opt {
                        write!(f, "{}", inner_obj).ok();
                        if i + 1 < obj.size {
                            write!(f, ", ").ok();
                        }
                    } else {
                        write!(f, "Nil ").ok();
                    }
                }
                write!(f, ")")
            },
            Object::Dictionary(ref obj) => write!(f, "Dictionary(Length: {})", obj.size),
        }
    }
}

unsafe fn c_object_to_object(obj: *mut ffi::C_Object) -> Option<Object> {
    match (*obj).object_type {
        ffi::ObjectType::BufferType =>
            Some(Object::Buffer((*(obj as *mut ffi::C_Object_Buffer)).data)),
        ffi::ObjectType::WindowType =>
            Some(Object::Window((*(obj as *mut ffi::C_Object_Window)).data)),
        ffi::ObjectType::TabpageType =>
            Some(Object::Tabpage((*(obj as *mut ffi::C_Object_Tabpage)).data)),
        ffi::ObjectType::NilType =>
            None,
        ffi::ObjectType::BooleanType =>
            Some(Object::Boolean((*(obj as *mut ffi::C_Object_Boolean)).data)),
        ffi::ObjectType::IntegerType =>
            Some(Object::Integer((*(obj as *mut ffi::C_Object_Integer)).data)),
        ffi::ObjectType::FloatType =>
            Some(Object::Float((*(obj as *mut ffi::C_Object_Float)).data)),
        ffi::ObjectType::StringType => {
            let vim_str: ffi::C_String = (*(obj as *mut ffi::C_Object_String)).data;
            let s = String::from_raw_buf_len(vim_str.data as *const u8, vim_str.size as uint);
            Some(Object::String(s))
        },
        ffi::ObjectType::ArrayType =>
            Some(Object::Array((*(obj as *mut ffi::C_Object_Array)).data)),
        ffi::ObjectType::DictionaryType =>
            Some(Object::Dictionary((*(obj as *mut ffi::C_Object_Dictionary)).data)),
    }
}

pub fn main_setup(args: Vec<String>) -> i32 {
    let v: Vec<CString> = args.iter().map(|s| s.as_slice().to_c_str()).collect();
    let vp: Vec<*const ffi::c_char> = v.iter().map(|s| s.as_ptr()).collect();
    let p_vp: *const *const ffi::c_char = vp.as_ptr();

    unsafe { ffi::nvim_main_setup(vp.len() as i32, p_vp) }
}

pub fn main_loop() -> i32 {
    unsafe { ffi::nvim_main_loop() }
}

pub fn channel_from_fds(read_fd: i32, write_fd: i32) -> u64 {
    unsafe { ffi::channel_from_fds(read_fd, write_fd) }
}

pub fn serialize_message(id: u64, method: &'static str, args: &Array) -> String {
    unsafe {
        let buf = ffi::vim_msgpack_new();
        let vim_str = ffi::C_String {data: method.as_ptr() as *const i8, size: method.len() as u64};
        ffi::vim_serialize_request(id, vim_str, args.unwrap_value(), buf);
        let s = String::from_raw_buf_len((*buf).data as *const u8, (*buf).size as uint);
        ffi::vim_msgpack_free(buf);
        s
    }
}

pub fn deserialize_message(message: &String) -> Array {
    unsafe {
        let mut arr_raw = Array::new().unwrap_value();
        let s = ffi::C_String {
            data: message.as_slice().as_ptr() as *const i8,
            size: message.len() as u64
        };
        ffi::vim_msgpack_parse(s, &mut arr_raw);
        Array::wrap_value(arr_raw)
    }
}

#[derive(Copy)]
pub struct Array {
    value: ffi::C_Array
}

impl Array {
    pub fn new() -> Array {
        Array {
            value: ffi::C_Array {
                items: ::std::ptr::null_mut(),
                size: 0,
                capacity: 0,
            }
        }
    }

    pub fn add_buffer(&mut self, val: ffi::C_Buffer) {
        unsafe { ffi::vim_array_add_buffer(val, &mut self.value) }
    }

    pub fn add_window(&mut self, val: ffi::C_Window) {
        unsafe { ffi::vim_array_add_window(val, &mut self.value) }
    }

    pub fn add_tabpage(&mut self, val: ffi::C_Tabpage) {
        unsafe { ffi::vim_array_add_tabpage(val, &mut self.value) }
    }

    pub fn add_nil(&mut self) {
        unsafe { ffi::vim_array_add_nil(&mut self.value) }
    }

    pub fn add_boolean(&mut self, val: ffi::C_Boolean) {
        unsafe { ffi::vim_array_add_boolean(val, &mut self.value) }
    }

    pub fn add_integer(&mut self, val: ffi::C_Integer) {
        unsafe { ffi::vim_array_add_integer(val, &mut self.value) }
    }

    pub fn add_float(&mut self, val: ffi::C_Float) {
        unsafe { ffi::vim_array_add_float(val, &mut self.value) }
    }

    pub fn add_string(&mut self, val: &str) {
        unsafe {
            // we need to copy the string into memory not managed by Rust,
            // so api_free_array can clear it
            let ptr = ffi::malloc(val.len() as u64);
            ::std::ptr::copy_memory(ptr, val.as_ptr() as *const ffi::c_void, val.len());
            let vim_str = ffi::C_String {data: ptr as *const i8, size: val.len() as u64};
            ffi::vim_array_add_string(vim_str, &mut self.value)
        }
    }

    pub fn add_array(&mut self, val: &Array) {
        unsafe { ffi::vim_array_add_array(val.unwrap_value(), &mut self.value) }
    }

    pub fn add_dictionary(&mut self, val: ffi::C_Dictionary) {
        unsafe { ffi::vim_array_add_dictionary(val, &mut self.value) }
    }

    pub fn get(&self, index: u64) -> Option<Object> {
        if index >= self.len() {
            return None;
        }
        unsafe { c_object_to_object(self.value.items.offset(index as int)) }
    }

    pub fn len(&self) -> u64 {
        self.value.size
    }

    #[doc(hidden)]
    pub fn unwrap_value(&self) -> ffi::C_Array {
        self.value
    }

    #[doc(hidden)]
    pub fn wrap_value(c_array: ffi::C_Array) -> Array {
        Array {
            value: c_array
        }
    }
}

impl Drop for Array {
    fn drop(&mut self) {
        unsafe { ffi::api_free_array(self.value) };
    }
}

impl fmt::Show for Array {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Array(").ok();
        for i in range(0, self.len()) {
            if let Some(obj) = self.get(i) {
                write!(f, "{}", obj).ok();
            } else {
                write!(f, "Nil").ok();
            }
            if i + 1 < self.len() {
                write!(f, ", ").ok();
            }
        }
        write!(f, ")")
    }
}

#[test]
fn test_request() {
    let mut args = Array::new();
    args.add_integer(80);
    args.add_integer(24);
    args.add_string("hello");

    let msg = serialize_message(1, "attach_ui", &args);
    let arr = deserialize_message(&msg);
    println!("LENGTH: {}", arr.len());
    for i in range(0, arr.len()) {
        println!("{}", arr.get(i));
    }
}
<commit_msg>Update for latest version of Rust<commit_after>#![crate_name = "neovim"]
#![crate_type = "lib"]
#![crate_type = "rlib"]
#![allow(raw_pointer_derive)]

extern crate libc;

use std::ffi::CString;
use std::fmt;
use std::string::String;
use std::vec::Vec;

#[cfg(target_os="macos")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    #[link(name = "iconv")]
    extern{}
}

#[cfg(target_os="linux")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    extern{}
}

#[cfg(target_os="windows")]
mod platform {
    #[link(name = "nvim")]
    #[link(name = "uv")]
    #[link(name = "msgpack")]
    #[link(name = "curses")]
    extern{}
}

mod ffi;

pub enum Object {
    Buffer(ffi::C_Buffer),
    Window(ffi::C_Window),
    Tabpage(ffi::C_Tabpage),
    Boolean(ffi::C_Boolean),
    Integer(ffi::C_Integer),
    Float(ffi::C_Float),
    String(String),
    Array(ffi::C_Array),
    Dictionary(ffi::C_Dictionary),
}

impl fmt::Show for Object {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Object::Buffer(ref obj) => write!(f, "Buffer({})", obj),
            Object::Window(ref obj) => write!(f, "Window({})", obj),
            Object::Tabpage(ref obj) => write!(f, "Tabpage({})", obj),
            Object::Boolean(ref obj) => write!(f, "Boolean({})", obj),
            Object::Integer(ref obj) => write!(f, "Integer({})", obj),
            Object::Float(ref obj) => write!(f, "Float({})", obj),
            Object::String(ref obj) => write!(f, "String({})", obj),
            Object::Array(ref obj) => {
                write!(f, "Array(").ok();
                for i in range(0, obj.size) {
                    let inner_obj_opt = unsafe { c_object_to_object(obj.items.offset(i as int)) };
                    if let Some(inner_obj) = inner_obj_opt {
                        write!(f, "{}", inner_obj).ok();
                        if i + 1 < obj.size {
                            write!(f, ", ").ok();
                        }
                    } else {
                        write!(f, "Nil ").ok();
                    }
                }
                write!(f, ")")
            },
            Object::Dictionary(ref obj) => write!(f, "Dictionary(Length: {})", obj.size),
        }
    }
}

unsafe fn c_object_to_object(obj: *mut ffi::C_Object) -> Option<Object> {
    match (*obj).object_type {
        ffi::ObjectType::BufferType =>
            Some(Object::Buffer((*(obj as *mut ffi::C_Object_Buffer)).data)),
        ffi::ObjectType::WindowType =>
            Some(Object::Window((*(obj as *mut ffi::C_Object_Window)).data)),
        ffi::ObjectType::TabpageType =>
            Some(Object::Tabpage((*(obj as *mut ffi::C_Object_Tabpage)).data)),
        ffi::ObjectType::NilType =>
            None,
        ffi::ObjectType::BooleanType =>
            Some(Object::Boolean((*(obj as *mut ffi::C_Object_Boolean)).data)),
        ffi::ObjectType::IntegerType =>
            Some(Object::Integer((*(obj as *mut ffi::C_Object_Integer)).data)),
        ffi::ObjectType::FloatType =>
            Some(Object::Float((*(obj as *mut ffi::C_Object_Float)).data)),
        ffi::ObjectType::StringType => {
            let vim_str: ffi::C_String = (*(obj as *mut ffi::C_Object_String)).data;
            let v = Vec::from_raw_buf(vim_str.data as *const u8, vim_str.size as uint);
            Some(Object::String(String::from_utf8_unchecked(v)))
        },
        ffi::ObjectType::ArrayType =>
            Some(Object::Array((*(obj as *mut ffi::C_Object_Array)).data)),
        ffi::ObjectType::DictionaryType =>
            Some(Object::Dictionary((*(obj as *mut ffi::C_Object_Dictionary)).data)),
    }
}

pub fn main_setup(args: Vec<String>) -> i32 {
    let v: Vec<CString> = args.iter().map(|s| CString::from_slice(s.as_bytes())).collect();
    let vp: Vec<*const ffi::c_char> = v.iter().map(|s| s.as_ptr()).collect();
    let p_vp: *const *const ffi::c_char = vp.as_ptr();

    unsafe { ffi::nvim_main_setup(vp.len() as i32, p_vp) }
}

pub fn main_loop() -> i32 {
    unsafe { ffi::nvim_main_loop() }
}

pub fn channel_from_fds(read_fd: i32, write_fd: i32) -> u64 {
    unsafe { ffi::channel_from_fds(read_fd, write_fd) }
}

pub fn serialize_message(id: u64, method: &'static str, args: &Array) -> String {
    unsafe {
        let buf = ffi::vim_msgpack_new();
        let vim_str = ffi::C_String {data: method.as_ptr() as *const i8, size: method.len() as u64};
        ffi::vim_serialize_request(id, vim_str, args.unwrap_value(), buf);
        let v = Vec::from_raw_buf((*buf).data as *const u8, (*buf).size as uint);
        ffi::vim_msgpack_free(buf);
        String::from_utf8_unchecked(v)
    }
}

pub fn deserialize_message(message: &String) -> Array {
    unsafe {
        let mut arr_raw = Array::new().unwrap_value();
        let s = ffi::C_String {
            data: message.as_slice().as_ptr() as *const i8,
            size: message.len() as u64
        };
        ffi::vim_msgpack_parse(s, &mut arr_raw);
        Array::wrap_value(arr_raw)
    }
}

#[derive(Copy)]
pub struct Array {
    value: ffi::C_Array
}

impl Array {
    pub fn new() -> Array {
        Array {
            value: ffi::C_Array {
                items: ::std::ptr::null_mut(),
                size: 0,
                capacity: 0,
            }
        }
    }

    pub fn add_buffer(&mut self, val: ffi::C_Buffer) {
        unsafe { ffi::vim_array_add_buffer(val, &mut self.value) }
    }

    pub fn add_window(&mut self, val: ffi::C_Window) {
        unsafe { ffi::vim_array_add_window(val, &mut self.value) }
    }

    pub fn add_tabpage(&mut self, val: ffi::C_Tabpage) {
        unsafe { ffi::vim_array_add_tabpage(val, &mut self.value) }
    }

    pub fn add_nil(&mut self) {
        unsafe { ffi::vim_array_add_nil(&mut self.value) }
    }

    pub fn add_boolean(&mut self, val: ffi::C_Boolean) {
        unsafe { ffi::vim_array_add_boolean(val, &mut self.value) }
    }

    pub fn add_integer(&mut self, val: ffi::C_Integer) {
        unsafe { ffi::vim_array_add_integer(val, &mut self.value) }
    }

    pub fn add_float(&mut self, val: ffi::C_Float) {
        unsafe { ffi::vim_array_add_float(val, &mut self.value) }
    }

    pub fn add_string(&mut self, val: &str) {
        unsafe {
            // we need to copy the string into memory not managed by Rust,
            // so api_free_array can clear it
            let ptr = ffi::malloc(val.len() as u64);
            ::std::ptr::copy_memory(ptr, val.as_ptr() as *const ffi::c_void, val.len());
            let vim_str = ffi::C_String {data: ptr as *const i8, size: val.len() as u64};
            ffi::vim_array_add_string(vim_str, &mut self.value)
        }
    }

    pub fn add_array(&mut self, val: &Array) {
        unsafe { ffi::vim_array_add_array(val.unwrap_value(), &mut self.value) }
    }

    pub fn add_dictionary(&mut self, val: ffi::C_Dictionary) {
        unsafe { ffi::vim_array_add_dictionary(val, &mut self.value) }
    }

    pub fn get(&self, index: u64) -> Option<Object> {
        if index >= self.len() {
            return None;
        }
        unsafe { c_object_to_object(self.value.items.offset(index as int)) }
    }

    pub fn len(&self) -> u64 {
        self.value.size
    }

    #[doc(hidden)]
    pub fn unwrap_value(&self) -> ffi::C_Array {
        self.value
    }

    #[doc(hidden)]
    pub fn wrap_value(c_array: ffi::C_Array) -> Array {
        Array {
            value: c_array
        }
    }
}

impl Drop for Array {
    fn drop(&mut self) {
        unsafe { ffi::api_free_array(self.value) };
    }
}

impl fmt::Show for Array {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Array(").ok();
        for i in range(0, self.len()) {
            if let Some(obj) = self.get(i) {
                write!(f, "{}", obj).ok();
            } else {
                write!(f, "Nil").ok();
            }
            if i + 1 < self.len() {
                write!(f, ", ").ok();
            }
        }
        write!(f, ")")
    }
}

#[test]
fn test_request() {
    let mut args = Array::new();
    args.add_integer(80);
    args.add_integer(24);
    args.add_string("hello");

    let msg = serialize_message(1, "attach_ui", &args);
    let arr = deserialize_message(&msg);
    println!("LENGTH: {}", arr.len());
    for i in range(0, arr.len()) {
        println!("{}", arr.get(i));
    }
}
<|endoftext|>


<commit_before>#![recursion_limit = "1024"]
#[macro_use]
extern crate log;
extern crate proc_macro;
#[macro_use]
extern crate quote;
extern crate syn;

extern crate walkdir;

use proc_macro::TokenStream;
use syn::*;
use quote::Tokens;

#[cfg(debug_assertions)]
fn generate_assets(ident: &syn::Ident, folder_path: String) -> quote::Tokens {
  quote!{
      use std::fs::File;
      use std::io::Read;
      use std::path::Path;

      impl #ident {
          pub fn get(file_path: &str) -> Option<Vec<u8>> {
              let folder_path = #folder_path;
              let name = &format!("{}{}", folder_path, file_path);
              let path = &Path::new(name);
              let key = String::from(path.to_str().expect("Path does not have a string representation"));
              info!("file: {}", key);
              let mut file = match File::open(path) {
                  Ok(mut file) => file,
                  Err(e) => {
                      error!("file: {} {}", key, e);
                      return None
                  }
              };
              let mut data: Vec<u8> = Vec::new();
              match file.read_to_end(&mut data) {
                  Ok(_) => Some(data),
                  Err(e) =>  {
                      error!("file: {} {}", key, e);
                      return None
                  }
              }
          }
      }
  }
}

#[cfg(not(debug_assertions))]
fn generate_assets(ident: &syn::Ident, folder_path: String) -> quote::Tokens {
  use std::fs::File;
  use std::io::Read;
  use std::path::Path;
  use walkdir::WalkDir;
  let mut values = Vec::<Tokens>::new();
  for entry in WalkDir::new(folder_path.clone())
    .into_iter()
    .filter_map(|e| e.ok())
    .filter(|e| e.file_type().is_file())
  {
    info!("file: {}", entry.path().display());
    let base = &folder_path.clone();
    let key = String::from(
      entry
        .path()
        .to_str()
        .expect("Path does not have a string representation"),
    ).replace(base, "");
    let mut file = File::open(&Path::new(&entry.path())).unwrap_or_else(|e| {
      panic!("could not open file -> {} {}", key, e);
    });
    let mut data: Vec<u8> = Vec::new();
    file.read_to_end(&mut data).unwrap_or_else(|e| {
      panic!("could not read file -> {} {}", key, e);
    });
    let value = quote!{
      #key => Some(vec!#data),
    };
    values.push(value);
  }
  quote!{
      impl #ident {
          pub fn get(file_path: &str) -> Option<Vec<u8>> {
              match file_path {
                  #(#values)*
                  _ => None,
              }
          }
      }
  }
}

fn help() {
  panic!("#[derive(RustEmbed)] should contain one attribute like this #[folder(\"examples/public/\")]");
}

fn impl_rust_embed(ast: &syn::DeriveInput) -> Tokens {
  match ast.body {
    Body::Enum(_) => help(),
    Body::Struct(ref data) => match data {
      &VariantData::Struct(_) => help(),
      _ => {}
    },
  };
  let ident = &ast.ident;
  if ast.attrs.len() == 0 || ast.attrs.len() > 1 {
    help();
  }
  let value = &ast.attrs[0].value;
  let items = match value {
    &MetaItem::List(ref attr_name, ref items) => {
      if attr_name == "folder" {
        items
      } else {
        panic!("#[derive(RustEmbed)] attribute name must be folder");
      }
    }
    _ => {
      panic!("#[derive(RustEmbed)] attribute name must be folder");
    }
  };
  let item = &items[0];
  let lit = match item {
    &NestedMetaItem::Literal(ref l) => l,
    _ => {
      panic!("Hello");
    }
  };
  let folder_path = match lit {
    &Lit::Str(ref val, _) => val.clone(),
    _ => {
      panic!("#[derive(RustEmbed)] attribute value must be a string literal");
    }
  };
  info!("folder: {}", folder_path);
  generate_assets(ident, folder_path)
}

#[proc_macro_derive(RustEmbed, attributes(folder))]
pub fn derive_input_object(input: TokenStream) -> TokenStream {
  let s = input.to_string();
  let ast = syn::parse_derive_input(&s).unwrap();
  let gen = impl_rust_embed(&ast);
  gen.parse().unwrap()
}
<commit_msg>add proper log when compiling files in release<commit_after>#![recursion_limit = "1024"]
#[macro_use]
extern crate log;
extern crate proc_macro;
#[macro_use]
extern crate quote;
extern crate syn;

extern crate walkdir;

use proc_macro::TokenStream;
use syn::*;
use quote::Tokens;

#[cfg(debug_assertions)]
fn generate_assets(ident: &syn::Ident, folder_path: String) -> quote::Tokens {
  quote!{
      use std::fs::File;
      use std::io::Read;
      use std::path::Path;

      impl #ident {
          pub fn get(file_path: &str) -> Option<Vec<u8>> {
              let folder_path = #folder_path;
              let name = &format!("{}{}", folder_path, file_path);
              let path = &Path::new(name);
              let key = String::from(path.to_str().expect("Path does not have a string representation"));
              info!("file: {}", key);
              let mut file = match File::open(path) {
                  Ok(mut file) => file,
                  Err(e) => {
                      error!("file: {} {}", key, e);
                      return None
                  }
              };
              let mut data: Vec<u8> = Vec::new();
              match file.read_to_end(&mut data) {
                  Ok(_) => Some(data),
                  Err(e) =>  {
                      error!("file: {} {}", key, e);
                      return None
                  }
              }
          }
      }
  }
}

#[cfg(not(debug_assertions))]
fn generate_assets(ident: &syn::Ident, folder_path: String) -> quote::Tokens {
  use std::fs::File;
  use std::io::Read;
  use std::path::Path;
  use walkdir::WalkDir;
  let mut values = Vec::<Tokens>::new();
  for entry in WalkDir::new(folder_path.clone())
    .into_iter()
    .filter_map(|e| e.ok())
    .filter(|e| e.file_type().is_file())
  {
    println!("   \x1b[92mCompiling\x1b[0m {}", entry.path().display());
    let base = &folder_path.clone();
    let key = String::from(
      entry
        .path()
        .to_str()
        .expect("Path does not have a string representation"),
    ).replace(base, "");
    let mut file = File::open(&Path::new(&entry.path())).unwrap_or_else(|e| {
      panic!("could not open file -> {} {}", key, e);
    });
    let mut data: Vec<u8> = Vec::new();
    file.read_to_end(&mut data).unwrap_or_else(|e| {
      panic!("could not read file -> {} {}", key, e);
    });
    let value = quote!{
      #key => Some(vec!#data),
    };
    values.push(value);
  }
  quote!{
      impl #ident {
          pub fn get(file_path: &str) -> Option<Vec<u8>> {
              match file_path {
                  #(#values)*
                  _ => None,
              }
          }
      }
  }
}

fn help() {
  panic!("#[derive(RustEmbed)] should contain one attribute like this #[folder(\"examples/public/\")]");
}

fn impl_rust_embed(ast: &syn::DeriveInput) -> Tokens {
  match ast.body {
    Body::Enum(_) => help(),
    Body::Struct(ref data) => match data {
      &VariantData::Struct(_) => help(),
      _ => {}
    },
  };
  let ident = &ast.ident;
  if ast.attrs.len() == 0 || ast.attrs.len() > 1 {
    help();
  }
  let value = &ast.attrs[0].value;
  let items = match value {
    &MetaItem::List(ref attr_name, ref items) => {
      if attr_name == "folder" {
        items
      } else {
        panic!("#[derive(RustEmbed)] attribute name must be folder");
      }
    }
    _ => {
      panic!("#[derive(RustEmbed)] attribute name must be folder");
    }
  };
  let item = &items[0];
  let lit = match item {
    &NestedMetaItem::Literal(ref l) => l,
    _ => {
      panic!("Hello");
    }
  };
  let folder_path = match lit {
    &Lit::Str(ref val, _) => val.clone(),
    _ => {
      panic!("#[derive(RustEmbed)] attribute value must be a string literal");
    }
  };
  info!("folder: {}", folder_path);
  generate_assets(ident, folder_path)
}

#[proc_macro_derive(RustEmbed, attributes(folder))]
pub fn derive_input_object(input: TokenStream) -> TokenStream {
  let s = input.to_string();
  let ast = syn::parse_derive_input(&s).unwrap();
  let gen = impl_rust_embed(&ast);
  gen.parse().unwrap()
}
<|endoftext|>


<commit_before>#![feature(slicing_syntax)]
extern crate lzf;
use std::str;
use lzf::decompress;
use std::io::MemReader;

mod version {
    pub const SUPPORTED_MINIMUM : u32 = 1;
    pub const SUPPORTED_MAXIMUM : u32 = 6;
}

mod constants {
    pub const RDB_6BITLEN : u8 = 0;
    pub const RDB_14BITLEN : u8 = 1;
    pub const RDB_32BITLEN : u8 = 2;
    pub const RDB_ENCVAL : u8 = 3;
    // REDIS
    pub const RDB_MAGIC : &'static str = "REDIS";
}

mod op_codes {
    pub const EXPIRETIME_MS : u8 = 252;
    pub const EXPIRETIME : u8 = 253;
    pub const SELECTDB   : u8 = 254;
    pub const EOF : u8 = 255;
}

mod types {
    pub const STRING : u8 = 0;
    pub const LIST : u8 = 1;
    pub const SET : u8 = 2;
    pub const ZSET : u8 = 3;
    pub const HASH : u8 = 4;
    pub const HASH_ZIPMAP : u8 = 9;
    pub const LIST_ZIPLIST : u8 = 10;
    pub const SET_INTSET : u8 = 11;
    pub const ZSET_ZIPLIST : u8 = 12;
    pub const HASH_ZIPLIST : u8 = 13;
}

mod encoding {
    pub const INT8 : u32 = 0;
    pub const INT16 : u32 = 1;
    pub const INT32 : u32 = 2;
    pub const LZF : u32 = 3;
}

#[derive(Show)]
pub enum DataType {
    String(Vec<u8>),
    Number(i64),
    ListOfTypes(Vec<DataType>),
    HashOfTypes(Vec<DataType>),
    SortedSetOfTypes(Vec<DataType>),
    Intset(Vec<i64>),
    List(Vec<Vec<u8>>),
    Set(Vec<Vec<u8>>),
    SortedSet(Vec<(f64,Vec<u8>)>),
    Hash(Vec<Vec<u8>>),
    Unknown
}

//pub fn type_mapping(in_type: u8) -> DataType {
    //match in_type {
        //0 => DataType::String,
        //1 | 10 => DataType::List,
        //2 | 11 => DataType::Set,
        //3 | 12 => DataType::SortedSet,
        //4 | 9 | 13 => DataType::Hash,
        //_ => DataType::Unknown
    //}
//}

pub fn parse<T: Reader>(input: &mut T) {

    assert!(verify_magic(input));
    assert!(verify_version(input));

    loop {
        let next_op = input.read_byte().unwrap();
        //println!("next_op: {:x}", next_op);

        match next_op {
            op_codes::SELECTDB => {
                let db = input.read_byte().unwrap();
                println!("SELECTDB: {}", db);
            },
            op_codes::EOF => {
                println!("EOF");
                break ;
            },
            op_codes::EXPIRETIME_MS => {
                let expiretime_ms = input.read_le_u64().unwrap();
                println!("EXPIRETIME_MS: {}", expiretime_ms);
            },
            op_codes::EXPIRETIME => {
                let expiretime = input.read_be_u32().unwrap();
                println!("EXPIRETIME: {}", expiretime);
            }
            _ => {
                let key = read_blob(input);
                println!("Key: {}",
                         str::from_utf8(key.as_slice()).unwrap());
                println!("Val: {}", read_type(next_op, input));
            }
        }

    }

    return;

}

fn read_linked_list<T: Reader>(input: &mut T) -> Vec<Vec<u8>> {
    let mut len = read_length(input);
    let mut list = vec![];

    while len > 0 {
        let blob = read_blob(input);
        list.push(blob);
        len -= 1;
    }

    list
}

fn read_sorted_set<T: Reader>(input: &mut T) -> Vec<(f64,Vec<u8>)> {
    let mut set = vec![];
    let mut set_items = read_length(input);

    while set_items > 0 {
        let val = read_blob(input);
        let score_length = input.read_byte().unwrap();
        let score = input.read_exact(score_length as uint).unwrap();
        let score = unsafe{str::from_utf8_unchecked(score[])}.
            parse::<f64>().unwrap();

        set.push((score, val));

        set_items -= 1;
    }

    set
}

fn read_hash<T: Reader>(input: &mut T) -> Vec<Vec<u8>> {
    let mut hash = vec![];
    let mut hash_items = read_length(input);
    hash_items = 2*hash_items;

    while hash_items > 0 {
        let val = read_blob(input);
        hash.push(val);

        hash_items -= 1;
    }

    hash
}

fn read_list_ziplist<T: Reader>(input: &mut T) -> Vec<DataType> {
    let mut list = vec![];

    let ziplist = read_blob(input);

    let mut reader = MemReader::new(ziplist);

    let _zlbytes = reader.read_le_u32().unwrap();
    let _zltail = reader.read_le_u32().unwrap();
    let zllen = reader.read_le_u16().unwrap();

    for _ in range(0, zllen) {
        // 1. 1 or 5 bytes length of previous entry
        match reader.read_byte().unwrap() {
            254 => {
                let _ = reader.read_exact(4).unwrap();
            },
            _ => {}
        }

        let mut length : u64;
        let mut number_value : i64;

        // 2. Read flag or number value
        let flag = reader.read_byte().unwrap();

        match (flag & 0xC0) >> 6 {
            0 => { length = (flag & 0x3F) as u64 },
            1 => {
                let next_byte = reader.read_byte().unwrap();
                length = (((flag & 0x3F) as u64) << 8) | next_byte as u64;
            },
            2 => {
                length = input.read_le_u32().unwrap() as u64;
            },
            _ => {
                match (flag & 0xF0) >> 4 {
                    0xC => {
                        number_value = reader.read_le_i16().unwrap() as i64 },
                    0xD => {
                        number_value = reader.read_le_i32().unwrap() as i64 },
                    0xE => {
                        number_value = reader.read_le_i64().unwrap() as i64 },
                    0xF => {
                        match flag & 0xF {
                            0 => {
                                let bytes = reader.read_exact(3).unwrap();
                                number_value = ((bytes[0] as i64) << 16) ^
                                    ((bytes[1] as i64) << 8) ^
                                    (bytes[2] as i64);
                            },
                            0xE => {
                                number_value = reader.read_byte().unwrap() as i64 },
                            _ => { number_value = (flag & 0xF) as i64 - 1; }
                        }
                    },
                    _ => {
                        println!("Flag not handled: {}", flag);
                        continue;
                    }

                }

                list.push(DataType::Number(number_value));
                continue;
            }
        }

        // 3. Read value
        let rawval = reader.read_exact(length as uint).unwrap();
        list.push(DataType::String(rawval));
    }

    list
}
fn read_set_intset<T: Reader>(input: &mut T) -> Vec<i64> {
    let mut set = vec![];

    let intset = read_blob(input);

    let mut reader = MemReader::new(intset);
    let byte_size = reader.read_le_u32().unwrap();
    let intset_length = reader.read_le_u32().unwrap();

    for _ in range(0, intset_length) {
        let val = match byte_size {
            2 => reader.read_le_i16().unwrap() as i64,
            4 => reader.read_le_i32().unwrap() as i64,
            8 => reader.read_le_i64().unwrap(),
            _ => panic!("unhandled byte size in intset: {}", byte_size)
        };

        set.push(val);
    }

    set
}

fn read_type<T: Reader>(value_type: u8, input: &mut T) -> DataType {
    match value_type {
        types::STRING => {
            DataType::String(read_blob(input))
        },
        types::LIST => {
            DataType::List(read_linked_list(input))
        },
        types::SET => {
            DataType::Set(read_linked_list(input))
        },
        types::ZSET => {
            DataType::SortedSet(read_sorted_set(input))
        },
        types::HASH => {
            DataType::Hash(read_hash(input))
        },
        types::HASH_ZIPMAP => {
            println!("Value Type not implemented: {}", "HASH_ZIPMAP");
            //DataType::Hash(read_hash_zipmap(input))
            DataType::Hash(vec![])
        },
        types::LIST_ZIPLIST => {
            DataType::ListOfTypes(read_list_ziplist(input))
        },
        types::SET_INTSET => {
            DataType::Intset(read_set_intset(input))
        },
        types::ZSET_ZIPLIST => {
            DataType::SortedSetOfTypes(read_list_ziplist(input))
        },
        types::HASH_ZIPLIST => {
            DataType::ListOfTypes(read_list_ziplist(input))
        },
        _ => { panic!("Value Type not implemented: {}", value_type) }
    }
}

fn verify_magic<T: Reader>(input: &mut T) -> bool {
    let magic = input.read_exact(5).unwrap();

    // Meeeeeh.
    magic[0] == constants::RDB_MAGIC.as_bytes()[0] &&
        magic[1] == constants::RDB_MAGIC.as_bytes()[1] &&
        magic[2] == constants::RDB_MAGIC.as_bytes()[2] &&
        magic[3] == constants::RDB_MAGIC.as_bytes()[3] &&
        magic[4] == constants::RDB_MAGIC.as_bytes()[4]
}

fn verify_version<T: Reader>(input: &mut T) -> bool {
    let version = input.read_exact(4).unwrap();

    let version = (version[0]-48) as u32 * 1000 +
        (version[1]-48) as u32 * 100 +
        (version[2]-48) as u32 * 10 +
        (version[3]-48) as u32;

    version >= version::SUPPORTED_MINIMUM &&
        version <= version::SUPPORTED_MAXIMUM
}

#[test]
fn test_verify_magic() {
    assert_eq!(
        true,
        verify_magic(&mut MemReader::new(vec!(0x52, 0x45, 0x44, 0x49, 0x53)))
    );

    assert_eq!(
        false,
        verify_magic(&mut MemReader::new(vec!(0x52, 0x0, 0x0, 0x0, 0x0)))
    );
}

#[test]
fn test_verify_version() {
    assert_eq!(
        true,
        verify_version(&mut MemReader::new(vec!(0x30, 0x30, 0x30, 0x33)))
    );

    assert_eq!(
        false,
        verify_version(&mut MemReader::new(vec!(0x30, 0x30, 0x30, 0x3a)))
    );
}

#[derive(Show,PartialEq)]
enum LengthEncoded {
    LE(u32, bool)
    //Encoded(u32),
    //Plain(u32)
}

fn read_length_with_encoding<T: Reader>(input: &mut T) -> LengthEncoded {
    let mut length;
    let mut is_encoded = false;

    let enc_type = input.read_byte().unwrap();

    match (enc_type & 0xC0) >> 6 {
        constants::RDB_ENCVAL => {
            is_encoded = true;
            length = (enc_type & 0x3F) as u32;
        },
        constants::RDB_6BITLEN => {
            length = (enc_type & 0x3F) as u32;
        },
        constants::RDB_14BITLEN => {
            let next_byte = input.read_byte().unwrap();
            length = (((enc_type & 0x3F) as u32) <<8) | next_byte as u32;
        },
        _ => {
            println!("foo");
            length = input.read_le_u32().unwrap();
        }
    }

    LengthEncoded::LE(length, is_encoded)
}

fn read_length<T: Reader>(input: &mut T) -> u32 {
    let LengthEncoded::LE(length, _) = read_length_with_encoding(input);
    length
}

fn read_blob<T: Reader>(input: &mut T) -> Vec<u8> {
    let LengthEncoded::LE(length, is_encoded) = read_length_with_encoding(input);

    if is_encoded {
        match length {
            encoding::INT8 => { input.read_exact(1).unwrap() },
            encoding::INT16 => { input.read_exact(2).unwrap() },
            encoding::INT32 => { input.read_exact(4).unwrap() },
            encoding::LZF => {
                let compressed_length = read_length(input);
                let real_length = read_length(input);
                let data = input.read_exact(compressed_length as uint).unwrap();
                lzf::decompress(data.as_slice(), real_length as uint).unwrap()
            },
            _ => { panic!("Unknown encoding: {}", length) }
        }
    } else {
        input.read_exact(length as uint).unwrap()
    }
}

#[test]
fn test_read_length() {
    assert_eq!(
        LengthEncoded::LE(0, false),
        read_length_with_encoding(&mut MemReader::new(vec!(0x0)))
    );

    assert_eq!(
        LengthEncoded::LE(16383, false),
        read_length_with_encoding(&mut MemReader::new(vec!(0x7f, 0xff)))
    );

    assert_eq!(
        LengthEncoded::LE(4294967295, false),
        read_length_with_encoding(&mut MemReader::new(
                vec!(0x80, 0xff, 0xff, 0xff, 0xff)))
    );

    assert_eq!(
        LengthEncoded::LE(0, true),
        read_length_with_encoding(&mut MemReader::new(
                vec!(0xC0)))
    );
}

#[test]
fn test_read_blob() {
    assert_eq!(
        vec!(0x61, 0x62, 0x63, 0x64),
        read_blob(&mut MemReader::new(vec!(
                    4, 0x61, 0x62, 0x63, 0x64))));
}
<commit_msg>Encode integers to strings<commit_after>#![feature(slicing_syntax)]
extern crate lzf;
use std::str;
use lzf::decompress;
use std::io::MemReader;
use std::io;

mod version {
    pub const SUPPORTED_MINIMUM : u32 = 1;
    pub const SUPPORTED_MAXIMUM : u32 = 6;
}

mod constants {
    pub const RDB_6BITLEN : u8 = 0;
    pub const RDB_14BITLEN : u8 = 1;
    pub const RDB_32BITLEN : u8 = 2;
    pub const RDB_ENCVAL : u8 = 3;
    // REDIS
    pub const RDB_MAGIC : &'static str = "REDIS";
}

mod op_codes {
    pub const EXPIRETIME_MS : u8 = 252;
    pub const EXPIRETIME : u8 = 253;
    pub const SELECTDB   : u8 = 254;
    pub const EOF : u8 = 255;
}

mod types {
    pub const STRING : u8 = 0;
    pub const LIST : u8 = 1;
    pub const SET : u8 = 2;
    pub const ZSET : u8 = 3;
    pub const HASH : u8 = 4;
    pub const HASH_ZIPMAP : u8 = 9;
    pub const LIST_ZIPLIST : u8 = 10;
    pub const SET_INTSET : u8 = 11;
    pub const ZSET_ZIPLIST : u8 = 12;
    pub const HASH_ZIPLIST : u8 = 13;
}

mod encoding {
    pub const INT8 : u32 = 0;
    pub const INT16 : u32 = 1;
    pub const INT32 : u32 = 2;
    pub const LZF : u32 = 3;
}

#[derive(Show)]
pub enum DataType {
    String(Vec<u8>),
    Number(i64),
    ListOfTypes(Vec<DataType>),
    HashOfTypes(Vec<DataType>),
    SortedSetOfTypes(Vec<DataType>),
    Intset(Vec<i64>),
    List(Vec<Vec<u8>>),
    Set(Vec<Vec<u8>>),
    SortedSet(Vec<(f64,Vec<u8>)>),
    Hash(Vec<Vec<u8>>),
    Unknown
}

//pub fn type_mapping(in_type: u8) -> DataType {
    //match in_type {
        //0 => DataType::String,
        //1 | 10 => DataType::List,
        //2 | 11 => DataType::Set,
        //3 | 12 => DataType::SortedSet,
        //4 | 9 | 13 => DataType::Hash,
        //_ => DataType::Unknown
    //}
//}

pub fn parse<T: Reader>(input: &mut T) {

    assert!(verify_magic(input));
    assert!(verify_version(input));

    let mut out = io::stdout();
    loop {
        let next_op = input.read_byte().unwrap();
        //println!("next_op: {:x}", next_op);

        match next_op {
            op_codes::SELECTDB => {
                let db = input.read_byte().unwrap();
                println!("SELECTDB: {}", db);
            },
            op_codes::EOF => {
                println!("EOF");
                break ;
            },
            op_codes::EXPIRETIME_MS => {
                let expiretime_ms = input.read_le_u64().unwrap();
                println!("EXPIRETIME_MS: {}", expiretime_ms);
            },
            op_codes::EXPIRETIME => {
                let expiretime = input.read_be_u32().unwrap();
                println!("EXPIRETIME: {}", expiretime);
            }
            _ => {
                let key = read_blob(input);
                let _ = out.write(key[]);
                let _ = out.write_str(": ");

                match read_type(next_op, input) {
                    DataType::String(t) => {
                        let _ = out.write(t[]);
                    },
                    _ => {  }
                }
                let _ = out.write_str("\n");
            }
        }

    }

    return;

}

fn read_linked_list<T: Reader>(input: &mut T) -> Vec<Vec<u8>> {
    let mut len = read_length(input);
    let mut list = vec![];

    while len > 0 {
        let blob = read_blob(input);
        list.push(blob);
        len -= 1;
    }

    list
}

fn read_sorted_set<T: Reader>(input: &mut T) -> Vec<(f64,Vec<u8>)> {
    let mut set = vec![];
    let mut set_items = read_length(input);

    while set_items > 0 {
        let val = read_blob(input);
        let score_length = input.read_byte().unwrap();
        let score = input.read_exact(score_length as uint).unwrap();
        let score = unsafe{str::from_utf8_unchecked(score[])}.
            parse::<f64>().unwrap();

        set.push((score, val));

        set_items -= 1;
    }

    set
}

fn read_hash<T: Reader>(input: &mut T) -> Vec<Vec<u8>> {
    let mut hash = vec![];
    let mut hash_items = read_length(input);
    hash_items = 2*hash_items;

    while hash_items > 0 {
        let val = read_blob(input);
        hash.push(val);

        hash_items -= 1;
    }

    hash
}

fn read_list_ziplist<T: Reader>(input: &mut T) -> Vec<DataType> {
    let mut list = vec![];

    let ziplist = read_blob(input);

    let mut reader = MemReader::new(ziplist);

    let _zlbytes = reader.read_le_u32().unwrap();
    let _zltail = reader.read_le_u32().unwrap();
    let zllen = reader.read_le_u16().unwrap();

    for _ in range(0, zllen) {
        // 1. 1 or 5 bytes length of previous entry
        match reader.read_byte().unwrap() {
            254 => {
                let _ = reader.read_exact(4).unwrap();
            },
            _ => {}
        }

        let mut length : u64;
        let mut number_value : i64;

        // 2. Read flag or number value
        let flag = reader.read_byte().unwrap();

        match (flag & 0xC0) >> 6 {
            0 => { length = (flag & 0x3F) as u64 },
            1 => {
                let next_byte = reader.read_byte().unwrap();
                length = (((flag & 0x3F) as u64) << 8) | next_byte as u64;
            },
            2 => {
                length = input.read_le_u32().unwrap() as u64;
            },
            _ => {
                match (flag & 0xF0) >> 4 {
                    0xC => {
                        number_value = reader.read_le_i16().unwrap() as i64 },
                    0xD => {
                        number_value = reader.read_le_i32().unwrap() as i64 },
                    0xE => {
                        number_value = reader.read_le_i64().unwrap() as i64 },
                    0xF => {
                        match flag & 0xF {
                            0 => {
                                let bytes = reader.read_exact(3).unwrap();
                                number_value = ((bytes[0] as i64) << 16) ^
                                    ((bytes[1] as i64) << 8) ^
                                    (bytes[2] as i64);
                            },
                            0xE => {
                                number_value = reader.read_byte().unwrap() as i64 },
                            _ => { number_value = (flag & 0xF) as i64 - 1; }
                        }
                    },
                    _ => {
                        println!("Flag not handled: {}", flag);
                        continue;
                    }

                }

                list.push(DataType::Number(number_value));
                continue;
            }
        }

        // 3. Read value
        let rawval = reader.read_exact(length as uint).unwrap();
        list.push(DataType::String(rawval));
    }

    list
}
fn read_set_intset<T: Reader>(input: &mut T) -> Vec<i64> {
    let mut set = vec![];

    let intset = read_blob(input);

    let mut reader = MemReader::new(intset);
    let byte_size = reader.read_le_u32().unwrap();
    let intset_length = reader.read_le_u32().unwrap();

    for _ in range(0, intset_length) {
        let val = match byte_size {
            2 => reader.read_le_i16().unwrap() as i64,
            4 => reader.read_le_i32().unwrap() as i64,
            8 => reader.read_le_i64().unwrap(),
            _ => panic!("unhandled byte size in intset: {}", byte_size)
        };

        set.push(val);
    }

    set
}

fn read_type<T: Reader>(value_type: u8, input: &mut T) -> DataType {
    match value_type {
        types::STRING => {
            DataType::String(read_blob(input))
        },
        types::LIST => {
            DataType::List(read_linked_list(input))
        },
        types::SET => {
            DataType::Set(read_linked_list(input))
        },
        types::ZSET => {
            DataType::SortedSet(read_sorted_set(input))
        },
        types::HASH => {
            DataType::Hash(read_hash(input))
        },
        types::HASH_ZIPMAP => {
            println!("Value Type not implemented: {}", "HASH_ZIPMAP");
            //DataType::Hash(read_hash_zipmap(input))
            DataType::Hash(vec![])
        },
        types::LIST_ZIPLIST => {
            DataType::ListOfTypes(read_list_ziplist(input))
        },
        types::SET_INTSET => {
            DataType::Intset(read_set_intset(input))
        },
        types::ZSET_ZIPLIST => {
            DataType::SortedSetOfTypes(read_list_ziplist(input))
        },
        types::HASH_ZIPLIST => {
            DataType::ListOfTypes(read_list_ziplist(input))
        },
        _ => { panic!("Value Type not implemented: {}", value_type) }
    }
}

fn verify_magic<T: Reader>(input: &mut T) -> bool {
    let magic = input.read_exact(5).unwrap();

    // Meeeeeh.
    magic[0] == constants::RDB_MAGIC.as_bytes()[0] &&
        magic[1] == constants::RDB_MAGIC.as_bytes()[1] &&
        magic[2] == constants::RDB_MAGIC.as_bytes()[2] &&
        magic[3] == constants::RDB_MAGIC.as_bytes()[3] &&
        magic[4] == constants::RDB_MAGIC.as_bytes()[4]
}

fn verify_version<T: Reader>(input: &mut T) -> bool {
    let version = input.read_exact(4).unwrap();

    let version = (version[0]-48) as u32 * 1000 +
        (version[1]-48) as u32 * 100 +
        (version[2]-48) as u32 * 10 +
        (version[3]-48) as u32;

    version >= version::SUPPORTED_MINIMUM &&
        version <= version::SUPPORTED_MAXIMUM
}

#[test]
fn test_verify_magic() {
    assert_eq!(
        true,
        verify_magic(&mut MemReader::new(vec!(0x52, 0x45, 0x44, 0x49, 0x53)))
    );

    assert_eq!(
        false,
        verify_magic(&mut MemReader::new(vec!(0x52, 0x0, 0x0, 0x0, 0x0)))
    );
}

#[test]
fn test_verify_version() {
    assert_eq!(
        true,
        verify_version(&mut MemReader::new(vec!(0x30, 0x30, 0x30, 0x33)))
    );

    assert_eq!(
        false,
        verify_version(&mut MemReader::new(vec!(0x30, 0x30, 0x30, 0x3a)))
    );
}

#[derive(Show,PartialEq)]
enum LengthEncoded {
    LE(u32, bool)
    //Encoded(u32),
    //Plain(u32)
}

fn read_length_with_encoding<T: Reader>(input: &mut T) -> LengthEncoded {
    let mut length;
    let mut is_encoded = false;

    let enc_type = input.read_byte().unwrap();

    match (enc_type & 0xC0) >> 6 {
        constants::RDB_ENCVAL => {
            is_encoded = true;
            length = (enc_type & 0x3F) as u32;
        },
        constants::RDB_6BITLEN => {
            length = (enc_type & 0x3F) as u32;
        },
        constants::RDB_14BITLEN => {
            let next_byte = input.read_byte().unwrap();
            length = (((enc_type & 0x3F) as u32) <<8) | next_byte as u32;
        },
        _ => {
            length = input.read_le_u32().unwrap();
        }
    }

    LengthEncoded::LE(length, is_encoded)
}

fn read_length<T: Reader>(input: &mut T) -> u32 {
    let LengthEncoded::LE(length, _) = read_length_with_encoding(input);
    length
}

fn int_to_vec(number: i32) -> Vec<u8> {
    let number = number.to_string();
    let mut result = Vec::with_capacity(number.len());
    for &c in number.as_bytes().iter() {
        result.push(c);
    }
    result
}

fn read_blob<T: Reader>(input: &mut T) -> Vec<u8> {
    let LengthEncoded::LE(length, is_encoded) = read_length_with_encoding(input);

    if is_encoded {
        match length {
            encoding::INT8 => { int_to_vec(input.read_i8().unwrap() as i32) },
            encoding::INT16 => { int_to_vec(input.read_le_i16().unwrap() as i32) },
            encoding::INT32 => { int_to_vec(input.read_le_i32().unwrap() as i32) },
            encoding::LZF => {
                let compressed_length = read_length(input);
                let real_length = read_length(input);
                let data = input.read_exact(compressed_length as uint).unwrap();
                lzf::decompress(data.as_slice(), real_length as uint).unwrap()
            },
            _ => { panic!("Unknown encoding: {}", length) }
        }
    } else {
        input.read_exact(length as uint).unwrap()
    }
}

#[test]
fn test_read_length() {
    assert_eq!(
        LengthEncoded::LE(0, false),
        read_length_with_encoding(&mut MemReader::new(vec!(0x0)))
    );

    assert_eq!(
        LengthEncoded::LE(16383, false),
        read_length_with_encoding(&mut MemReader::new(vec!(0x7f, 0xff)))
    );

    assert_eq!(
        LengthEncoded::LE(4294967295, false),
        read_length_with_encoding(&mut MemReader::new(
                vec!(0x80, 0xff, 0xff, 0xff, 0xff)))
    );

    assert_eq!(
        LengthEncoded::LE(0, true),
        read_length_with_encoding(&mut MemReader::new(
                vec!(0xC0)))
    );
}

#[test]
fn test_read_blob() {
    assert_eq!(
        vec!(0x61, 0x62, 0x63, 0x64),
        read_blob(&mut MemReader::new(vec!(
                    4, 0x61, 0x62, 0x63, 0x64))));
}
<|endoftext|>


<commit_before>#![feature(libc)]
// #![feature(phase)]

// #![feature(plugin)]
// #![plugin(bindgen_plugin)]

// #![feature(phase)]
// #[phase(plugin)] extern crate bindgen;

extern crate libc;

pub mod mbed;

#[test]
fn it_works() {
    mbed::ssl::SSLContext::new();
    mbed::entropy::EntropyContext::new();
    mbed::ctr_drbg::CtrDrbgContext::new();
}
<commit_msg>Fix test<commit_after>#![feature(libc)]
// #![feature(phase)]

// #![feature(plugin)]
// #![plugin(bindgen_plugin)]

// #![feature(phase)]
// #[phase(plugin)] extern crate bindgen;

extern crate libc;

pub mod mbed;

#[test]
fn it_works() {
    mbed::ssl::SSLContext::new();
    mbed::entropy::EntropyContext::new();
}
<|endoftext|>


<commit_before>#![feature(catch_panic, fnbox)]

//! Work-stealing job-queue implementation.
//! Inspired by the Molecular Matters blog posts.

extern crate crossbeam;
extern crate rand;

use crossbeam::mem::CachePadded;
use crossbeam::sync::TreiberStack;

use std::any::Any;
use std::boxed::FnBox;
use std::cell::RefCell;
use std::collections::VecDeque;
use std::sync::{Arc, Mutex, RwLock};
use std::sync::mpsc::{channel, Sender, Receiver, TryRecvError};
use std::thread;

use ::rand::thread_rng;
use ::rand::distributions::{IndependentSample, Range};

/// Messages that can be sent to workers.
enum WorkerMessage {
    Stop,
}

type Job = Box<FnBox(&Worker) + 'static + Send>;

/// A job's id in the pool.
pub struct JobId {
    idx: usize,
}

struct PoolEntry {
    job: Job,
    sender: Sender<JobExitStatus>,
}

enum JobExitStatus {
    Success,
    Panic(Box<Any>),
}

/// A mostly lock-free pool allocator of jobs.
/// It only locks when doubling the size of the vector.
/// Eventually, I may implement a lock-free vector based off of
/// Stroustroup-et-al's paper on the subject, but for now, I've
/// decided to "keep it simple, stupid".
/// However, this should be mostly wait free unless *many* jobs
/// are being submitted.
struct JobPool {
    // we ensure with the lock-free stack of free indices that
    // an index is only used by one thread at a time.
    // mutable, disjoint borrows are therefore acceptable
    // as long as they occur within the umbrella of a read
    // lock and will not be interfered with by doubling of the
    // vector.
    jobs: RwLock<Vec<RefCell<Option<CachePadded<PoolEntry>>>>>,
    unused: TreiberStack<usize>,
}

impl JobPool {
    /// Initialize this pool with the given capacity.
    fn with_capacity(size: usize) -> Self {
        let mut v = Vec::with_capacity(size);
        let unused = TreiberStack::new();

        for i in (0..size).rev() {
            v.push(RefCell::new(None));

            unused.push(i);
        }

        JobPool {
            jobs: RwLock::new(v),
            unused: unused,
        }
    }

    /// Submit a job to be stored in the pool.
    fn submit(&self, job: Job) -> (JobId, Receiver<JobExitStatus>) {
        let next = if let Some(next) = self.unused.pop() {
            next
        } else {
            self.double();
            self.unused.pop().unwrap()
        };

        let jobs = match self.jobs.read() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        let (tx, rx) = channel();

        let entry = &jobs[next];
        *entry.borrow_mut() = Some(CachePadded::new(PoolEntry {
            job: job,
            sender: tx,
        }));

        (JobId { idx: next, }, rx)
    }

    // double the vector size
    fn double(&self) {
        let mut jobs = match self.jobs.write() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        let old_len = jobs.len();
        let new_len = old_len * 2;

        jobs.reserve(new_len);

        for i in (old_len..new_len).rev() {
            jobs.push(RefCell::new(None));

            self.unused.push(i);
        }
    }

    /// Run the job given by id, passing the worker
    /// as a parameter.
    fn run(&self, id: JobId, worker: &Worker) {
        use std::mem;
        use std::ptr;

        let jobs = match self.jobs.read() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        // take the job out before pushing the index back onto
        // the unused stack.
        let entry = jobs[id.idx].borrow_mut().take().unwrap();

        // CachePadded doesn't run destructors, so this is ok.
        let PoolEntry { job, sender } = unsafe { ptr::read(&*entry) } ;

        // is there a better way to do this than to box again?
        let unbounded_fn: Box<FnBox() + Send> = Box::new(move || job.call_box((worker,)));
        let bounded_fn: Box<FnBox() + 'static + Send> = unsafe { mem::transmute(unbounded_fn) };

        let exit_status = match thread::catch_panic(|| bounded_fn()) {
            Ok(_) => { JobExitStatus::Success }
            Err(e) => { JobExitStatus::Panic(e) }
        };

        // We don't really care if they receive it or not.
        let _result = sender.send(exit_status);
        self.unused.push(id.idx);
    }
}

unsafe impl Send for JobPool {}
unsafe impl Sync for JobPool {}

/// A thread-safe double-ended queue for jobs.
/// Right now, it's just a dumb locking wrapper of VecDeque.
/// It stores indices of jobs in the global job pool.
struct JobQueue {
    inner: Mutex<VecDeque<JobId>>,
}

impl JobQueue {
    fn new() -> Self {
        JobQueue {
            inner: Mutex::new(VecDeque::new()),
        }
    }

    // pop a job from the private end of the queue.
    // only the owner of this queue should call this.
    fn pop(&self) -> Option<JobId> {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.pop_back()
    }

    // steal a job from the public end of the queue.
    fn steal(&self) -> Option<JobId> {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.pop_front()
    }

    // push another job to the queue.
    fn push(&self, job: JobId) {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.push_back(job);
    }
}

unsafe impl Send for JobQueue {}
unsafe impl Sync for JobQueue {}

/// A worker executes jobs.
pub struct Worker {
    pool: Arc<JobPool>,
    queue: Arc<JobQueue>,
    siblings: Box<[Arc<JobQueue>]>,
}

impl Worker {
    pub fn submit<F: FnOnce(&Worker) + 'static + Send>(&self, f: F) -> JoinHandle {
        let job = Box::new(f);
        let (id, notifier) = self.pool.submit(job);
        self.queue.push(id);

        JoinHandle {
            notifier: notifier,
            worker: Some(self),
        }
    }

    fn next_job(&self) -> Option<JobId> {
        if let Some(id) = self.queue.pop() {
            Some(id)
        } else {
            let mut rng = thread_rng();
            let idx = Range::new(0, self.siblings.len()).ind_sample(&mut rng);
            let other_queue: &Arc<JobQueue> = &self.siblings[idx];

            // make sure we don't steal from our own queue.
            if &*self.queue as *const JobQueue != &**other_queue as *const JobQueue {
                other_queue.steal()
            } else {
                None
            }
        }
    }

    fn run_job(&self) {
        if let Some(id) = self.next_job() {
            self.pool.run(id, self);
        }
    }

    // run this worker to completion.
    fn run_to_completion(&self) {
        while let Some(id) = self.queue.pop() {
            self.pool.run(id, self);
        }

        // help run all the sibling queues to completion as well.
        for sibling in self.siblings.iter() {
            while let Some(id) = sibling.steal() {
                self.pool.run(id, self);
            }
        }
    }
}

/// A JoinHandle can be used to manually wait for
/// a job to be completed.
pub struct JoinHandle<'a> {
    notifier: Receiver<JobExitStatus>,
    worker: Option<&'a Worker>,
}

impl<'a> JoinHandle<'a> {
    pub fn wait(self) -> Result<(), Box<Any>> {
        loop {
            match self.notifier.try_recv() {
                Ok(exit_status) => {
                    return match exit_status {
                        JobExitStatus::Success => Ok(()),
                        JobExitStatus::Panic(err) => Err(err),
                    }
                }

                // sender _never_ hangs up before the job is done.
                Err(TryRecvError::Disconnected) => unreachable!(),

                // nothing yet, just do some more work.
                _ => {
                    if let &Some(ref worker) = &self.worker {
                        worker.run_job()
                    }
                }
            }
        }
    }
}

pub struct JobSystem {
    handles: Vec<Sender<WorkerMessage>>,
    pool: Arc<JobPool>,
    queue: Arc<JobQueue>,
}

impl JobSystem {
    pub fn submit<F: FnOnce(&Worker) + 'static + Send>(&self, f: F) -> JoinHandle {
        let job = Box::new(f);
        let (id, notifier) = self.pool.submit(job);
        self.queue.push(id);

        JoinHandle {
            notifier: notifier,
            worker: None,
        }
    }
}

impl Drop for JobSystem {
    fn drop(&mut self) {
        for handle in &self.handles {
            match handle.send(WorkerMessage::Stop) {
                _ => {}
            }
        }
    }
}

/// Create a new `JobSystem` with `n_threads` worker threads.
/// This will fail if `n_threads` is less than 1.
pub fn make_pool(n_threads: usize) -> Option<JobSystem> {
    if n_threads < 1 {
        return None;
    }

    let job_pool = Arc::new(JobPool::with_capacity(4096));

    // make a queue for every thread, including this one.
    let queues: Vec<_> = (0..n_threads+1)
        .map(|_| Arc::new(JobQueue::new())).collect();

    let queues = queues.into_boxed_slice();

    let mut handles = Vec::new();

    // Spawn a worker thread for every queue but the last one,
    // since that belongs to the thread this is spawned on.
    for queue in queues.iter().cloned().take(n_threads) {
        let (tx, rx) = channel();
        let siblings = queues.clone();
        let pool = job_pool.clone();

        thread::spawn(move || {
            let worker = Worker {
                pool: pool,
                queue: queue,
                siblings: siblings,
            };

            worker_main(worker, rx);
        });
        handles.push(tx);
    }

    Some(JobSystem {
        handles: handles,
        pool: job_pool,
        queue: queues.last().unwrap().clone(),
    })
}

// the main thread for each worker
fn worker_main(worker: Worker, rx: Receiver<WorkerMessage>) {
    loop {
        // check for message
        match rx.try_recv() {
            // time to stop!
            Ok(WorkerMessage::Stop) => {
                worker.run_to_completion();
                break;
            }

            // somehow the JobSystem got dropped without disconnecting.
            Err(TryRecvError::Disconnected) => {
                worker.run_to_completion();
                break;
            }

            _ => {}
        }
        // run a job
        worker.run_job();
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    #[should_panic]
    fn no_zero_thread_pool() {
        make_pool(0).unwrap();
    }

    #[test]
    fn basic_spawning() {
        // four worker threads, including this one.
        let pool = make_pool(4).unwrap();

        let mut handles = Vec::new();
        for _ in 0..100 {
            // empty job
            let handle = pool.submit(|_| {});
            handles.push(handle);
        }

        for handle in handles {
            // make sure every job runs and
            // no job panics.
            handle.wait().unwrap();
        }
    }

    #[test]
    fn child_jobs() {
        let pool = make_pool(4).unwrap();

        pool.submit(|worker| {
            for _ in 0..100 {
                // empty job
                worker.submit(|_| {});
            };
        }).wait().unwrap();
    }

    #[test]
    fn panic_in_the_jobqueue() {
        let pool = make_pool(4).unwrap();

        let mut handles = Vec::new();
        for i in 0..100 {
            // 4 threads, 20 panics.
            let handle = pool.submit(move |_| match i % 5 {
                0 => panic!(i),
                _ => {}
            });

            handles.push(handle);
        }

        for handle in handles {
            match handle.wait() {
                Ok(_) => {},
                Err(b) => {
                    let panic_val = b.downcast_ref::<i32>().unwrap();
                    assert_eq!(panic_val % 5, 0);
                }
            }
        }

    }

/*    #[test]
    fn scoping() {
        let pool = make_pool(4).unwrap();
        let this_worker = pool.get_worker();
        this_worker.submit(|worker| {
            let mut v = vec![0; 256];

            worker.scope(|scope| {
                for chunk in v.chunks_mut(32) {
                    scope.submit(move |_| {
                        for i in chunk { *i += 1 }
                    });
                }
            });

            for i in v {
                assert_eq!(i, 1);
            }
        }).wait();
    }*/
}<commit_msg>Added associated counter to PoolEntry<commit_after>#![feature(catch_panic, fnbox)]

//! Work-stealing job-queue implementation.
//! Inspired by the Molecular Matters blog posts.

extern crate crossbeam;
extern crate rand;

use crossbeam::mem::CachePadded;
use crossbeam::sync::TreiberStack;

use std::any::Any;
use std::boxed::FnBox;
use std::cell::RefCell;
use std::collections::VecDeque;
use std::sync::{Arc, Mutex, RwLock};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::mpsc::{channel, Sender, Receiver, TryRecvError};
use std::thread;

use ::rand::thread_rng;
use ::rand::distributions::{IndependentSample, Range};

/// Messages that can be sent to workers.
enum WorkerMessage {
    Stop,
}

type Job = Box<FnBox(&Worker) + 'static + Send>;

/// A job's id in the pool.
pub struct JobId {
    idx: usize,
}

struct PoolEntry {
    job: Job,
    sender: Sender<JobExitStatus>,
    associated_counter: *const AtomicUsize,
}

unsafe impl Send for PoolEntry {}

enum JobExitStatus {
    Success,
    Panic(Box<Any>),
}

/// A mostly lock-free pool allocator of jobs.
/// It only locks when doubling the size of the vector.
/// Eventually, I may implement a lock-free vector based off of
/// Stroustroup-et-al's paper on the subject, but for now, I've
/// decided to "keep it simple, stupid".
/// However, this should be mostly wait free unless *many* jobs
/// are being submitted.
struct JobPool {
    // we ensure with the lock-free stack of free indices that
    // an index is only used by one thread at a time.
    // mutable, disjoint borrows are therefore acceptable
    // as long as they occur within the umbrella of a read
    // lock and will not be interfered with by doubling of the
    // vector.
    jobs: RwLock<Vec<RefCell<Option<CachePadded<PoolEntry>>>>>,
    unused: TreiberStack<usize>,
}

impl JobPool {
    /// Initialize this pool with the given capacity.
    fn with_capacity(size: usize) -> Self {
        let mut v = Vec::with_capacity(size);
        let unused = TreiberStack::new();

        for i in (0..size).rev() {
            v.push(RefCell::new(None));

            unused.push(i);
        }

        JobPool {
            jobs: RwLock::new(v),
            unused: unused,
        }
    }

    /// Submit a job to be stored in the pool.
    fn submit(&self, job: Job) -> (JobId, Receiver<JobExitStatus>) {
        self.submit_with_counter(job, ::std::ptr::null())
    }

    /// Submit a job to be stored in the pool.
    fn submit_with_counter(&self, job: Job, counter: *const AtomicUsize) -> (JobId, Receiver<JobExitStatus>) {
        let next = if let Some(next) = self.unused.pop() {
            next
        } else {
            self.double();
            self.unused.pop().unwrap()
        };

        let jobs = match self.jobs.read() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        let (tx, rx) = channel();

        let entry = &jobs[next];
        *entry.borrow_mut() = Some(CachePadded::new(PoolEntry {
            job: job,
            sender: tx,
            associated_counter: counter,
        }));

        (JobId { idx: next, }, rx)
    }

    // double the vector size
    fn double(&self) {
        let mut jobs = match self.jobs.write() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        let old_len = jobs.len();
        let new_len = old_len * 2;

        jobs.reserve(new_len);

        for i in (old_len..new_len).rev() {
            jobs.push(RefCell::new(None));

            self.unused.push(i);
        }
    }

    /// Get the job given by id, passing the worker
    /// as a parameter.
    fn get_job(&self, id: JobId) -> PoolEntry {
        use std::ptr;

        let jobs = match self.jobs.read() {
            Ok(j) => j,
            Err(p) => p.into_inner(),
        };

        // take the job out before pushing the index back onto
        // the unused stack.
        let entry = jobs[id.idx].borrow_mut().take().unwrap();

        // CachePadded doesn't run destructors, so this is ok.
        let job = unsafe { ptr::read(&*entry) };
        self.unused.push(id.idx);
        job
    }
}

unsafe impl Send for JobPool {}
unsafe impl Sync for JobPool {}

/// A thread-safe double-ended queue for jobs.
/// Right now, it's just a dumb locking wrapper of VecDeque.
/// It stores indices of jobs in the global job pool.
struct JobQueue {
    inner: Mutex<VecDeque<JobId>>,
}

impl JobQueue {
    fn new() -> Self {
        JobQueue {
            inner: Mutex::new(VecDeque::new()),
        }
    }

    // pop a job from the private end of the queue.
    // only the owner of this queue should call this.
    fn pop(&self) -> Option<JobId> {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.pop_back()
    }

    // steal a job from the public end of the queue.
    fn steal(&self) -> Option<JobId> {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.pop_front()
    }

    // push another job to the queue.
    fn push(&self, job: JobId) {
        let mut guard = match self.inner.lock() {
            Ok(g) => g,
            Err(p) => p.into_inner(),
        };

        guard.push_back(job);
    }
}

unsafe impl Send for JobQueue {}
unsafe impl Sync for JobQueue {}

/// A worker executes jobs.
pub struct Worker {
    pool: Arc<JobPool>,
    queue: Arc<JobQueue>,
    siblings: Box<[Arc<JobQueue>]>,
}

impl Worker {
    pub fn submit<F: FnOnce(&Worker) + 'static + Send>(&self, f: F) -> JoinHandle {
        let job = Box::new(f);
        let (id, notifier) = self.pool.submit(job);
        self.queue.push(id);

        JoinHandle {
            notifier: notifier,
            worker: Some(self),
        }
    }

    fn run_next_job(&self) {
        let id = if let Some(id) = self.queue.pop() {
            Some(id)
        } else {
            let mut rng = thread_rng();

            let idx = Range::new(0, self.siblings.len()).ind_sample(&mut rng);
            let other_queue: &Arc<JobQueue> = &self.siblings[idx];

            // make sure we don't steal from our own queue.
            if &*self.queue as *const JobQueue != &**other_queue as *const JobQueue {
                other_queue.steal()
            } else {
                None
            }
        };

        if let Some(id) = id {
            self.run_job(id);
        }
    }

    fn run_job(&self, id: JobId) {
        use std::mem;
        let entry = self.pool.get_job(id);
        let PoolEntry { job, sender, associated_counter: counter } = entry;

        // is there a better way to do this than to box again?
        let unbounded_fn: Box<FnBox() + Send> = Box::new(move || job.call_box((self,)));
        let bounded_fn: Box<FnBox() + 'static + Send> = unsafe { mem::transmute(unbounded_fn) };

        let exit_status = match thread::catch_panic(|| bounded_fn()) {
            Ok(_) => { JobExitStatus::Success }
            Err(e) => { JobExitStatus::Panic(e) }
        };

        // decrement the associated counter and send the exit status
        if !counter.is_null() {
             unsafe { (*counter).fetch_sub(1, Ordering::SeqCst); }
        }
        drop(sender.send(exit_status)); // message doesn't _have_ to arrive
    }

    fn run_to_completion(self) {
        while let Some(job) = self.queue.pop() {
            self.run_job(job);
        }
    }

}

/// A JoinHandle can be used to manually wait for
/// a job to be completed.
pub struct JoinHandle<'a> {
    notifier: Receiver<JobExitStatus>,
    worker: Option<&'a Worker>,
}

impl<'a> JoinHandle<'a> {
    pub fn wait(self) -> Result<(), Box<Any>> {
        loop {
            match self.notifier.try_recv() {
                Ok(exit_status) => {
                    return match exit_status {
                        JobExitStatus::Success => Ok(()),
                        JobExitStatus::Panic(err) => Err(err),
                    }
                }

                // sender _never_ hangs up before the job is done.
                Err(TryRecvError::Disconnected) => unreachable!(),

                // nothing yet, just do some more work.
                _ => {
                    if let &Some(ref worker) = &self.worker {
                        worker.run_next_job()
                    }
                }
            }
        }
    }
}

pub struct JobSystem {
    handles: Vec<Sender<WorkerMessage>>,
    pool: Arc<JobPool>,
    queue: Arc<JobQueue>,
}

impl JobSystem {
    pub fn submit<F: FnOnce(&Worker) + 'static + Send>(&self, f: F) -> JoinHandle {
        let job = Box::new(f);
        let (id, notifier) = self.pool.submit(job);
        self.queue.push(id);

        JoinHandle {
            notifier: notifier,
            worker: None,
        }
    }
}

impl Drop for JobSystem {
    fn drop(&mut self) {
        for handle in &self.handles {
            match handle.send(WorkerMessage::Stop) {
                _ => {}
            }
        }
    }
}

/// Create a new `JobSystem` with `n_threads` worker threads.
/// This will fail if `n_threads` is less than 1.
pub fn make_pool(n_threads: usize) -> Option<JobSystem> {
    if n_threads < 1 {
        return None;
    }

    let job_pool = Arc::new(JobPool::with_capacity(4096));

    // make a queue for every thread, including this one.
    let queues: Vec<_> = (0..n_threads+1)
        .map(|_| Arc::new(JobQueue::new())).collect();

    let queues = queues.into_boxed_slice();
    let mut handles = Vec::new();

    // Spawn a worker thread for every queue but the last one,
    // since that belongs to the thread this is spawned on.
    for queue in queues.iter().cloned().take(n_threads) {
        let (tx, rx) = channel();
        let siblings = queues.clone();
        let pool = job_pool.clone();

        thread::spawn(move || {
            let worker = Worker {
                pool: pool,
                queue: queue,
                siblings: siblings,
            };

            worker_main(worker, rx);
        });
        handles.push(tx);
    }

    Some(JobSystem {
        handles: handles,
        pool: job_pool,
        queue: queues.last().unwrap().clone(),
    })
}

// the main thread for each worker
fn worker_main(worker: Worker, rx: Receiver<WorkerMessage>) {
    loop {
        // check for message
        match rx.try_recv() {
            // time to stop!
            Ok(WorkerMessage::Stop) => {
                worker.run_to_completion();
                break;
            }

            // somehow the JobSystem got dropped without disconnecting.
            Err(TryRecvError::Disconnected) => {
                worker.run_to_completion();
                break;
            }

            _ => {}
        }
        // run a job
        worker.run_next_job();
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    #[should_panic]
    fn no_zero_thread_pool() {
        make_pool(0).unwrap();
    }

    #[test]
    fn basic_spawning() {
        // four worker threads, including this one.
        let pool = make_pool(4).unwrap();

        let mut handles = Vec::new();
        for _ in 0..100 {
            // empty job
            let handle = pool.submit(|_| {});
            handles.push(handle);
        }

        for handle in handles {
            // make sure every job runs and
            // no job panics.
            handle.wait().unwrap();
        }
    }

    #[test]
    fn child_jobs() {
        let pool = make_pool(4).unwrap();

        pool.submit(|worker| {
            for _ in 0..100 {
                // empty job
                worker.submit(|_| {});
            };
        }).wait().unwrap();
    }

    #[test]
    fn panic_in_the_jobqueue() {
        let pool = make_pool(4).unwrap();

        let mut handles = Vec::new();
        for i in 0..100 {
            // 4 threads, 20 panics.
            let handle = pool.submit(move |_| match i % 5 {
                0 => panic!(i),
                _ => {}
            });

            handles.push(handle);
        }

        for handle in handles {
            match handle.wait() {
                Ok(_) => {},
                Err(b) => {
                    let panic_val = b.downcast_ref::<i32>().unwrap();
                    assert_eq!(panic_val % 5, 0);
                }
            }
        }

    }

/*    #[test]
    fn scoping() {
        let pool = make_pool(4).unwrap();
        let this_worker = pool.get_worker();
        this_worker.submit(|worker| {
            let mut v = vec![0; 256];

            worker.scope(|scope| {
                for chunk in v.chunks_mut(32) {
                    scope.submit(move |_| {
                        for i in chunk { *i += 1 }
                    });
                }
            });

            for i in v {
                assert_eq!(i, 1);
            }
        }).wait();
    }*/
}<|endoftext|>