<commit_before>use std::ops::{Mul, Add};

use std::fmt::{Display};

extern crate nalgebra;
use self::nalgebra::{DMat, Transpose};

extern crate rand;
use self::rand::{Rand};

extern crate num;
use self::num::{Float, Zero, FromPrimitive};

pub struct OrthogonalNMF<FloatT> {
    // TODO add docstrings
    pub hidden: DMat<FloatT>,
    pub weights: DMat<FloatT>,
    pub iteration: usize,
}

impl<FloatT: Rand + Float + FromPrimitive + Mul + Zero + Display> OrthogonalNMF<FloatT> {
    pub fn init_randomly(nhidden: usize, nobserved: usize, nsamples: usize) -> OrthogonalNMF<FloatT> {
        OrthogonalNMF {
            hidden: DMat::new_random(nhidden, nobserved),
            weights: DMat::new_random(nsamples, nhidden),
            iteration: 0,
        }
    }

    // TODO initialize with values from last time step t-1 instead of choosing randomly
    pub fn init(hidden: DMat<FloatT>, weights: DMat<FloatT>) -> OrthogonalNMF<FloatT> {
        OrthogonalNMF {
            hidden: hidden,
            weights: weights,
            iteration: 0,
        }
    }

    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// it gets better and better with each iteration.
    /// one observed per column.
    /// one sample per row.
    pub fn iterate(&mut self, data: &DMat<FloatT>) {
        let nhidden = self.hidden.nrows();
        let nobserved = self.hidden.ncols();
        let nsamples = self.weights.nrows();
        assert_eq!(nsamples, data.nrows());
        assert_eq!(nobserved, data.ncols());

        // this gets smaller and smaller with each iteration
        let alpha: FloatT =
            FloatT::from_f32(0.1).unwrap() *
            FloatT::from_f32(1.01).unwrap().powi(self.iteration as i32);

        println!("iteration = {} alpha = {}", self.iteration, alpha);

        let hidden_transposed = self.hidden.transpose();
        let weights_transposed = self.weights.transpose();

        // has the same shape as weights
        let new_weights_dividend = data.clone().mul(&hidden_transposed);
        // has the same shape as weights
        let new_weights_divisor = self.weights.clone().mul(&self.hidden).mul(&hidden_transposed);

        // has the same shape as hidden
        let new_hidden_dividend = weights_transposed.clone().mul(data);

        // gamma is a symetric matrix with diagonal elements equal to zero
        // and other elements = alpha
        let gamma_size = nhidden;
        let mut gamma = DMat::from_elem(gamma_size, gamma_size, alpha);

        // set diagonal to zero
        for i in 0..gamma_size {
            gamma[(i, i)] = FloatT::zero();
        }

        // has the same shape as hidden
        let new_hidden_divisor = weights_transposed.clone()
            .mul(&self.weights)
            .mul(&self.hidden)
            // we add the previous latents
            // multiplied by alpha except for the diag which is set to zero
            .add(gamma.mul(&self.hidden));

        // compute new weights
        // TODO efficient order ?
        for col in 0..self.weights.ncols() {
            for row in 0..self.weights.nrows() {
                let index = (row, col);
                debug_assert!(FloatT::zero() != new_weights_divisor[index]);
                self.weights[index] =
                    self.weights[index] *
                    new_weights_dividend[index] /
                    // these get larger and larger
                    new_weights_divisor[index];
            }
        }

        // compute new hidden
        // TODO efficient order ?
        for col in 0..self.hidden.ncols() {
            for row in 0..self.hidden.nrows() {
                let index = (row, col);
                debug_assert!(FloatT::zero() != new_hidden_divisor[index]);
                self.hidden[index] =
                    self.hidden[index] *
                    new_hidden_dividend[index] /
                    new_hidden_divisor[index];
            }
        }

        self.iteration += 1;
    }
}
<commit_msg>write comments<commit_after>use std::ops::{Mul, Add};

use std::fmt::{Display};

extern crate nalgebra;
use self::nalgebra::{DMat, Transpose};

extern crate rand;
use self::rand::{Rand};

extern crate num;
use self::num::{Float, Zero, FromPrimitive};

pub struct OrthogonalNMF<FloatT> {
    // TODO add docstrings
    pub hidden: DMat<FloatT>,
    pub weights: DMat<FloatT>,
    pub iteration: usize,
}

impl<FloatT: Rand + Float + FromPrimitive + Mul + Zero + Display> OrthogonalNMF<FloatT> {
    pub fn init_randomly(nhidden: usize, nobserved: usize, nsamples: usize) -> OrthogonalNMF<FloatT> {
        OrthogonalNMF {
            hidden: DMat::new_random(nhidden, nobserved),
            weights: DMat::new_random(nsamples, nhidden),
            iteration: 0,
        }
    }

    // TODO initialize with values from last time step t-1 instead of choosing randomly
    pub fn init(hidden: DMat<FloatT>, weights: DMat<FloatT>) -> OrthogonalNMF<FloatT> {
        OrthogonalNMF {
            hidden: hidden,
            weights: weights,
            iteration: 0,
        }
    }

    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// it gets better and better with each iteration.
    /// one observed per column.
    /// one sample per row.
    pub fn iterate(&mut self, data: &DMat<FloatT>) {
        let nhidden = self.hidden.nrows();
        let nobserved = self.hidden.ncols();
        let nsamples = self.weights.nrows();
        assert_eq!(nsamples, data.nrows());
        assert_eq!(nobserved, data.ncols());

        // alpha gets larger and larger with each iteration
        // 0.1, 0.101, 0.102, 0.103, ...
        // at iteration 232 alpha first goes above 1.0
        // iteration = 232 -> alpha = 1.005
        let alpha: FloatT =
            FloatT::from_f32(0.1).unwrap() *
            FloatT::from_f32(1.01).unwrap().powi(self.iteration as i32);

        println!("iteration = {} alpha = {}", self.iteration, alpha);

        let hidden_transposed = self.hidden.transpose();
        let weights_transposed = self.weights.transpose();

        // has the same shape as weights
        let new_weights_dividend = data.clone().mul(&hidden_transposed);
        // has the same shape as weights
        let new_weights_divisor = self.weights.clone().mul(&self.hidden).mul(&hidden_transposed);

        // has the same shape as hidden
        let new_hidden_dividend = weights_transposed.clone().mul(data);

        // gamma is a symetric matrix with diagonal elements equal to zero
        // and other elements = alpha
        let gamma_size = nhidden;
        let mut gamma = DMat::from_elem(gamma_size, gamma_size, alpha);

        // set diagonal to zero
        for i in 0..gamma_size {
            gamma[(i, i)] = FloatT::zero();
        }

        // has the same shape as hidden
        let new_hidden_divisor = weights_transposed.clone()
            .mul(&self.weights)
            .mul(&self.hidden)
            // we add the previous latents
            // multiplied by alpha except for the diag which is set to zero
            .add(gamma.mul(&self.hidden));

        // compute new weights
        // TODO efficient order ?
        for col in 0..self.weights.ncols() {
            for row in 0..self.weights.nrows() {
                let index = (row, col);
                // if we have any zero in any of the matrizes
                // self.weights or self.hidden then
                // new_weights_divisor[index] will be zero
                // TODO how to deal with this
                debug_assert!(FloatT::zero() != new_weights_divisor[index]);
                self.weights[index] =
                    self.weights[index] *
                    new_weights_dividend[index] /
                    // these get larger and larger
                    new_weights_divisor[index];
            }
        }

        // compute new hidden
        // TODO efficient order ?
        for col in 0..self.hidden.ncols() {
            for row in 0..self.hidden.nrows() {
                let index = (row, col);
                debug_assert!(FloatT::zero() != new_hidden_divisor[index]);
                self.hidden[index] =
                    self.hidden[index] *
                    new_hidden_dividend[index] /
                    new_hidden_divisor[index];
            }
        }

        self.iteration += 1;
    }
}
<|endoftext|>


<commit_before>/// Windows implementation of dynamic library loading.
///
/// This module should be expanded with more Windows-specific functionality in the future.
extern crate winapi;
extern crate kernel32;

use util::{CowCString, CStringAsRef};

use std::ffi::{OsStr, OsString};
use std::marker;
use std::mem;
use std::ptr;
use std::sync::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering};
use std::os::windows::ffi::{OsStrExt, OsStringExt};

/// A platform-specific equivalent of the cross-platform `Library`.
pub struct Library(winapi::HMODULE);

impl Library {
    /// Find and load a shared library (module).
    ///
    /// Locations where library is searched for is platform specific and can’t be adjusted
    /// portably.
    ///
    /// Corresponds to `LoadLibraryW(filename)`.
    #[inline]
    pub fn new<P: AsRef<OsStr>>(filename: P) -> ::Result<Library> {
        let mut wide_filename: Vec<u16> = filename.as_ref().encode_wide().collect();
        wide_filename.push(0);
        wide_filename.shrink_to_fit();
        let _guard = ErrorModeGuard::new();

        let ret = with_get_last_error(|| {
            // Make sure no winapi calls as a result of drop happen inside this closure, because
            // otherwise that might change the return value of the GetLastError.
            let handle = unsafe { kernel32::LoadLibraryW(wide_filename.as_ptr()) };
            if handle.is_null()  {
                None
            } else {
                Some(Library(handle))
            }
        }).map_err(|e| e.unwrap_or_else(||
            panic!("LoadLibraryW failed but GetLastError did not report the error")
        ));

        drop(wide_filename); // Drop wide_filename here to ensure it doesn’t get moved and dropped
                             // inside the closure by mistake.
        ret
    }

    /// Get a symbol by name.
    ///
    /// Mangling or symbol rustification is not done: trying to `get` something like `x::y`
    /// will not work.
    ///
    /// You may append a null byte at the end of the byte string to avoid string allocation in some
    /// cases. E.g. for symbol `sin` you may write `b"sin\0"` instead of `b"sin"`.
    ///
    /// # Unsafety
    ///
    /// Symbol of arbitrary requested type is returned. Using a symbol with wrong type is not
    /// memory safe.
    pub unsafe fn get<T>(&self, symbol: &[u8]) -> ::Result<Symbol<T>> {
        let symbol = try!(CowCString::from_bytes(symbol));
        with_get_last_error(|| {
            let symbol = kernel32::GetProcAddress(self.0, symbol.cstring_ref());
            if symbol.is_null() {
                None
            } else {
                Some(Symbol {
                    pointer: symbol,
                    pd: marker::PhantomData
                })
            }
        }).map_err(|e| e.unwrap_or_else(||
            panic!("GetProcAddress failed but GetLastError did not report the error")
        ))
    }
}

impl Drop for Library {
    fn drop(&mut self) {
        with_get_last_error(|| {
            if unsafe { kernel32::FreeLibrary(self.0) == 0 } {
                None
            } else {
                Some(())
            }
        }).unwrap()
    }
}

impl ::std::fmt::Debug for Library {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        unsafe {
            let mut buf: [winapi::WCHAR; 1024] = mem::uninitialized();
            let len = kernel32::GetModuleFileNameW(self.0,
                                                   (&mut buf[..]).as_mut_ptr(), 1024) as usize;
            if len == 0 {
                f.write_str(&format!("Library@{:p}", self.0))
            } else {
                let string: OsString = OsString::from_wide(&buf[..len]);
                f.write_str(&format!("Library@{:p} from {:?}", self.0, string))
            }
        }
    }
}

/// Symbol from a library.
///
/// A major difference compared to the cross-platform `Symbol` is that this does not ensure the
/// `Symbol` does not outlive `Library` it comes from.
pub struct Symbol<T> {
    pointer: winapi::FARPROC,
    pd: marker::PhantomData<T>
}

impl<T> ::std::ops::Deref for Symbol<T> {
    type Target = T;
    fn deref(&self) -> &T {
        unsafe {
            // Additional reference level for a dereference on `deref` return value.
            ::std::mem::transmute(&self.pointer)
        }
    }
}

impl<T> ::std::fmt::Debug for Symbol<T> {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        f.write_str(&format!("Symbol@{:p}", self.pointer))
    }
}

static USE_ERRORMODE: AtomicBool = ATOMIC_BOOL_INIT;
struct ErrorModeGuard(winapi::DWORD);

impl ErrorModeGuard {
    fn new() -> ErrorModeGuard {
        let mut ret = ErrorModeGuard(0);

        if !USE_ERRORMODE.load(Ordering::Acquire) {
            if unsafe { kernel32::SetThreadErrorMode(1, &mut ret.0) == 0
                        && kernel32::GetLastError() == winapi::ERROR_CALL_NOT_IMPLEMENTED } {
                USE_ERRORMODE.store(true, Ordering::Release);
            } else {
                return ret;
            }
        }
        ret.0 = unsafe { kernel32::SetErrorMode(1) };
        ret
    }
}

impl Drop for ErrorModeGuard {
    fn drop(&mut self) {
        unsafe {
            if !USE_ERRORMODE.load(Ordering::Relaxed) {
                kernel32::SetThreadErrorMode(self.0, ptr::null_mut());
            } else {
                kernel32::SetErrorMode(self.0);
            }
        }
    }
}

fn with_get_last_error<T, F>(closure: F) -> Result<T, Option<::std::io::Error>>
where F: FnOnce() -> Option<T> {
    closure().ok_or_else(|| {
        let error = unsafe { kernel32::GetLastError() };
        if error == 0 {
            None
        } else {
            Some(::std::io::Error::from_raw_os_error(error as i32))
        }
    })
}

#[test]
fn works_GetLastError() {
    let that = Library::new("kernel32.dll").unwrap();
    unsafe {
        that.get::<*mut usize>(b"GetLastError").unwrap();
    }
}

#[test]
fn works_GetLastError0() {
    let that = Library::new("kernel32.dll").unwrap();
    unsafe {
        that.get::<*mut usize>(b"GetLastError\0").unwrap();
    }
}

#[test]
fn fails_new_kernel23() {
    Library::new("kernel23").err().unwrap();
}
<commit_msg>Avoid reallocation in Windows filename building<commit_after>/// Windows implementation of dynamic library loading.
///
/// This module should be expanded with more Windows-specific functionality in the future.
extern crate winapi;
extern crate kernel32;

use util::{CowCString, CStringAsRef};

use std::ffi::{OsStr, OsString};
use std::marker;
use std::mem;
use std::ptr;
use std::sync::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering};
use std::os::windows::ffi::{OsStrExt, OsStringExt};

/// A platform-specific equivalent of the cross-platform `Library`.
pub struct Library(winapi::HMODULE);

impl Library {
    /// Find and load a shared library (module).
    ///
    /// Locations where library is searched for is platform specific and can’t be adjusted
    /// portably.
    ///
    /// Corresponds to `LoadLibraryW(filename)`.
    #[inline]
    pub fn new<P: AsRef<OsStr>>(filename: P) -> ::Result<Library> {
        let mut wide_filename: Vec<u16> = filename.as_ref().encode_wide().chain(Some(0)).collect();
        let _guard = ErrorModeGuard::new();

        let ret = with_get_last_error(|| {
            // Make sure no winapi calls as a result of drop happen inside this closure, because
            // otherwise that might change the return value of the GetLastError.
            let handle = unsafe { kernel32::LoadLibraryW(wide_filename.as_ptr()) };
            if handle.is_null()  {
                None
            } else {
                Some(Library(handle))
            }
        }).map_err(|e| e.unwrap_or_else(||
            panic!("LoadLibraryW failed but GetLastError did not report the error")
        ));

        drop(wide_filename); // Drop wide_filename here to ensure it doesn’t get moved and dropped
                             // inside the closure by mistake.
        ret
    }

    /// Get a symbol by name.
    ///
    /// Mangling or symbol rustification is not done: trying to `get` something like `x::y`
    /// will not work.
    ///
    /// You may append a null byte at the end of the byte string to avoid string allocation in some
    /// cases. E.g. for symbol `sin` you may write `b"sin\0"` instead of `b"sin"`.
    ///
    /// # Unsafety
    ///
    /// Symbol of arbitrary requested type is returned. Using a symbol with wrong type is not
    /// memory safe.
    pub unsafe fn get<T>(&self, symbol: &[u8]) -> ::Result<Symbol<T>> {
        let symbol = try!(CowCString::from_bytes(symbol));
        with_get_last_error(|| {
            let symbol = kernel32::GetProcAddress(self.0, symbol.cstring_ref());
            if symbol.is_null() {
                None
            } else {
                Some(Symbol {
                    pointer: symbol,
                    pd: marker::PhantomData
                })
            }
        }).map_err(|e| e.unwrap_or_else(||
            panic!("GetProcAddress failed but GetLastError did not report the error")
        ))
    }
}

impl Drop for Library {
    fn drop(&mut self) {
        with_get_last_error(|| {
            if unsafe { kernel32::FreeLibrary(self.0) == 0 } {
                None
            } else {
                Some(())
            }
        }).unwrap()
    }
}

impl ::std::fmt::Debug for Library {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        unsafe {
            let mut buf: [winapi::WCHAR; 1024] = mem::uninitialized();
            let len = kernel32::GetModuleFileNameW(self.0,
                                                   (&mut buf[..]).as_mut_ptr(), 1024) as usize;
            if len == 0 {
                f.write_str(&format!("Library@{:p}", self.0))
            } else {
                let string: OsString = OsString::from_wide(&buf[..len]);
                f.write_str(&format!("Library@{:p} from {:?}", self.0, string))
            }
        }
    }
}

/// Symbol from a library.
///
/// A major difference compared to the cross-platform `Symbol` is that this does not ensure the
/// `Symbol` does not outlive `Library` it comes from.
pub struct Symbol<T> {
    pointer: winapi::FARPROC,
    pd: marker::PhantomData<T>
}

impl<T> ::std::ops::Deref for Symbol<T> {
    type Target = T;
    fn deref(&self) -> &T {
        unsafe {
            // Additional reference level for a dereference on `deref` return value.
            ::std::mem::transmute(&self.pointer)
        }
    }
}

impl<T> ::std::fmt::Debug for Symbol<T> {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        f.write_str(&format!("Symbol@{:p}", self.pointer))
    }
}

static USE_ERRORMODE: AtomicBool = ATOMIC_BOOL_INIT;
struct ErrorModeGuard(winapi::DWORD);

impl ErrorModeGuard {
    fn new() -> ErrorModeGuard {
        let mut ret = ErrorModeGuard(0);

        if !USE_ERRORMODE.load(Ordering::Acquire) {
            if unsafe { kernel32::SetThreadErrorMode(1, &mut ret.0) == 0
                        && kernel32::GetLastError() == winapi::ERROR_CALL_NOT_IMPLEMENTED } {
                USE_ERRORMODE.store(true, Ordering::Release);
            } else {
                return ret;
            }
        }
        ret.0 = unsafe { kernel32::SetErrorMode(1) };
        ret
    }
}

impl Drop for ErrorModeGuard {
    fn drop(&mut self) {
        unsafe {
            if !USE_ERRORMODE.load(Ordering::Relaxed) {
                kernel32::SetThreadErrorMode(self.0, ptr::null_mut());
            } else {
                kernel32::SetErrorMode(self.0);
            }
        }
    }
}

fn with_get_last_error<T, F>(closure: F) -> Result<T, Option<::std::io::Error>>
where F: FnOnce() -> Option<T> {
    closure().ok_or_else(|| {
        let error = unsafe { kernel32::GetLastError() };
        if error == 0 {
            None
        } else {
            Some(::std::io::Error::from_raw_os_error(error as i32))
        }
    })
}

#[test]
fn works_GetLastError() {
    let that = Library::new("kernel32.dll").unwrap();
    unsafe {
        that.get::<*mut usize>(b"GetLastError").unwrap();
    }
}

#[test]
fn works_GetLastError0() {
    let that = Library::new("kernel32.dll").unwrap();
    unsafe {
        that.get::<*mut usize>(b"GetLastError\0").unwrap();
    }
}

#[test]
fn fails_new_kernel23() {
    Library::new("kernel23").err().unwrap();
}
<|endoftext|>


<commit_before>// Rust TNetStrings serialization library.

use std;
import option::{some, none};
import std::io;
import std::map;
import std::rand;

import io::writer_util;
import io::reader_util;

export t;
export to_writer;
export to_bytes;
export to_str;
export from_reader;
export from_bytes;
export from_str;
export string;
export integer;
export floating;
export boolean;
export null;
export map;
export list;

/*
Tag: t

Represents a tnetstring value.
*/
enum t {
    /* Variant: string */
    string([u8]),
    /* Variant: integer */
    integer(int),
    /* Variant: floating */
    floating(float),
    /* Variant: boolean */
    boolean(bool),
    /* Variant: null */
    null,
    /* Variant: map */
    map(map::hashmap<[u8], t>),
    /* Variant: list */
    list(@[t]),
}

/*
Function: to_writer

Serializes a tnetstring value into a io::writer.
*/
fn to_writer(writer: io::writer, t: t) {
    alt t {
        string(s) {
            writer.write_str(#fmt("%u:", vec::len(s)));
            writer.write(s);
            writer.write_char(',');
        }
        integer(i) {
            let s = int::str(i);
            writer.write_str(#fmt("%u:%s#", str::byte_len(s), s));
        }
        floating(f) {
            let s = float::to_str(f, 6u);
            writer.write_str(#fmt("%u:%s^", str::byte_len(s), s));
        }
        boolean(b) {
            let s = bool::to_str(b);
            writer.write_str(#fmt("%u:%s!", str::byte_len(s), s));
        }
        map(m) {
            let buf = io::mk_mem_buffer();
            let wr = io::mem_buffer_writer(buf);
            m.items({ |key, value|
                to_writer(wr, string(key));
                to_writer(wr, value);
            });
            let payload = io::mem_buffer_buf(buf);
            writer.write_str(#fmt("%u:", vec::len(payload)));
            writer.write(payload);
            writer.write_char('}');
        }
        list(@l) {
            let buf = io::mk_mem_buffer();
            let wr = io::mem_buffer_writer(buf);
            vec::iter(l, {|e| to_writer(wr, e) });
            let payload = io::mem_buffer_buf(buf);
            writer.write_str(#fmt("%u:", vec::len(payload)));
            writer.write(payload);
            writer.write_char(']');
        }
        null {
            writer.write_str("0:~");
        }
    }
}

/*
Function: to_bytes

Serializes a tnetstring value into a byte string.
*/
fn to_bytes(t: t) -> [u8] {
    let buf = io::mk_mem_buffer();
    let wr = io::mem_buffer_writer(buf);
    to_writer(wr, t);
    io::mem_buffer_buf(buf)
}

/*
Function: to_str

Serializes a tnetstring value into a string.
*/
fn to_str(t: t) -> str {
    let buf = io::mk_mem_buffer();
    let wr = io::mem_buffer_writer(buf);
    to_writer(wr, t);
    io::mem_buffer_str(buf)
}

/*
Function: from_reader

Deserializes a tnetstring value from an io::reader.
*/
fn from_reader(reader: io::reader) -> option::t<t> {
    assert !reader.eof();

    let c = reader.read_byte();
    let len = 0u;

    // Note that netstring spec explicitly forbids padding zeros.
    // If the first char is zero, it must be the only char.
    if c < '0' as int || c > '9' as int {
        fail "Not a tnetstring: invalid or missing length prefix";
    } else if c == '0' as int {
        c = reader.read_byte();
    } else {
        do {
            len = (10u * len) + ((c as uint) - ('0' as uint));

            if reader.eof() {
                fail "Not a tnetstring: invalid or missing length prefix";
            }
            c = reader.read_byte();
        } while c >= '0' as int && c <= '9' as int;
    }

    // Validate end-of-length-prefix marker.
    if c != ':' as int {
        fail "Not a tnetstring: missing length prefix";
    }

    // Read the data plus terminating type tag.
    let payload = reader.read_bytes(len);

    if vec::len(payload) != len {
        fail "Not a tnetstring: invalid length prefix";
    }

    if reader.eof() {
        fail "Not a tnetstring: missing type tag";
    }

    alt reader.read_byte() as char {
        '#' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = int::from_str(s);
            some(integer(v))
        }
        '}' { some(map(parse_map(payload))) }
        ']' { some(list(parse_list(payload))) }
        '!' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = bool::from_str(s);
            some(boolean(v))
        }
        '^' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = float::from_str(s);
            some(floating(v))
        }
        '~' {
            assert vec::len(payload) == 0u;
            some(null)
        }
        ',' { some(string(payload)) }
        c {
            let s = str::from_char(c);
            fail #fmt("Invalid payload type: %s", str::escape(s))
        }
    }
}

fn parse_list(data: [u8]) -> @[t] {
    if vec::len(data) == 0u { ret @[]; }

    let reader = io::bytes_reader(data);

    let value = from_reader(reader);
    assert option::is_some(value);
    let result = [option::get(value)];

    while !reader.eof() {
        let value = from_reader(reader);
        assert option::is_some(value);
        vec::push(result, option::get(value));
    }

    ret @result;
}

fn parse_pair(reader: io::reader) -> ([u8], t) {
    let key = from_reader(reader);
    assert option::is_some(key);
    assert !reader.eof();

    let key = alt option::get(key) {
        string(key) { key }
        _ { fail "Keys can only be strings." }
    };

    let value = from_reader(reader);
    assert option::is_some(value);

    ret (key, option::get(value));
}

fn parse_map(data: [u8]) -> map::hashmap<[u8], t> {
    if vec::len(data) == 0u { ret map::new_bytes_hash(); }

    let reader = io::bytes_reader(data);
    let (key, value) = parse_pair(reader);

    let result = map::new_bytes_hash();
    result.insert(key, value);

    while !reader.eof() {
        let (key, value) = parse_pair(reader);
        result.insert(key, value);
    }

    ret result;
}

/*
Function: from_bytes

Deserializes a tnetstring value from a byte string.
*/
fn from_bytes(data: [u8]) -> (option::t<t>, [u8]) {
    let reader = io::bytes_reader(data);
    let tnetstring = from_reader(reader);
    (tnetstring, reader.read_whole_stream())
}

/*
Function: from_str

Deserializes a tnetstring value from a string.
*/
fn from_str(data: str) -> (option::t<t>, str) {
    let reader = io::string_reader(data);
    let tnetstring = from_reader(reader);
    (tnetstring, unsafe { str::unsafe::from_bytes(reader.read_whole_stream()) })

}

fn eq(t0: t, t1: t) -> bool {
    alt (t0, t1) {
        (string(s0), string(s1)) { s0 == s1 }
        (integer(i0), integer(i1)) { i0 == i1 }
        (floating(f0), floating(f1)) { f0 == f1 }
        (boolean(b0), boolean(b1)) { b0 == b1 }
        (null, null) { true }
        (map(d0), map(d1)) {
            if d0.size() == d1.size() {
                let equal = true;
                d0.items() { |k, v|
                    if !d1.contains_key(k) || !eq(d1.get(k), v) {
                        equal = false;
                    }
                };
                equal
            } else {
                false
            }
        }
        (list(l0), list(l1)) {
            vec::all2(*l0, *l1, eq)
        }
        _ { false }
    }
}

#[cfg(test)]
mod tests {
    // Tests inspired by https://github.com/rfk/tnetstring.

    fn test(string: str, expected: t) {
        let (actual, rest) = from_str(string);
        assert option::is_some(actual);
        assert rest == "";

        let actual = option::get(actual);
        assert eq(actual, expected);
        assert to_str(expected) == string;
    }

    #[test]
    fn test_format() {
        test("11:hello world,", string(str::bytes("hello world")));
        test("0:}", map(map::new_bytes_hash()));
        test("0:]", list(@[]));

        let d = map::new_bytes_hash();
        d.insert(str::bytes("hello"),
                list(@[
                    integer(12345678901),
                    string(str::bytes("this")),
                    boolean(true),
                    null,
                    string(str::bytes("\x00\x00\x00\x00"))]));

        test("51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00" +
            "\x00,]}", map(d));

        test("5:12345#", integer(12345));
        test("12:this is cool,", string(str::bytes("this is cool")));
        test("0:,", string(str::bytes("")));
        test("0:~", null);
        test("4:true!", boolean(true));
        test("5:false!", boolean(false));
        test("10:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,",
            string(str::bytes("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")));
        test("24:5:12345#5:67890#5:xxxxx,]",
            list(
                @[integer(12345),
                integer(67890),
                string(str::bytes("xxxxx"))]));
        test("18:3:0.1^3:0.2^3:0.4^]",
           list(@[floating(0.1), floating(0.2), floating(0.4)]));
        test("243:238:233:228:223:218:213:208:203:198:193:188:183:178:173:" +
            "168:163:158:153:148:143:138:133:128:123:118:113:108:103:99:95:" +
            "91:87:83:79:75:71:67:63:59:55:51:47:43:39:35:31:27:23:19:15:" +
            "11:hello-there,]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" +
            "]]]]",
            list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[
                    string(str::bytes("hello-there"))
                ])])])])])])])])])])])])])])])])])])])])])])])])])])])])
                ])])])])])])])])])])])])])])])])])])])])])])]));
    }

    #[test]
    fn test_random() {
        fn randint(rng: rand::rng, a: u32, b: u32) -> u32 {
            (rng.next() % (b - a + 1u32)) + a
        }

        fn get_random_object(rng: rand::rng, depth: u32) -> t {
            if randint(rng, depth, 10u32) <= 4u32 {
                if randint(rng, 0u32, 1u32) == 0u32 {
                    let n = randint(rng, 0u32, 10u32);
                    list(@vec::init_fn(n as uint) { |_i|
                        get_random_object(rng, depth + 1u32)
                    })
                } else {
                    let d = map::new_bytes_hash();

                    let i = randint(rng, 0u32, 10u32);
                    while i != 0u32 {
                        let s = rng.gen_bytes(randint(rng, 0u32, 100u32) as uint);
                        d.insert(s, get_random_object(rng, depth + 1u32));
                        i -= 1u32;
                    }
                    map(d)
                }
            } else {
                alt randint(rng, 0u32, 5u32) {
                    0u32 { null }
                    1u32 { boolean(true) }
                    2u32 { boolean(false) }
                    3u32 {
                        if randint(rng, 0u32, 1u32) == 0u32 {
                            integer(rng.next() as int)
                        } else {
                            integer(-rng.next() as int)
                        }
                    }
                    4u32 {
                        let f = rng.next_float();

                        // Generate a float that can be exactly converted to
                        // and from a string.
                        while true {
                            let f1 = float::from_str(float::to_str(f, 6u));
                            if f == f1 { break; }
                            f = f1;
                        }

                        if randint(rng, 0u32, 1u32) == 0u32 {
                            floating(f)
                        } else {
                            floating(-f)
                        }
                    }
                    5u32 {
                        string(rng.gen_bytes(randint(rng, 0u32, 100u32) as uint))
                    }
                    _ { fail }
                }
            }
        }

        let rng = rand::mk_rng();

        let i = 500u;
        while i != 0u {
            let v0 = get_random_object(rng, 0u32);

            alt from_bytes(to_bytes(v0)) {
                (some(v1), rest) if rest == [] {
                    assert eq(v0, v1);
                }
                _ { fail; }
            }
            i -= 1u;
        }
    }
}
<commit_msg>rewrite to use rustdoc.<commit_after>#[doc = "Rust TNetStrings serialization library."];

use std;
import option::{some, none};
import std::io;
import std::map;
import std::rand;

import io::writer_util;
import io::reader_util;

export t;
export to_writer;
export to_bytes;
export to_str;
export from_reader;
export from_bytes;
export from_str;
export string;
export integer;
export floating;
export boolean;
export null;
export map;
export list;

#[doc = "Represents a tnetstring value."]
enum t {
    #[doc = "string"]
    string([u8]),
    #[doc = "integer"]
    integer(int),
    #[doc = "floating"]
    floating(float),
    #[doc = "boolean"]
    boolean(bool),
    #[doc = "null"]
    null,
    #[doc = "map"]
    map(map::hashmap<[u8], t>),
    #[doc = "list"]
    list(@[t]),
}

#[doc = "Serializes a tnetstring value into a io::writer."]
fn to_writer(writer: io::writer, t: t) {
    alt t {
        string(s) {
            writer.write_str(#fmt("%u:", vec::len(s)));
            writer.write(s);
            writer.write_char(',');
        }
        integer(i) {
            let s = int::str(i);
            writer.write_str(#fmt("%u:%s#", str::byte_len(s), s));
        }
        floating(f) {
            let s = float::to_str(f, 6u);
            writer.write_str(#fmt("%u:%s^", str::byte_len(s), s));
        }
        boolean(b) {
            let s = bool::to_str(b);
            writer.write_str(#fmt("%u:%s!", str::byte_len(s), s));
        }
        map(m) {
            let buf = io::mk_mem_buffer();
            let wr = io::mem_buffer_writer(buf);
            m.items({ |key, value|
                to_writer(wr, string(key));
                to_writer(wr, value);
            });
            let payload = io::mem_buffer_buf(buf);
            writer.write_str(#fmt("%u:", vec::len(payload)));
            writer.write(payload);
            writer.write_char('}');
        }
        list(@l) {
            let buf = io::mk_mem_buffer();
            let wr = io::mem_buffer_writer(buf);
            vec::iter(l, {|e| to_writer(wr, e) });
            let payload = io::mem_buffer_buf(buf);
            writer.write_str(#fmt("%u:", vec::len(payload)));
            writer.write(payload);
            writer.write_char(']');
        }
        null {
            writer.write_str("0:~");
        }
    }
}

#[doc = "Serializes a tnetstring value into a byte string."]
fn to_bytes(t: t) -> [u8] {
    let buf = io::mk_mem_buffer();
    let wr = io::mem_buffer_writer(buf);
    to_writer(wr, t);
    io::mem_buffer_buf(buf)
}

#[doc = "Serializes a tnetstring value into a string."]
fn to_str(t: t) -> str {
    let buf = io::mk_mem_buffer();
    let wr = io::mem_buffer_writer(buf);
    to_writer(wr, t);
    io::mem_buffer_str(buf)
}

#[doc = "Deserializes a tnetstring value from an io::reader."]
fn from_reader(reader: io::reader) -> option::t<t> {
    assert !reader.eof();

    let c = reader.read_byte();
    let len = 0u;

    // Note that netstring spec explicitly forbids padding zeros.
    // If the first char is zero, it must be the only char.
    if c < '0' as int || c > '9' as int {
        fail "Not a tnetstring: invalid or missing length prefix";
    } else if c == '0' as int {
        c = reader.read_byte();
    } else {
        do {
            len = (10u * len) + ((c as uint) - ('0' as uint));

            if reader.eof() {
                fail "Not a tnetstring: invalid or missing length prefix";
            }
            c = reader.read_byte();
        } while c >= '0' as int && c <= '9' as int;
    }

    // Validate end-of-length-prefix marker.
    if c != ':' as int {
        fail "Not a tnetstring: missing length prefix";
    }

    // Read the data plus terminating type tag.
    let payload = reader.read_bytes(len);

    if vec::len(payload) != len {
        fail "Not a tnetstring: invalid length prefix";
    }

    if reader.eof() {
        fail "Not a tnetstring: missing type tag";
    }

    alt reader.read_byte() as char {
        '#' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = int::from_str(s);
            some(integer(v))
        }
        '}' { some(map(parse_map(payload))) }
        ']' { some(list(parse_list(payload))) }
        '!' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = bool::from_str(s);
            some(boolean(v))
        }
        '^' {
            let s = unsafe { str::unsafe::from_bytes(payload) };
            let v = float::from_str(s);
            some(floating(v))
        }
        '~' {
            assert vec::len(payload) == 0u;
            some(null)
        }
        ',' { some(string(payload)) }
        c {
            let s = str::from_char(c);
            fail #fmt("Invalid payload type: %s", str::escape(s))
        }
    }
}

fn parse_list(data: [u8]) -> @[t] {
    if vec::len(data) == 0u { ret @[]; }

    let reader = io::bytes_reader(data);

    let value = from_reader(reader);
    assert option::is_some(value);
    let result = [option::get(value)];

    while !reader.eof() {
        let value = from_reader(reader);
        assert option::is_some(value);
        vec::push(result, option::get(value));
    }

    ret @result;
}

fn parse_pair(reader: io::reader) -> ([u8], t) {
    let key = from_reader(reader);
    assert option::is_some(key);
    assert !reader.eof();

    let key = alt option::get(key) {
        string(key) { key }
        _ { fail "Keys can only be strings." }
    };

    let value = from_reader(reader);
    assert option::is_some(value);

    ret (key, option::get(value));
}

fn parse_map(data: [u8]) -> map::hashmap<[u8], t> {
    if vec::len(data) == 0u { ret map::new_bytes_hash(); }

    let reader = io::bytes_reader(data);
    let (key, value) = parse_pair(reader);

    let result = map::new_bytes_hash();
    result.insert(key, value);

    while !reader.eof() {
        let (key, value) = parse_pair(reader);
        result.insert(key, value);
    }

    ret result;
}

#[doc = "Deserializes a tnetstring value from a byte string."]
fn from_bytes(data: [u8]) -> (option::t<t>, [u8]) {
    let reader = io::bytes_reader(data);
    let tnetstring = from_reader(reader);
    (tnetstring, reader.read_whole_stream())
}

#[doc = "Deserializes a tnetstring value from a string."]
fn from_str(data: str) -> (option::t<t>, str) {
    let reader = io::string_reader(data);
    let tnetstring = from_reader(reader);
    (tnetstring, unsafe { str::unsafe::from_bytes(reader.read_whole_stream()) })

}

#[doc = "Test the equality between two tnetstring values"]
fn eq(t0: t, t1: t) -> bool {
    alt (t0, t1) {
        (string(s0), string(s1)) { s0 == s1 }
        (integer(i0), integer(i1)) { i0 == i1 }
        (floating(f0), floating(f1)) { f0 == f1 }
        (boolean(b0), boolean(b1)) { b0 == b1 }
        (null, null) { true }
        (map(d0), map(d1)) {
            if d0.size() == d1.size() {
                let equal = true;
                d0.items() { |k, v|
                    if !d1.contains_key(k) || !eq(d1.get(k), v) {
                        equal = false;
                    }
                };
                equal
            } else {
                false
            }
        }
        (list(l0), list(l1)) {
            vec::all2(*l0, *l1, eq)
        }
        _ { false }
    }
}

#[cfg(test)]
mod tests {
    // Tests inspired by https://github.com/rfk/tnetstring.

    fn test(string: str, expected: t) {
        let (actual, rest) = from_str(string);
        assert option::is_some(actual);
        assert rest == "";

        let actual = option::get(actual);
        assert eq(actual, expected);
        assert to_str(expected) == string;
    }

    #[test]
    fn test_format() {
        test("11:hello world,", string(str::bytes("hello world")));
        test("0:}", map(map::new_bytes_hash()));
        test("0:]", list(@[]));

        let d = map::new_bytes_hash();
        d.insert(str::bytes("hello"),
                list(@[
                    integer(12345678901),
                    string(str::bytes("this")),
                    boolean(true),
                    null,
                    string(str::bytes("\x00\x00\x00\x00"))]));

        test("51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00" +
            "\x00,]}", map(d));

        test("5:12345#", integer(12345));
        test("12:this is cool,", string(str::bytes("this is cool")));
        test("0:,", string(str::bytes("")));
        test("0:~", null);
        test("4:true!", boolean(true));
        test("5:false!", boolean(false));
        test("10:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,",
            string(str::bytes("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")));
        test("24:5:12345#5:67890#5:xxxxx,]",
            list(
                @[integer(12345),
                integer(67890),
                string(str::bytes("xxxxx"))]));
        test("18:3:0.1^3:0.2^3:0.4^]",
           list(@[floating(0.1), floating(0.2), floating(0.4)]));
        test("243:238:233:228:223:218:213:208:203:198:193:188:183:178:173:" +
            "168:163:158:153:148:143:138:133:128:123:118:113:108:103:99:95:" +
            "91:87:83:79:75:71:67:63:59:55:51:47:43:39:35:31:27:23:19:15:" +
            "11:hello-there,]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" +
            "]]]]",
            list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[list(@[list(@[list(@[list(@[list(@[list(
                @[list(@[list(@[
                    string(str::bytes("hello-there"))
                ])])])])])])])])])])])])])])])])])])])])])])])])])])])])
                ])])])])])])])])])])])])])])])])])])])])])])]));
    }

    #[test]
    fn test_random() {
        fn randint(rng: rand::rng, a: u32, b: u32) -> u32 {
            (rng.next() % (b - a + 1u32)) + a
        }

        fn get_random_object(rng: rand::rng, depth: u32) -> t {
            if randint(rng, depth, 10u32) <= 4u32 {
                if randint(rng, 0u32, 1u32) == 0u32 {
                    let n = randint(rng, 0u32, 10u32);
                    list(@vec::init_fn(n as uint) { |_i|
                        get_random_object(rng, depth + 1u32)
                    })
                } else {
                    let d = map::new_bytes_hash();

                    let i = randint(rng, 0u32, 10u32);
                    while i != 0u32 {
                        let s = rng.gen_bytes(randint(rng, 0u32, 100u32) as uint);
                        d.insert(s, get_random_object(rng, depth + 1u32));
                        i -= 1u32;
                    }
                    map(d)
                }
            } else {
                alt randint(rng, 0u32, 5u32) {
                    0u32 { null }
                    1u32 { boolean(true) }
                    2u32 { boolean(false) }
                    3u32 {
                        if randint(rng, 0u32, 1u32) == 0u32 {
                            integer(rng.next() as int)
                        } else {
                            integer(-rng.next() as int)
                        }
                    }
                    4u32 {
                        let f = rng.next_float();

                        // Generate a float that can be exactly converted to
                        // and from a string.
                        while true {
                            let f1 = float::from_str(float::to_str(f, 6u));
                            if f == f1 { break; }
                            f = f1;
                        }

                        if randint(rng, 0u32, 1u32) == 0u32 {
                            floating(f)
                        } else {
                            floating(-f)
                        }
                    }
                    5u32 {
                        string(rng.gen_bytes(randint(rng, 0u32, 100u32) as uint))
                    }
                    _ { fail }
                }
            }
        }

        let rng = rand::mk_rng();

        let i = 500u;
        while i != 0u {
            let v0 = get_random_object(rng, 0u32);

            alt from_bytes(to_bytes(v0)) {
                (some(v1), rest) if rest == [] {
                    assert eq(v0, v1);
                }
                _ { fail; }
            }
            i -= 1u;
        }
    }
}
<|endoftext|>


<commit_before>use std::sync::Arc;
use ::flota::config;
use ::flota::template;
use ::util::errors::*;

pub mod host;
use self::host::Host;

#[derive(Debug)]
pub struct Cluster<'a> {
    pub name: String,
    pub hosts: Vec<Host<'a>>,
}

impl<'a> Cluster<'a> {
    pub fn new(cluster: &config::cluster::Cluster,
               templates: &Vec<Arc<template::Template<'a>>>)
                              -> Result<Self>
    {
        let mut res = Vec::new();
        for host in cluster.hosts.iter() {
            // search for a template matched to the host
            let template = match templates.iter().find(|&t| t.name == host.template.name) {
                Some(v) => v,
                None => continue,
            };
            match Host::new(host, &template) {
                Ok(h) => res.push(h),
                Err(e) => {
                    error!("{}", e);
                    // who needs a cluster missing any of its defined hosts
                    return Err(format!("failed to create Cluster: {}", cluster.name).into());
                }
            }
        }
        Ok(Cluster {
            name: cluster.name.clone(),
            hosts: res,
        })
    }
}
<commit_msg>hello cluster automated test<commit_after>use std::sync::Arc;
use ::exec::session::*;
use ::flota::config;
use ::flota::template;
use ::util::errors::*;

pub mod host;
use self::host::Host;

#[derive(Debug)]
pub struct Cluster<'a> {
    pub name: String,
    pub hosts: Vec<Host<'a>>,
}

impl<'a> Cluster<'a> {
    pub fn new(cluster: &config::cluster::Cluster,
               templates: &Vec<Arc<template::Template<'a>>>)
                              -> Result<Self>
    {
        let mut hosts = Vec::new();
        for host in cluster.hosts.iter() {
            // search for a template matched to the host
            let template = match templates.iter().find(|&t| t.name == host.template.name) {
                Some(v) => v,
                None => continue,
            };
            match Host::new(host, &template) {
                Ok(h) => hosts.push(h),
                Err(e) => {
                    error!("{}", e);
                    // who needs a cluster missing any of its defined hosts
                    return Err(format!("failed to create Cluster: {}", cluster.name).into());
                }
            }
        }

        // pre-tests --> tests --> post-tests
        for tests in vec![
            &cluster.pre_tests,
            &cluster.tests,
            &cluster.post_tests
        ].iter() {
            for one_exec in tests.iter() {
                // XXX: just ugly. help me.
                if let Some(host) = hosts.iter().find(|h| h.domain.name() == one_exec.host) {
                    if let Some(seed_type) = SeedType::from_exec_type(&one_exec.exec_type) {
                        if let Some(seed) = host.template.session_seeds.iter().find(|s| {
                            s.seed_type() == seed_type
                        }) {
                            let sess = seed.spawn().unwrap();
                            match sess.exec(&one_exec.command) {
                                Ok(ret) => {
                                    info!("exit status: {}", ret.status);
                                    info!("stdout: {}", ret.stdout);
                                    info!("stderr: {}", ret.stderr);
                                },
                                Err(e) => {
                                    error!("{}", e);
                                }
                            }
                        } else {
                            error!("requested method is not provided of that host");
                        }
                    } else {
                        panic!("would not panic")
                    }
                }
            }
        }

        Ok(Cluster {
            name: cluster.name.clone(),
            hosts: hosts,
        })
    }
}
<|endoftext|>


<commit_before>//! leveldb iterators
//!
//! Iteration is one of the most important parts of leveldb. This module provides
//! Iterators to iterate over key, values and pairs of both.
use cbits::leveldb::{leveldb_iterator_t,leveldb_iter_seek_to_first,leveldb_iter_destroy,leveldb_iter_seek_to_last,
leveldb_create_iterator,leveldb_iter_valid,leveldb_iter_next,leveldb_iter_key,leveldb_iter_value,leveldb_readoptions_destroy};
use libc::{size_t};
use std::vec::raw::from_buf;
use std::iter;
use super::Database;
use super::options::{ReadOptions,c_readoptions};
use super::db_key::{Key,from_u8};

#[allow(missing_docs)]
struct RawIterator {
  ptr: *mut leveldb_iterator_t,
}

#[allow(missing_docs)]
impl Drop for RawIterator {
  fn drop(&mut self) {
    unsafe { leveldb_iter_destroy(self.ptr) }
  }
}

/// An iterator over the leveldb keyspace.
///
/// Returns key and value as a tuple.
pub struct Iterator<K: Key, V> {
  start: bool,
  iter: RawIterator
}

/// An iterator over the leveldb keyspace.
///
/// Returns just the keys.
pub struct KeyIterator<K: Key> {
  start: bool,
  iter: RawIterator
}

/// An iterator over the leveldb keyspace.
///
/// Returns just the value.
pub struct ValueIterator<V> {
  start: bool,
  iter: RawIterator
}

/// A trait to allow access to the three main iteration styles of leveldb.
pub trait Iterable<K: Key,V> {
  /// Return an Iterator iterating over (Key,Value) pairs
  fn iter(&self, options: ReadOptions) -> Iterator<K,V>;
  /// Returns an Iterator iterating over Keys only.
  fn keys_iter(&self, options: ReadOptions) -> KeyIterator<K>;
  /// Returns an Iterator iterating over Values only.
  fn value_iter(&self, options: ReadOptions) -> ValueIterator<V>;
}

impl<K: Key + Ord, V> Iterable<K, V> for Database<K> {
  fn iter(&self, options: ReadOptions) -> Iterator<K,V> {
    Iterator::new(self, options)
  }
  fn keys_iter(&self, options: ReadOptions) -> KeyIterator<K> {
    KeyIterator::new(self, options)
  }

  fn value_iter(&self, options: ReadOptions) -> ValueIterator<V> {
    ValueIterator::new(self, options)
  }
}

#[allow(missing_docs)]
trait LevelDBIterator {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t;

  #[inline]
  fn start(&self) -> bool;

  #[inline]
  fn started(&mut self);

  fn valid(&self) -> bool {
    unsafe { leveldb_iter_valid(self.raw_iterator()) != 0 }
  }

  fn seek_to_first(&self) {
    unsafe { leveldb_iter_seek_to_first(self.raw_iterator()) }
  }

  fn advance(&mut self) -> bool {
    unsafe {
      if !self.start() {
        leveldb_iter_next(self.raw_iterator());
      } else {
        self.started();
      }
    }
    self.valid()
  }

  fn seek_last(&mut self) {
    unsafe {
      leveldb_iter_seek_to_last(self.raw_iterator());
    }
  }
}

#[allow(missing_docs)]
trait ValueAccess<V> : LevelDBIterator {
  fn value(&self) -> V {
    unsafe {
      let length: size_t = 0;
      let value = leveldb_iter_value(self.raw_iterator(),
                                     &length) as *const u8;
      let vec = from_buf(value, length as uint);
      self.convert_value(vec)
    }
  }

  fn convert_value(&self, v: Vec<u8>) -> V;
}

#[allow(missing_docs)]
trait KeyAccess<K: Key> : LevelDBIterator {
  fn key(&self) -> K {
    unsafe {
      let length: size_t = 0;
      let value = leveldb_iter_key(self.raw_iterator(),
                                   &length) as *const u8;
      from_u8(from_buf(value, length as uint).as_slice())
    }
  }
}

impl<K: Key, V> Iterator<K, V> {
  fn new(database: &Database<K>,
         options: ReadOptions) -> Iterator<K,V> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      Iterator { start: true, iter: RawIterator { ptr: ptr } }
    }
  }
}

impl<K: Key, V> LevelDBIterator for Iterator<K,V> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl<K: Key, V> KeyAccess<K> for Iterator<K,V> {}
impl<K: Key> ValueAccess<Vec<u8>> for Iterator<K,Vec<u8>> {
  fn convert_value(&self, val: Vec<u8>) -> Vec<u8> {
    val
  }
}

impl<K: Key> KeyIterator<K> {
  fn new(database: &Database<K>,
         options: ReadOptions) -> KeyIterator<K> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      KeyIterator { start: true, iter: RawIterator { ptr: ptr } }
    }
  }
}

impl<K: Key> LevelDBIterator for KeyIterator<K> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl<K: Key> KeyAccess<K> for KeyIterator<K> { }

impl<K, V> ValueIterator<V> {
  fn new(database: &Database<K>,
         options: ReadOptions) -> ValueIterator<V> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      ValueIterator { start: true, iter: RawIterator { ptr: ptr } }
    }
  }
}

impl<V> LevelDBIterator for ValueIterator<V> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl ValueAccess<Vec<u8>> for ValueIterator<Vec<u8>> {
  fn convert_value(&self, val: Vec<u8>) -> Vec<u8> {
    val
  }
}

impl<K: Key> iter::Iterator<(K,Vec<u8>)> for Iterator<K,Vec<u8>> {
  fn next(&mut self) -> Option<(K,Vec<u8>)> {
    if self.advance() {
      Some((self.key(), self.value()))
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<(K,Vec<u8>)> {
    self.seek_last();
    Some((self.key(), self.value()))
  }
}

impl<K: Key> iter::Iterator<K> for KeyIterator<K> {
  fn next(&mut self) -> Option<K> {
    if self.advance() {
      Some(self.key())
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<K> {
     self.seek_last();
     Some(self.key())
  }
}

impl iter::Iterator<Vec<u8>> for ValueIterator<Vec<u8>> {
  fn next(&mut self) -> Option<Vec<u8>> {
    if self.advance() {
      Some(self.value())
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<Vec<u8>> {
     self.seek_last();
     Some(self.value())
  }
}
<commit_msg>Make iterators safe<commit_after>//! leveldb iterators
//!
//! Iteration is one of the most important parts of leveldb. This module provides
//! Iterators to iterate over key, values and pairs of both.
use cbits::leveldb::{leveldb_iterator_t,leveldb_iter_seek_to_first,leveldb_iter_destroy,leveldb_iter_seek_to_last,
leveldb_create_iterator,leveldb_iter_valid,leveldb_iter_next,leveldb_iter_key,leveldb_iter_value,leveldb_readoptions_destroy};
use libc::{size_t};
use std::vec::raw::from_buf;
use std::iter;
use super::Database;
use super::options::{ReadOptions,c_readoptions};
use super::db_key::{Key,from_u8};

#[allow(missing_docs)]
struct RawIterator {
  ptr: *mut leveldb_iterator_t,
}

#[allow(missing_docs)]
impl Drop for RawIterator {
  fn drop(&mut self) {
    unsafe { leveldb_iter_destroy(self.ptr) }
  }
}

/// An iterator over the leveldb keyspace.
///
/// Returns key and value as a tuple.
pub struct Iterator<'a, K: Key, V> {
  start: bool,
  // Iterator accesses the Database through a leveldb_iter_t pointer
  #[allow(dead_code)]
  database: &'a Database<K>,
  iter: RawIterator
}

/// An iterator over the leveldb keyspace.
///
/// Returns just the keys.
pub struct KeyIterator<'a, K: Key> {
  start: bool,
  // Iterator accesses the Database through a leveldb_iter_t pointer
  #[allow(dead_code)]
  database: &'a Database<K>,
  iter: RawIterator
}

/// An iterator over the leveldb keyspace.
///
/// Returns just the value.
pub struct ValueIterator<'a,K: Key,V> {
  start: bool,
  // Iterator accesses the Database through a leveldb_iter_t pointer
  #[allow(dead_code)]
  database: &'a Database<K>,
  iter: RawIterator
}

/// A trait to allow access to the three main iteration styles of leveldb.
pub trait Iterable<K: Key,V> {
  /// Return an Iterator iterating over (Key,Value) pairs
  fn iter(&self, options: ReadOptions) -> Iterator<K,V>;
  /// Returns an Iterator iterating over Keys only.
  fn keys_iter(&self, options: ReadOptions) -> KeyIterator<K>;
  /// Returns an Iterator iterating over Values only.
  fn value_iter(&self, options: ReadOptions) -> ValueIterator<K,V>;
}

impl<K: Key + Ord, V> Iterable<K, V> for Database<K> {
  fn iter(&self, options: ReadOptions) -> Iterator<K,V> {
    Iterator::new(self, options)
  }
  fn keys_iter(&self, options: ReadOptions) -> KeyIterator<K> {
    KeyIterator::new(self, options)
  }

  fn value_iter(&self, options: ReadOptions) -> ValueIterator<K,V> {
    ValueIterator::new(self, options)
  }
}

#[allow(missing_docs)]
trait LevelDBIterator {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t;

  #[inline]
  fn start(&self) -> bool;

  #[inline]
  fn started(&mut self);

  fn valid(&self) -> bool {
    unsafe { leveldb_iter_valid(self.raw_iterator()) != 0 }
  }

  fn seek_to_first(&self) {
    unsafe { leveldb_iter_seek_to_first(self.raw_iterator()) }
  }

  fn advance(&mut self) -> bool {
    unsafe {
      if !self.start() {
        leveldb_iter_next(self.raw_iterator());
      } else {
        self.started();
      }
    }
    self.valid()
  }

  fn seek_last(&mut self) {
    unsafe {
      leveldb_iter_seek_to_last(self.raw_iterator());
    }
  }
}

#[allow(missing_docs)]
trait ValueAccess<V> : LevelDBIterator {
  fn value(&self) -> V {
    unsafe {
      let length: size_t = 0;
      let value = leveldb_iter_value(self.raw_iterator(),
                                     &length) as *const u8;
      let vec = from_buf(value, length as uint);
      self.convert_value(vec)
    }
  }

  fn convert_value(&self, v: Vec<u8>) -> V;
}

#[allow(missing_docs)]
trait KeyAccess<K: Key> : LevelDBIterator {
  fn key(&self) -> K {
    unsafe {
      let length: size_t = 0;
      let value = leveldb_iter_key(self.raw_iterator(),
                                   &length) as *const u8;
      from_u8(from_buf(value, length as uint).as_slice())
    }
  }
}

impl<'a, K: Key, V> Iterator<'a,K, V> {
  fn new(database: &'a Database<K>,
         options: ReadOptions) -> Iterator<'a,K,V> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      Iterator { start: true, iter: RawIterator { ptr: ptr }, database: database }
    }
  }
}

impl<'a, K: Key, V> LevelDBIterator for Iterator<'a,K,V> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl<'a, K: Key, V> KeyAccess<K> for Iterator<'a,K,V> {}
impl<'a, K: Key> ValueAccess<Vec<u8>> for Iterator<'a,K,Vec<u8>> {
  fn convert_value(&self, val: Vec<u8>) -> Vec<u8> {
    val
  }
}

impl<'a,K: Key> KeyIterator<'a,K> {
  fn new(database: &'a Database<K>,
         options: ReadOptions) -> KeyIterator<K> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      KeyIterator { start: true, iter: RawIterator { ptr: ptr }, database: database }
    }
  }
}

impl<'a,K: Key> LevelDBIterator for KeyIterator<'a,K> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl<'a,K: Key> KeyAccess<K> for KeyIterator<'a,K> { }

impl<'a,K: Key,V> ValueIterator<'a,K,V> {
  fn new(database: &'a Database<K>,
         options: ReadOptions) -> ValueIterator<K,V> {
    unsafe {
      let c_readoptions = c_readoptions(options);
      let ptr = leveldb_create_iterator(database.database.ptr,
                                        c_readoptions);
      leveldb_readoptions_destroy(c_readoptions);
      leveldb_iter_seek_to_first(ptr);
      ValueIterator { start: true, iter: RawIterator { ptr: ptr }, database: database }
    }
  }
}

impl<'a,K: Key,V> LevelDBIterator for ValueIterator<'a,K,V> {
  #[inline]
  fn raw_iterator(&self) -> *mut leveldb_iterator_t {
    self.iter.ptr
  }

  #[inline]
  fn start(&self) -> bool {
    self.start
  }

  #[inline]
  fn started(&mut self) {
    self.start = false
  }
}

impl<'a,K: Key> ValueAccess<Vec<u8>> for ValueIterator<'a,K,Vec<u8>> {
  fn convert_value(&self, val: Vec<u8>) -> Vec<u8> {
    val
  }
}

impl<'a,K: Key> iter::Iterator<(K,Vec<u8>)> for Iterator<'a,K,Vec<u8>> {
  fn next(&mut self) -> Option<(K,Vec<u8>)> {
    if self.advance() {
      Some((self.key(), self.value()))
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<(K,Vec<u8>)> {
    self.seek_last();
    Some((self.key(), self.value()))
  }
}

impl<'a, K: Key> iter::Iterator<K> for KeyIterator<'a,K> {
  fn next(&mut self) -> Option<K> {
    if self.advance() {
      Some(self.key())
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<K> {
     self.seek_last();
     Some(self.key())
  }
}

impl<'a, K: Key> iter::Iterator<Vec<u8>> for ValueIterator<'a,K,Vec<u8>> {
  fn next(&mut self) -> Option<Vec<u8>> {
    if self.advance() {
      Some(self.value())
    } else {
      None
    }
  }

  fn last(&mut self) -> Option<Vec<u8>> {
     self.seek_last();
     Some(self.value())
  }
}
<|endoftext|>


<commit_before>extern crate cassandra;

use cassandra::*;

static CREATE_KEYSPACE:&'static str = "CREATE KEYSPACE examples WITH replication = { \'class\': \
                                       \'SimpleStrategy\', \'replication_factor\': \'3\' };";
static CREATE_TABLE:&'static str = "CREATE TABLE examples.basic (key text, bln boolean, flt float, \
                                    dbl double, i32 int, i64 bigint, PRIMARY KEY (key));";
static INSERT_QUERY:&'static str = "INSERT INTO examples.basic (key, bln, flt, dbl, i32, i64) \
                                    VALUES (?, ?, ?, ?, ?, ?);";
static SELECT_QUERY:&'static str = "SELECT * FROM examples.basic WHERE key = ?";

#[derive(Debug,PartialEq)]
struct Basic {
    bln: bool,
    flt: f32,
    dbl: f64,
    i32: i32,
    i64: i64,
}

fn insert_into_basic(session: &mut Session,
                     key: &str,
                     basic: &mut Basic)
                     -> Result<(), CassandraError> {
    println!("Creating statement");
    let mut statement = Statement::new(INSERT_QUERY, 6);
    statement
        .bind_string(0, key).unwrap()
        .bind_bool(1, basic.bln).unwrap()
        .bind_float(2, basic.flt).unwrap()
        .bind_double(3, basic.dbl).unwrap()
        .bind_int32(4, basic.i32).unwrap();
//        .bind_int64(5, basic.i64).unwrap();
    println!("Executing insert statement");
    try!(session.execute_statement(&statement).wait());
    Ok(())
}

fn select_from_basic(session: &mut Session,
                     prepared: &PreparedStatement,
                     key: &str,
                     basic: &mut Basic)
                     -> Result<(), CassandraError> {
    let mut statement = prepared.bind();
    try!(statement.bind_string(0, key));
    let mut future = session.execute_statement(&statement);
    match future.wait() {
        Ok(result) => {
            println!("{:?}", result);
            for row in result.iter() {
                basic.bln = try!(try!(row.get_column(1)).get_bool());
                basic.dbl = try!(try!(row.get_column(2)).get_double());
                basic.flt = try!(try!(row.get_column(3)).get_float());
                basic.i32 = try!(try!(row.get_column(4)).get_int32());
                basic.i64 = try!(try!(row.get_column(5)).get_int64());
            }
            Ok(())
        }
        Err(err) => panic!("{:?}",err),
    }
}

fn main() {
    let mut cluster = Cluster::new();
    cluster.set_contact_points("127.0.0.1").unwrap();
    cluster.set_protocol_version(3).unwrap();
    println!("Proto set");
    let mut session = cluster.connect().unwrap();

    println!("Connected");
    let mut input = Basic { bln: true, flt: 0.001f32, dbl: 0.0002f64, i32: 1, i64: 2 };
    let mut output = Basic { bln: false, flt: 0f32, dbl: 0f64, i32: 0, i64: 0 };
    println!("Executing create keyspace");
    session.execute(CREATE_KEYSPACE,0);
    println!("Creating table");
    session.execute(CREATE_TABLE,0);

    insert_into_basic(&mut session, "prepared_test", &mut input).unwrap();
    println!("Preparing");
    match session.prepare(SELECT_QUERY).unwrap().wait() {
        Ok(prepared) => {
            select_from_basic(&mut session, &prepared, "prepared_test", &mut output).unwrap();
            println!("input: {:?}\nouput: {:?}", input,output);
            assert_eq!(input,output);
        }
        Err(err) => panic!(err),
    }
    let close_future = session.close();
    close_future.wait().unwrap();
}
<commit_msg>prepared statement is working, not sure why it was broken<commit_after>extern crate cassandra;

use cassandra::*;

static CREATE_KEYSPACE:&'static str = "CREATE KEYSPACE examples WITH replication = { \'class\': \
                                       \'SimpleStrategy\', \'replication_factor\': \'3\' };";
static CREATE_TABLE:&'static str = "CREATE TABLE examples.basic (key text, bln boolean, flt float, \
                                    dbl double, i32 int, i64 bigint, PRIMARY KEY (key));";
static INSERT_QUERY:&'static str = "INSERT INTO examples.basic (key, bln, flt, dbl, i32, i64) \
                                    VALUES (?, ?, ?, ?, ?, ?);";
static SELECT_QUERY:&'static str = "SELECT * FROM examples.basic WHERE key = ?";

#[derive(Debug,PartialEq)]
struct Basic {
    bln: bool,
    flt: f32,
    dbl: f64,
    i32: i32,
    i64: i64,
}

fn insert_into_basic(session: &mut Session,
                     key: &str,
                     basic: &mut Basic)
                     -> Result<(), CassandraError> {
    println!("Creating statement");
    let mut statement = Statement::new(INSERT_QUERY, 6);
    statement
        .bind_string(0, key).unwrap()
        .bind_bool(1, basic.bln).unwrap()
        .bind_float(2, basic.flt).unwrap()
        .bind_double(3, basic.dbl).unwrap()
        .bind_int32(4, basic.i32).unwrap()
        .bind_int64(5, basic.i64).unwrap();
    println!("Executing insert statement");
    try!(session.execute_statement(&statement).wait());
    println!("Insert execute OK");
    Ok(())
}

fn select_from_basic(session: &mut Session,
                     prepared: &PreparedStatement,
                     key: &str,
                     basic: &mut Basic)
                     -> Result<(), CassandraError> {
    let mut statement = prepared.bind();
    try!(statement.bind_string(0, key));
    let mut future = session.execute_statement(&statement);
    match future.wait() {
        Ok(result) => {
            println!("{:?}", result);
            for row in result.iter() {
                basic.bln = try!(try!(row.get_column(1)).get_bool());
                basic.dbl = try!(try!(row.get_column(2)).get_double());
                basic.flt = try!(try!(row.get_column(3)).get_float());
                basic.i32 = try!(try!(row.get_column(4)).get_int32());
                basic.i64 = try!(try!(row.get_column(5)).get_int64());
            }
            Ok(())
        }
        Err(err) => panic!("{:?}",err),
    }
}

fn main() {
    let mut cluster = Cluster::new();
    cluster.set_contact_points("127.0.0.1").unwrap();
    cluster.set_protocol_version(3).unwrap();
    println!("Proto set");
    let mut session = cluster.connect().unwrap();

    println!("Connected");
    let mut input = Basic { bln: true, flt: 0.001f32, dbl: 0.0002f64, i32: 1, i64: 2 };
    let mut output = Basic { bln: false, flt: 0f32, dbl: 0f64, i32: 0, i64: 0 };
    println!("Executing create keyspace");
    session.execute(CREATE_KEYSPACE,0);
    println!("Creating table");
    session.execute(CREATE_TABLE,0);

    println!("Basic insertions");
    insert_into_basic(&mut session, "prepared_test", &mut input).unwrap();
    println!("Preparing");
    match session.prepare(SELECT_QUERY).unwrap().wait() {
        Ok(prepared) => {
            select_from_basic(&mut session, &prepared, "prepared_test", &mut output).unwrap();
            println!("input: {:?}\nouput: {:?}", input,output);
            assert_eq!(input,output);
        }
        Err(err) => panic!(err),
    }
    let close_future = session.close();
    close_future.wait().unwrap();
}
<|endoftext|>