<commit_before>//! Source map for translating source entity names to parsed entities.
//!
//! When the parser reads in a source file, entities like instructions, EBBs, and values get new
//! entity numbers. The parser maintains a mapping from the entity names in the source to the final
//! entity references.
//!
//! The `SourceMap` struct defined in this module makes the same mapping available to parser
//! clients.

use std::collections::HashMap;
use cretonne::ir::{StackSlot, JumpTable, Ebb, Value};
use cretonne::ir::entities::AnyEntity;
use error::{Result, Location};

/// Mapping from source entity names to entity references that are valid in the parsed function.
#[derive(Debug)]
pub struct SourceMap {
    values: HashMap<Value, Value>, // vNN, vxNN
    ebbs: HashMap<Ebb, Ebb>, // ebbNN
    stack_slots: HashMap<u32, StackSlot>, // ssNN
    jump_tables: HashMap<u32, JumpTable>, // jtNN
}

/// Read-only interface which is exposed outside the parser crate.
impl SourceMap {
    /// Look up a value entity by its source number.
    pub fn get_value(&self, src: Value) -> Option<Value> {
        self.values.get(&src).cloned()
    }

    /// Look up a EBB entity by its source number.
    pub fn get_ebb(&self, src: Ebb) -> Option<Ebb> {
        self.ebbs.get(&src).cloned()
    }

    /// Look up a stack slot entity by its source number.
    pub fn get_ss(&self, src_num: u32) -> Option<StackSlot> {
        self.stack_slots.get(&src_num).cloned()
    }

    /// Look up a jump table entity by its source number.
    pub fn get_jt(&self, src_num: u32) -> Option<JumpTable> {
        self.jump_tables.get(&src_num).cloned()
    }

    /// Look up an entity by source name.
    /// Returns the entity reference corresponding to `name`, if it exists.
    pub fn lookup_str(&self, name: &str) -> Option<AnyEntity> {
        split_entity_name(name).and_then(|(ent, num)| {
            match ent {
                "v" => {
                    Value::direct_with_number(num)
                        .and_then(|v| self.get_value(v))
                        .map(AnyEntity::Value)
                }
                "vx" => {
                    Value::table_with_number(num)
                        .and_then(|v| self.get_value(v))
                        .map(AnyEntity::Value)
                }
                "ebb" => Ebb::with_number(num).and_then(|e| self.get_ebb(e)).map(AnyEntity::Ebb),
                "ss" => self.get_ss(num).map(AnyEntity::StackSlot),
                "jt" => self.get_jt(num).map(AnyEntity::JumpTable),
                _ => None,
            }
        })
    }

    /// Rewrite an Ebb reference.
    pub fn rewrite_ebb(&self, ebb: &mut Ebb, loc: &Location) -> Result<()> {
        match self.get_ebb(*ebb) {
            Some(new) => {
                *ebb = new;
                Ok(())
            }
            None => err!(loc, "undefined reference: {}", ebb),
        }
    }

    /// Rewrite a value reference.
    pub fn rewrite_value(&self, val: &mut Value, loc: &Location) -> Result<()> {
        match self.get_value(*val) {
            Some(new) => {
                *val = new;
                Ok(())
            }
            None => err!(loc, "undefined reference: {}", val),
        }
    }

    /// Rewrite a slice of value references.
    pub fn rewrite_values(&self, vals: &mut [Value], loc: &Location) -> Result<()> {
        for val in vals {
            try!(self.rewrite_value(val, loc));
        }
        Ok(())
    }
}

/// Get the number of decimal digits at the end of `s`.
fn trailing_digits(s: &str) -> usize {
    // It's faster to iterate backwards over bytes, and we're only counting ASCII digits.
    s.as_bytes().iter().rev().cloned().take_while(|&b| b'0' <= b && b <= b'9').count()
}

/// Pre-parse a supposed entity name by splitting it into two parts: A head of lowercase ASCII
/// letters and numeric tail.
fn split_entity_name(name: &str) -> Option<(&str, u32)> {
    let (head, tail) = name.split_at(name.len() - trailing_digits(name));
    if tail.len() > 1 && tail.starts_with('0') {
        None
    } else {
        tail.parse().ok().map(|n| (head, n))
    }
}

/// Interface for mutating a source map.
///
/// This interface is provided for the parser itself, it is not made available outside the crate.
pub trait MutableSourceMap {
    fn new() -> Self;

    /// Define a value mapping from the source name `src` to the final `entity`.
    fn def_value(&mut self, src: Value, entity: Value, loc: &Location) -> Result<()>;
    fn def_ebb(&mut self, src: Ebb, entity: Ebb, loc: &Location) -> Result<()>;
    fn def_ss(&mut self, src_num: u32, entity: StackSlot, loc: &Location) -> Result<()>;
    fn def_jt(&mut self, src_num: u32, entity: JumpTable, loc: &Location) -> Result<()>;
}

impl MutableSourceMap for SourceMap {
    fn new() -> SourceMap {
        SourceMap {
            values: HashMap::new(),
            ebbs: HashMap::new(),
            stack_slots: HashMap::new(),
            jump_tables: HashMap::new(),
        }
    }

    fn def_value(&mut self, src: Value, entity: Value, loc: &Location) -> Result<()> {
        if self.values.insert(src, entity).is_some() {
            err!(loc, "duplicate value: {}", src)
        } else {
            Ok(())
        }
    }

    fn def_ebb(&mut self, src: Ebb, entity: Ebb, loc: &Location) -> Result<()> {
        if self.ebbs.insert(src, entity).is_some() {
            err!(loc, "duplicate EBB: {}", src)
        } else {
            Ok(())
        }
    }

    fn def_ss(&mut self, src_num: u32, entity: StackSlot, loc: &Location) -> Result<()> {
        if self.stack_slots.insert(src_num, entity).is_some() {
            err!(loc, "duplicate stack slot: ss{}", src_num)
        } else {
            Ok(())
        }
    }

    fn def_jt(&mut self, src_num: u32, entity: JumpTable, loc: &Location) -> Result<()> {
        if self.jump_tables.insert(src_num, entity).is_some() {
            err!(loc, "duplicate jump table: jt{}", src_num)
        } else {
            Ok(())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{trailing_digits, split_entity_name};
    use parse_test;

    #[test]
    fn digits() {
        assert_eq!(trailing_digits(""), 0);
        assert_eq!(trailing_digits("x"), 0);
        assert_eq!(trailing_digits("0x"), 0);
        assert_eq!(trailing_digits("x1"), 1);
        assert_eq!(trailing_digits("1x1"), 1);
        assert_eq!(trailing_digits("1x01"), 2);
    }

    #[test]
    fn entity_name() {
        assert_eq!(split_entity_name(""), None);
        assert_eq!(split_entity_name("x"), None);
        assert_eq!(split_entity_name("x+"), None);
        assert_eq!(split_entity_name("x+1"), Some(("x+", 1)));
        assert_eq!(split_entity_name("x-1"), Some(("x-", 1)));
        assert_eq!(split_entity_name("1"), Some(("", 1)));
        assert_eq!(split_entity_name("x1"), Some(("x", 1)));
        assert_eq!(split_entity_name("xy0"), Some(("xy", 0)));
        // Reject this non-canonical form.
        assert_eq!(split_entity_name("inst01"), None);
    }

    #[test]
    fn details() {
        let tf = parse_test("function detail() {
                               ss10 = stack_slot 13
                               jt10 = jump_table ebb0
                             ebb0(v4: i32, vx7: i32):
                               v10 = iadd v4, vx7
                             }")
            .unwrap();
        let map = &tf.functions[0].1.map;

        assert_eq!(map.lookup_str("v0"), None);
        assert_eq!(map.lookup_str("ss1"), None);
        assert_eq!(map.lookup_str("ss10").unwrap().to_string(), "ss0");
        assert_eq!(map.lookup_str("jt10").unwrap().to_string(), "jt0");
        assert_eq!(map.lookup_str("ebb0").unwrap().to_string(), "ebb0");
        assert_eq!(map.lookup_str("v4").unwrap().to_string(), "vx0");
        assert_eq!(map.lookup_str("vx7").unwrap().to_string(), "vx1");
        assert_eq!(map.lookup_str("v10").unwrap().to_string(), "v0");
    }
}
<commit_msg>Also record locations for tracked entities.<commit_after>//! Source map for translating source entity names to parsed entities.
//!
//! When the parser reads in a source file, entities like instructions, EBBs, and values get new
//! entity numbers. The parser maintains a mapping from the entity names in the source to the final
//! entity references.
//!
//! The `SourceMap` struct defined in this module makes the same mapping available to parser
//! clients.

use std::collections::HashMap;
use cretonne::ir::{StackSlot, JumpTable, Ebb, Value, Inst};
use cretonne::ir::entities::AnyEntity;
use error::{Result, Location};

/// Mapping from source entity names to entity references that are valid in the parsed function.
#[derive(Debug)]
pub struct SourceMap {
    values: HashMap<Value, Value>, // vNN, vxNN
    ebbs: HashMap<Ebb, Ebb>, // ebbNN
    stack_slots: HashMap<u32, StackSlot>, // ssNN
    jump_tables: HashMap<u32, JumpTable>, // jtNN

    // Store locations for entities, including instructions.
    locations: HashMap<AnyEntity, Location>,
}

/// Read-only interface which is exposed outside the parser crate.
impl SourceMap {
    /// Look up a value entity by its source number.
    pub fn get_value(&self, src: Value) -> Option<Value> {
        self.values.get(&src).cloned()
    }

    /// Look up a EBB entity by its source number.
    pub fn get_ebb(&self, src: Ebb) -> Option<Ebb> {
        self.ebbs.get(&src).cloned()
    }

    /// Look up a stack slot entity by its source number.
    pub fn get_ss(&self, src_num: u32) -> Option<StackSlot> {
        self.stack_slots.get(&src_num).cloned()
    }

    /// Look up a jump table entity by its source number.
    pub fn get_jt(&self, src_num: u32) -> Option<JumpTable> {
        self.jump_tables.get(&src_num).cloned()
    }

    /// Look up an entity by source name.
    /// Returns the entity reference corresponding to `name`, if it exists.
    pub fn lookup_str(&self, name: &str) -> Option<AnyEntity> {
        split_entity_name(name).and_then(|(ent, num)| {
            match ent {
                "v" => {
                    Value::direct_with_number(num)
                        .and_then(|v| self.get_value(v))
                        .map(AnyEntity::Value)
                }
                "vx" => {
                    Value::table_with_number(num)
                        .and_then(|v| self.get_value(v))
                        .map(AnyEntity::Value)
                }
                "ebb" => Ebb::with_number(num).and_then(|e| self.get_ebb(e)).map(AnyEntity::Ebb),
                "ss" => self.get_ss(num).map(AnyEntity::StackSlot),
                "jt" => self.get_jt(num).map(AnyEntity::JumpTable),
                _ => None,
            }
        })
    }

    /// Get the source location where an entity was defined.
    /// This looks up entities in the parsed function, not the source entity numbers.
    pub fn location(&self, entity: AnyEntity) -> Option<Location> {
        self.locations.get(&entity).cloned()
    }

    /// Rewrite an Ebb reference.
    pub fn rewrite_ebb(&self, ebb: &mut Ebb, loc: &Location) -> Result<()> {
        match self.get_ebb(*ebb) {
            Some(new) => {
                *ebb = new;
                Ok(())
            }
            None => err!(loc, "undefined reference: {}", ebb),
        }
    }

    /// Rewrite a value reference.
    pub fn rewrite_value(&self, val: &mut Value, loc: &Location) -> Result<()> {
        match self.get_value(*val) {
            Some(new) => {
                *val = new;
                Ok(())
            }
            None => err!(loc, "undefined reference: {}", val),
        }
    }

    /// Rewrite a slice of value references.
    pub fn rewrite_values(&self, vals: &mut [Value], loc: &Location) -> Result<()> {
        for val in vals {
            try!(self.rewrite_value(val, loc));
        }
        Ok(())
    }
}

/// Get the number of decimal digits at the end of `s`.
fn trailing_digits(s: &str) -> usize {
    // It's faster to iterate backwards over bytes, and we're only counting ASCII digits.
    s.as_bytes().iter().rev().cloned().take_while(|&b| b'0' <= b && b <= b'9').count()
}

/// Pre-parse a supposed entity name by splitting it into two parts: A head of lowercase ASCII
/// letters and numeric tail.
fn split_entity_name(name: &str) -> Option<(&str, u32)> {
    let (head, tail) = name.split_at(name.len() - trailing_digits(name));
    if tail.len() > 1 && tail.starts_with('0') {
        None
    } else {
        tail.parse().ok().map(|n| (head, n))
    }
}

/// Interface for mutating a source map.
///
/// This interface is provided for the parser itself, it is not made available outside the crate.
pub trait MutableSourceMap {
    fn new() -> Self;

    /// Define a value mapping from the source name `src` to the final `entity`.
    fn def_value(&mut self, src: Value, entity: Value, loc: &Location) -> Result<()>;
    fn def_ebb(&mut self, src: Ebb, entity: Ebb, loc: &Location) -> Result<()>;
    fn def_ss(&mut self, src_num: u32, entity: StackSlot, loc: &Location) -> Result<()>;
    fn def_jt(&mut self, src_num: u32, entity: JumpTable, loc: &Location) -> Result<()>;

    /// Define an instruction. Since instruction numbers never appear in source, only the location
    /// is recorded.
    fn def_inst(&mut self, entity: Inst, loc: &Location) -> Result<()>;
}

impl MutableSourceMap for SourceMap {
    fn new() -> SourceMap {
        SourceMap {
            values: HashMap::new(),
            ebbs: HashMap::new(),
            stack_slots: HashMap::new(),
            jump_tables: HashMap::new(),
            locations: HashMap::new(),
        }
    }

    fn def_value(&mut self, src: Value, entity: Value, loc: &Location) -> Result<()> {
        if self.values.insert(src, entity).is_some() {
            err!(loc, "duplicate value: {}", src)
        } else if self.locations.insert(entity.into(), loc.clone()).is_some() {
            err!(loc, "duplicate entity: {}", entity)
        } else {
            Ok(())
        }
    }

    fn def_ebb(&mut self, src: Ebb, entity: Ebb, loc: &Location) -> Result<()> {
        if self.ebbs.insert(src, entity).is_some() {
            err!(loc, "duplicate EBB: {}", src)
        } else if self.locations.insert(entity.into(), loc.clone()).is_some() {
            err!(loc, "duplicate entity: {}", entity)
        } else {
            Ok(())
        }
    }

    fn def_ss(&mut self, src_num: u32, entity: StackSlot, loc: &Location) -> Result<()> {
        if self.stack_slots.insert(src_num, entity).is_some() {
            err!(loc, "duplicate stack slot: ss{}", src_num)
        } else if self.locations.insert(entity.into(), loc.clone()).is_some() {
            err!(loc, "duplicate entity: {}", entity)
        } else {
            Ok(())
        }
    }

    fn def_jt(&mut self, src_num: u32, entity: JumpTable, loc: &Location) -> Result<()> {
        if self.jump_tables.insert(src_num, entity).is_some() {
            err!(loc, "duplicate jump table: jt{}", src_num)
        } else if self.locations.insert(entity.into(), loc.clone()).is_some() {
            err!(loc, "duplicate entity: {}", entity)
        } else {
            Ok(())
        }
    }

    fn def_inst(&mut self, entity: Inst, loc: &Location) -> Result<()> {
        if self.locations.insert(entity.into(), loc.clone()).is_some() {
            err!(loc, "duplicate entity: {}", entity)
        } else {
            Ok(())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{trailing_digits, split_entity_name};
    use parse_test;

    #[test]
    fn digits() {
        assert_eq!(trailing_digits(""), 0);
        assert_eq!(trailing_digits("x"), 0);
        assert_eq!(trailing_digits("0x"), 0);
        assert_eq!(trailing_digits("x1"), 1);
        assert_eq!(trailing_digits("1x1"), 1);
        assert_eq!(trailing_digits("1x01"), 2);
    }

    #[test]
    fn entity_name() {
        assert_eq!(split_entity_name(""), None);
        assert_eq!(split_entity_name("x"), None);
        assert_eq!(split_entity_name("x+"), None);
        assert_eq!(split_entity_name("x+1"), Some(("x+", 1)));
        assert_eq!(split_entity_name("x-1"), Some(("x-", 1)));
        assert_eq!(split_entity_name("1"), Some(("", 1)));
        assert_eq!(split_entity_name("x1"), Some(("x", 1)));
        assert_eq!(split_entity_name("xy0"), Some(("xy", 0)));
        // Reject this non-canonical form.
        assert_eq!(split_entity_name("inst01"), None);
    }

    #[test]
    fn details() {
        let tf = parse_test("function detail() {
                               ss10 = stack_slot 13
                               jt10 = jump_table ebb0
                             ebb0(v4: i32, vx7: i32):
                               v10 = iadd v4, vx7
                             }")
            .unwrap();
        let map = &tf.functions[0].1.map;

        assert_eq!(map.lookup_str("v0"), None);
        assert_eq!(map.lookup_str("ss1"), None);
        assert_eq!(map.lookup_str("ss10").unwrap().to_string(), "ss0");
        assert_eq!(map.lookup_str("jt10").unwrap().to_string(), "jt0");
        assert_eq!(map.lookup_str("ebb0").unwrap().to_string(), "ebb0");
        assert_eq!(map.lookup_str("v4").unwrap().to_string(), "vx0");
        assert_eq!(map.lookup_str("vx7").unwrap().to_string(), "vx1");
        assert_eq!(map.lookup_str("v10").unwrap().to_string(), "v0");
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Access to a single thread-local pointer.
//!
//! The runtime will use this for storing ~Task.
//!
//! XXX: Add runtime checks for usage of inconsistent pointer types.
//! and for overwriting an existing pointer.

use libc::c_void;
use cast;
use ptr;
use cell::Cell;
use option::{Option, Some, None};
use unstable::finally::Finally;
use tls = rt::thread_local_storage;

/// Initialize the TLS key. Other ops will fail if this isn't executed first.
pub fn init_tls_key() {
    unsafe {
        rust_initialize_rt_tls_key();
        extern {
            fn rust_initialize_rt_tls_key();
        }
    }
}

/// Give a pointer to thread-local storage.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn put<T>(sched: ~T) {
    let key = tls_key();
    let void_ptr: *mut c_void = cast::transmute(sched);
    tls::set(key, void_ptr);
}

/// Take ownership of a pointer from thread-local storage.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn take<T>() -> ~T {
    let key = tls_key();
    let void_ptr: *mut c_void = tls::get(key);
    rtassert!(void_ptr.is_not_null());
    let ptr: ~T = cast::transmute(void_ptr);
    tls::set(key, ptr::mut_null());
    return ptr;
}

/// Check whether there is a thread-local pointer installed.
pub fn exists() -> bool {
    unsafe {
        match maybe_tls_key() {
            Some(key) => tls::get(key).is_not_null(),
            None => false
        }
    }
}

/// Borrow the thread-local scheduler from thread-local storage.
/// While the scheduler is borrowed it is not available in TLS.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn borrow<T>(f: &fn(&mut T)) {
    let mut value = take();

    // XXX: Need a different abstraction from 'finally' here to avoid unsafety
    let unsafe_ptr = cast::transmute_mut_region(&mut *value);
    let value_cell = Cell::new(value);

    do (|| {
        f(unsafe_ptr);
    }).finally {
        put(value_cell.take());
    }
}

/// Borrow a mutable reference to the thread-local Scheduler
///
/// # Safety Note
///
/// Because this leaves the Scheduler in thread-local storage it is possible
/// For the Scheduler pointer to be aliased
pub unsafe fn unsafe_borrow<T>() -> *mut T {
    let key = tls_key();
    let mut void_sched: *mut c_void = tls::get(key);
    rtassert!(void_sched.is_not_null());
    {
        let sched: *mut *mut c_void = &mut void_sched;
        let sched: *mut ~T = sched as *mut ~T;
        let sched: *mut T = &mut **sched;
        return sched;
    }
}

fn tls_key() -> tls::Key {
    match maybe_tls_key() {
        Some(key) => key,
        None => rtabort!("runtime tls key not initialized")
    }
}

fn maybe_tls_key() -> Option<tls::Key> {
    unsafe {
        let key: *mut c_void = rust_get_rt_tls_key();
        let key: &mut tls::Key = cast::transmute(key);
        let key = *key;
        // Check that the key has been initialized.

        // NB: This is a little racy because, while the key is
        // initalized under a mutex and it's assumed to be initalized
        // in the Scheduler ctor by any thread that needs to use it,
        // we are not accessing the key under a mutex.  Threads that
        // are not using the new Scheduler but still *want to check*
        // whether they are running under a new Scheduler may see a 0
        // value here that is in the process of being initialized in
        // another thread. I think this is fine since the only action
        // they could take if it was initialized would be to check the
        // thread-local value and see that it's not set.
        if key != -1 {
            return Some(key);
        } else {
            return None;
        }
    }

    extern {
        #[fast_ffi]
        fn rust_get_rt_tls_key() -> *mut c_void;
    }

}
<commit_msg>std::rt: Improve the error message when the thread-local ptr is null<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Access to a single thread-local pointer.
//!
//! The runtime will use this for storing ~Task.
//!
//! XXX: Add runtime checks for usage of inconsistent pointer types.
//! and for overwriting an existing pointer.

use libc::c_void;
use cast;
use ptr;
use cell::Cell;
use option::{Option, Some, None};
use unstable::finally::Finally;
use tls = rt::thread_local_storage;

/// Initialize the TLS key. Other ops will fail if this isn't executed first.
pub fn init_tls_key() {
    unsafe {
        rust_initialize_rt_tls_key();
        extern {
            fn rust_initialize_rt_tls_key();
        }
    }
}

/// Give a pointer to thread-local storage.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn put<T>(sched: ~T) {
    let key = tls_key();
    let void_ptr: *mut c_void = cast::transmute(sched);
    tls::set(key, void_ptr);
}

/// Take ownership of a pointer from thread-local storage.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn take<T>() -> ~T {
    let key = tls_key();
    let void_ptr: *mut c_void = tls::get(key);
    if void_ptr.is_null() {
        rtabort!("thread-local pointer is null. bogus!");
    }
    let ptr: ~T = cast::transmute(void_ptr);
    tls::set(key, ptr::mut_null());
    return ptr;
}

/// Check whether there is a thread-local pointer installed.
pub fn exists() -> bool {
    unsafe {
        match maybe_tls_key() {
            Some(key) => tls::get(key).is_not_null(),
            None => false
        }
    }
}

/// Borrow the thread-local value from thread-local storage.
/// While the value is borrowed it is not available in TLS.
///
/// # Safety note
///
/// Does not validate the pointer type.
pub unsafe fn borrow<T>(f: &fn(&mut T)) {
    let mut value = take();

    // XXX: Need a different abstraction from 'finally' here to avoid unsafety
    let unsafe_ptr = cast::transmute_mut_region(&mut *value);
    let value_cell = Cell::new(value);

    do (|| {
        f(unsafe_ptr);
    }).finally {
        put(value_cell.take());
    }
}

/// Borrow a mutable reference to the thread-local value
///
/// # Safety Note
///
/// Because this leaves the value in thread-local storage it is possible
/// For the Scheduler pointer to be aliased
pub unsafe fn unsafe_borrow<T>() -> *mut T {
    let key = tls_key();
    let mut void_ptr: *mut c_void = tls::get(key);
    if void_ptr.is_null() {
        rtabort!("thread-local pointer is null. bogus!");
    }
    {
        let ptr: *mut *mut c_void = &mut void_ptr;
        let ptr: *mut ~T = ptr as *mut ~T;
        let ptr: *mut T = &mut **ptr;
        return ptr;
    }
}

fn tls_key() -> tls::Key {
    match maybe_tls_key() {
        Some(key) => key,
        None => rtabort!("runtime tls key not initialized")
    }
}

fn maybe_tls_key() -> Option<tls::Key> {
    unsafe {
        let key: *mut c_void = rust_get_rt_tls_key();
        let key: &mut tls::Key = cast::transmute(key);
        let key = *key;
        // Check that the key has been initialized.

        // NB: This is a little racy because, while the key is
        // initalized under a mutex and it's assumed to be initalized
        // in the Scheduler ctor by any thread that needs to use it,
        // we are not accessing the key under a mutex.  Threads that
        // are not using the new Scheduler but still *want to check*
        // whether they are running under a new Scheduler may see a 0
        // value here that is in the process of being initialized in
        // another thread. I think this is fine since the only action
        // they could take if it was initialized would be to check the
        // thread-local value and see that it's not set.
        if key != -1 {
            return Some(key);
        } else {
            return None;
        }
    }

    extern {
        #[fast_ffi]
        fn rust_get_rt_tls_key() -> *mut c_void;
    }

}
<|endoftext|>


<commit_before>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use ffi::CStr;
use io;
use libc::{self, c_int, size_t, sockaddr, socklen_t};
use net::{SocketAddr, Shutdown};
use str;
use sys::fd::FileDesc;
use sys_common::{AsInner, FromInner, IntoInner};
use sys_common::net::{getsockopt, setsockopt};
use time::Duration;

pub use sys::{cvt, cvt_r};
pub extern crate libc as netc;

pub type wrlen_t = size_t;

// See below for the usage of SOCK_CLOEXEC, but this constant is only defined on
// Linux currently (e.g. support doesn't exist on other platforms). In order to
// get name resolution to work and things to compile we just define a dummy
// SOCK_CLOEXEC here for other platforms. Note that the dummy constant isn't
// actually ever used (the blocks below are wrapped in `if cfg!` as well.
#[cfg(target_os = "linux")]
use libc::SOCK_CLOEXEC;
#[cfg(not(target_os = "linux"))]
const SOCK_CLOEXEC: c_int = 0;

pub struct Socket(FileDesc);

pub fn init() {}

pub fn cvt_gai(err: c_int) -> io::Result<()> {
    if err == 0 { return Ok(()) }

    let detail = unsafe {
        str::from_utf8(CStr::from_ptr(libc::gai_strerror(err)).to_bytes()).unwrap()
            .to_owned()
    };
    Err(io::Error::new(io::ErrorKind::Other,
                       &format!("failed to lookup address information: {}",
                                detail)[..]))
}

impl Socket {
    pub fn new(addr: &SocketAddr, ty: c_int) -> io::Result<Socket> {
        let fam = match *addr {
            SocketAddr::V4(..) => libc::AF_INET,
            SocketAddr::V6(..) => libc::AF_INET6,
        };
        Socket::new_raw(fam, ty)
    }

    pub fn new_raw(fam: c_int, ty: c_int) -> io::Result<Socket> {
        unsafe {
            // On linux we first attempt to pass the SOCK_CLOEXEC flag to
            // atomically create the socket and set it as CLOEXEC. Support for
            // this option, however, was added in 2.6.27, and we still support
            // 2.6.18 as a kernel, so if the returned error is EINVAL we
            // fallthrough to the fallback.
            if cfg!(target_os = "linux") {
                match cvt(libc::socket(fam, ty | SOCK_CLOEXEC, 0)) {
                    Ok(fd) => return Ok(Socket(FileDesc::new(fd))),
                    Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {}
                    Err(e) => return Err(e),
                }
            }

            let fd = cvt(libc::socket(fam, ty, 0))?;
            let fd = FileDesc::new(fd);
            fd.set_cloexec()?;
            Ok(Socket(fd))
        }
    }

    pub fn new_pair(fam: c_int, ty: c_int) -> io::Result<(Socket, Socket)> {
        unsafe {
            let mut fds = [0, 0];

            // Like above, see if we can set cloexec atomically
            if cfg!(target_os = "linux") {
                match cvt(libc::socketpair(fam, ty | SOCK_CLOEXEC, 0, fds.as_mut_ptr())) {
                    Ok(_) => {
                        return Ok((Socket(FileDesc::new(fds[0])), Socket(FileDesc::new(fds[1]))));
                    }
                    Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {},
                    Err(e) => return Err(e),
                }
            }

            cvt(libc::socketpair(fam, ty, 0, fds.as_mut_ptr()))?;
            let a = FileDesc::new(fds[0]);
            let b = FileDesc::new(fds[1]);
            a.set_cloexec()?;
            b.set_cloexec()?;
            Ok((Socket(a), Socket(b)))
        }
    }

    pub fn accept(&self, storage: *mut sockaddr, len: *mut socklen_t)
                  -> io::Result<Socket> {
        // Unfortunately the only known way right now to accept a socket and
        // atomically set the CLOEXEC flag is to use the `accept4` syscall on
        // Linux. This was added in 2.6.28, however, and because we support
        // 2.6.18 we must detect this support dynamically.
        if cfg!(target_os = "linux") {
            weak! {
                fn accept4(c_int, *mut sockaddr, *mut socklen_t, c_int) -> c_int
            }
            if let Some(accept) = accept4.get() {
                let res = cvt_r(|| unsafe {
                    accept(self.0.raw(), storage, len, SOCK_CLOEXEC)
                });
                match res {
                    Ok(fd) => return Ok(Socket(FileDesc::new(fd))),
                    Err(ref e) if e.raw_os_error() == Some(libc::ENOSYS) => {}
                    Err(e) => return Err(e),
                }
            }
        }

        let fd = cvt_r(|| unsafe {
            libc::accept(self.0.raw(), storage, len)
        })?;
        let fd = FileDesc::new(fd);
        fd.set_cloexec()?;
        Ok(Socket(fd))
    }

    pub fn duplicate(&self) -> io::Result<Socket> {
        self.0.duplicate().map(Socket)
    }

    pub fn read(&self, buf: &mut [u8]) -> io::Result<usize> {
        self.0.read(buf)
    }

    pub fn read_to_end(&self, buf: &mut Vec<u8>) -> io::Result<usize> {
        self.0.read_to_end(buf)
    }

    pub fn write(&self, buf: &[u8]) -> io::Result<usize> {
        self.0.write(buf)
    }

    pub fn set_timeout(&self, dur: Option<Duration>, kind: libc::c_int) -> io::Result<()> {
        let timeout = match dur {
            Some(dur) => {
                if dur.as_secs() == 0 && dur.subsec_nanos() == 0 {
                    return Err(io::Error::new(io::ErrorKind::InvalidInput,
                                              "cannot set a 0 duration timeout"));
                }

                let secs = if dur.as_secs() > libc::time_t::max_value() as u64 {
                    libc::time_t::max_value()
                } else {
                    dur.as_secs() as libc::time_t
                };
                let mut timeout = libc::timeval {
                    tv_sec: secs,
                    tv_usec: (dur.subsec_nanos() / 1000) as libc::suseconds_t,
                };
                if timeout.tv_sec == 0 && timeout.tv_usec == 0 {
                    timeout.tv_usec = 1;
                }
                timeout
            }
            None => {
                libc::timeval {
                    tv_sec: 0,
                    tv_usec: 0,
                }
            }
        };
        setsockopt(self, libc::SOL_SOCKET, kind, timeout)
    }

    pub fn timeout(&self, kind: libc::c_int) -> io::Result<Option<Duration>> {
        let raw: libc::timeval = getsockopt(self, libc::SOL_SOCKET, kind)?;
        if raw.tv_sec == 0 && raw.tv_usec == 0 {
            Ok(None)
        } else {
            let sec = raw.tv_sec as u64;
            let nsec = (raw.tv_usec as u32) * 1000;
            Ok(Some(Duration::new(sec, nsec)))
        }
    }

    pub fn shutdown(&self, how: Shutdown) -> io::Result<()> {
        let how = match how {
            Shutdown::Write => libc::SHUT_WR,
            Shutdown::Read => libc::SHUT_RD,
            Shutdown::Both => libc::SHUT_RDWR,
        };
        cvt(unsafe { libc::shutdown(self.0.raw(), how) })?;
        Ok(())
    }

    pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> {
        setsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY, nodelay as c_int)
    }

    pub fn nodelay(&self) -> io::Result<bool> {
        let raw: c_int = getsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY)?;
        Ok(raw != 0)
    }

    pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> {
        let mut nonblocking = nonblocking as libc::c_ulong;
        cvt(unsafe { libc::ioctl(*self.as_inner(), libc::FIONBIO, &mut nonblocking) }).map(|_| ())
    }

    pub fn take_error(&self) -> io::Result<Option<io::Error>> {
        let raw: c_int = getsockopt(self, libc::SOL_SOCKET, libc::SO_ERROR)?;
        if raw == 0 {
            Ok(None)
        } else {
            Ok(Some(io::Error::from_raw_os_error(raw as i32)))
        }
    }
}

impl AsInner<c_int> for Socket {
    fn as_inner(&self) -> &c_int { self.0.as_inner() }
}

impl FromInner<c_int> for Socket {
    fn from_inner(fd: c_int) -> Socket { Socket(FileDesc::new(fd)) }
}

impl IntoInner<c_int> for Socket {
    fn into_inner(self) -> c_int { self.0.into_raw() }
}
<commit_msg>Auto merge of #36322 - uweigand:nonblocking, r=alexcrichton<commit_after>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use ffi::CStr;
use io;
use libc::{self, c_int, size_t, sockaddr, socklen_t};
use net::{SocketAddr, Shutdown};
use str;
use sys::fd::FileDesc;
use sys_common::{AsInner, FromInner, IntoInner};
use sys_common::net::{getsockopt, setsockopt};
use time::Duration;

pub use sys::{cvt, cvt_r};
pub extern crate libc as netc;

pub type wrlen_t = size_t;

// See below for the usage of SOCK_CLOEXEC, but this constant is only defined on
// Linux currently (e.g. support doesn't exist on other platforms). In order to
// get name resolution to work and things to compile we just define a dummy
// SOCK_CLOEXEC here for other platforms. Note that the dummy constant isn't
// actually ever used (the blocks below are wrapped in `if cfg!` as well.
#[cfg(target_os = "linux")]
use libc::SOCK_CLOEXEC;
#[cfg(not(target_os = "linux"))]
const SOCK_CLOEXEC: c_int = 0;

pub struct Socket(FileDesc);

pub fn init() {}

pub fn cvt_gai(err: c_int) -> io::Result<()> {
    if err == 0 { return Ok(()) }

    let detail = unsafe {
        str::from_utf8(CStr::from_ptr(libc::gai_strerror(err)).to_bytes()).unwrap()
            .to_owned()
    };
    Err(io::Error::new(io::ErrorKind::Other,
                       &format!("failed to lookup address information: {}",
                                detail)[..]))
}

impl Socket {
    pub fn new(addr: &SocketAddr, ty: c_int) -> io::Result<Socket> {
        let fam = match *addr {
            SocketAddr::V4(..) => libc::AF_INET,
            SocketAddr::V6(..) => libc::AF_INET6,
        };
        Socket::new_raw(fam, ty)
    }

    pub fn new_raw(fam: c_int, ty: c_int) -> io::Result<Socket> {
        unsafe {
            // On linux we first attempt to pass the SOCK_CLOEXEC flag to
            // atomically create the socket and set it as CLOEXEC. Support for
            // this option, however, was added in 2.6.27, and we still support
            // 2.6.18 as a kernel, so if the returned error is EINVAL we
            // fallthrough to the fallback.
            if cfg!(target_os = "linux") {
                match cvt(libc::socket(fam, ty | SOCK_CLOEXEC, 0)) {
                    Ok(fd) => return Ok(Socket(FileDesc::new(fd))),
                    Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {}
                    Err(e) => return Err(e),
                }
            }

            let fd = cvt(libc::socket(fam, ty, 0))?;
            let fd = FileDesc::new(fd);
            fd.set_cloexec()?;
            Ok(Socket(fd))
        }
    }

    pub fn new_pair(fam: c_int, ty: c_int) -> io::Result<(Socket, Socket)> {
        unsafe {
            let mut fds = [0, 0];

            // Like above, see if we can set cloexec atomically
            if cfg!(target_os = "linux") {
                match cvt(libc::socketpair(fam, ty | SOCK_CLOEXEC, 0, fds.as_mut_ptr())) {
                    Ok(_) => {
                        return Ok((Socket(FileDesc::new(fds[0])), Socket(FileDesc::new(fds[1]))));
                    }
                    Err(ref e) if e.raw_os_error() == Some(libc::EINVAL) => {},
                    Err(e) => return Err(e),
                }
            }

            cvt(libc::socketpair(fam, ty, 0, fds.as_mut_ptr()))?;
            let a = FileDesc::new(fds[0]);
            let b = FileDesc::new(fds[1]);
            a.set_cloexec()?;
            b.set_cloexec()?;
            Ok((Socket(a), Socket(b)))
        }
    }

    pub fn accept(&self, storage: *mut sockaddr, len: *mut socklen_t)
                  -> io::Result<Socket> {
        // Unfortunately the only known way right now to accept a socket and
        // atomically set the CLOEXEC flag is to use the `accept4` syscall on
        // Linux. This was added in 2.6.28, however, and because we support
        // 2.6.18 we must detect this support dynamically.
        if cfg!(target_os = "linux") {
            weak! {
                fn accept4(c_int, *mut sockaddr, *mut socklen_t, c_int) -> c_int
            }
            if let Some(accept) = accept4.get() {
                let res = cvt_r(|| unsafe {
                    accept(self.0.raw(), storage, len, SOCK_CLOEXEC)
                });
                match res {
                    Ok(fd) => return Ok(Socket(FileDesc::new(fd))),
                    Err(ref e) if e.raw_os_error() == Some(libc::ENOSYS) => {}
                    Err(e) => return Err(e),
                }
            }
        }

        let fd = cvt_r(|| unsafe {
            libc::accept(self.0.raw(), storage, len)
        })?;
        let fd = FileDesc::new(fd);
        fd.set_cloexec()?;
        Ok(Socket(fd))
    }

    pub fn duplicate(&self) -> io::Result<Socket> {
        self.0.duplicate().map(Socket)
    }

    pub fn read(&self, buf: &mut [u8]) -> io::Result<usize> {
        self.0.read(buf)
    }

    pub fn read_to_end(&self, buf: &mut Vec<u8>) -> io::Result<usize> {
        self.0.read_to_end(buf)
    }

    pub fn write(&self, buf: &[u8]) -> io::Result<usize> {
        self.0.write(buf)
    }

    pub fn set_timeout(&self, dur: Option<Duration>, kind: libc::c_int) -> io::Result<()> {
        let timeout = match dur {
            Some(dur) => {
                if dur.as_secs() == 0 && dur.subsec_nanos() == 0 {
                    return Err(io::Error::new(io::ErrorKind::InvalidInput,
                                              "cannot set a 0 duration timeout"));
                }

                let secs = if dur.as_secs() > libc::time_t::max_value() as u64 {
                    libc::time_t::max_value()
                } else {
                    dur.as_secs() as libc::time_t
                };
                let mut timeout = libc::timeval {
                    tv_sec: secs,
                    tv_usec: (dur.subsec_nanos() / 1000) as libc::suseconds_t,
                };
                if timeout.tv_sec == 0 && timeout.tv_usec == 0 {
                    timeout.tv_usec = 1;
                }
                timeout
            }
            None => {
                libc::timeval {
                    tv_sec: 0,
                    tv_usec: 0,
                }
            }
        };
        setsockopt(self, libc::SOL_SOCKET, kind, timeout)
    }

    pub fn timeout(&self, kind: libc::c_int) -> io::Result<Option<Duration>> {
        let raw: libc::timeval = getsockopt(self, libc::SOL_SOCKET, kind)?;
        if raw.tv_sec == 0 && raw.tv_usec == 0 {
            Ok(None)
        } else {
            let sec = raw.tv_sec as u64;
            let nsec = (raw.tv_usec as u32) * 1000;
            Ok(Some(Duration::new(sec, nsec)))
        }
    }

    pub fn shutdown(&self, how: Shutdown) -> io::Result<()> {
        let how = match how {
            Shutdown::Write => libc::SHUT_WR,
            Shutdown::Read => libc::SHUT_RD,
            Shutdown::Both => libc::SHUT_RDWR,
        };
        cvt(unsafe { libc::shutdown(self.0.raw(), how) })?;
        Ok(())
    }

    pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> {
        setsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY, nodelay as c_int)
    }

    pub fn nodelay(&self) -> io::Result<bool> {
        let raw: c_int = getsockopt(self, libc::IPPROTO_TCP, libc::TCP_NODELAY)?;
        Ok(raw != 0)
    }

    pub fn set_nonblocking(&self, nonblocking: bool) -> io::Result<()> {
        let mut nonblocking = nonblocking as libc::c_int;
        cvt(unsafe { libc::ioctl(*self.as_inner(), libc::FIONBIO, &mut nonblocking) }).map(|_| ())
    }

    pub fn take_error(&self) -> io::Result<Option<io::Error>> {
        let raw: c_int = getsockopt(self, libc::SOL_SOCKET, libc::SO_ERROR)?;
        if raw == 0 {
            Ok(None)
        } else {
            Ok(Some(io::Error::from_raw_os_error(raw as i32)))
        }
    }
}

impl AsInner<c_int> for Socket {
    fn as_inner(&self) -> &c_int { self.0.as_inner() }
}

impl FromInner<c_int> for Socket {
    fn from_inner(fd: c_int) -> Socket { Socket(FileDesc::new(fd)) }
}

impl IntoInner<c_int> for Socket {
    fn into_inner(self) -> c_int { self.0.into_raw() }
}
<|endoftext|>


<commit_before>use dispatcher::request::{Request, RequestHandle};
use context::ContextMap;
use context::context_map::StreamingIterator;
use reactor;

pub struct MessageEventHandler;

impl MessageEventHandler {
    pub fn new() -> MessageEventHandler {
        MessageEventHandler
    }
}

impl reactor::EventHandler<Request, ContextMap> for MessageEventHandler {
    fn handle_event(&mut self, event: Request, data: &mut ContextMap) {
        trace!("MessageEventHandler: handle_event()");
        if let Request::Message(event) = event {
            for i in event.ids() {
                let mut iter = data.contexts_iter_mut(i);
                while let Some(context) = iter.next() {
                    context.on_event(Request::Message(event.clone()));
                }
            }
        } else {
            unreachable!("MessageEventHandler should only handle Message events");
        }
    }
    fn handle(&self) -> RequestHandle {
        RequestHandle::Message
    }
}
<commit_msg>MessageEventHandler: use higher-ranked lifetime<commit_after>use dispatcher::request::{Request, RequestHandle};
use context::context_map::StreamingIterator;
use reactor::{EventHandler, SharedData};

pub struct MessageEventHandler;

impl MessageEventHandler {
    pub fn new() -> MessageEventHandler {
        MessageEventHandler
    }
}

impl<'a> EventHandler<Request, SharedData<'a>> for MessageEventHandler {
    fn handle_event(&mut self, event: Request, data: &mut SharedData) {
        trace!("MessageEventHandler: handle_event()");
        if let Request::Message(event) = event {
            for i in event.ids() {
                let mut iter = data.map.contexts_iter_mut(i);
                while let Some(context) = iter.next() {
                    context.on_event(Request::Message(event.clone()));
                }
            }
        } else {
            unreachable!("MessageEventHandler should only handle Message events");
        }
    }
    fn handle(&self) -> RequestHandle {
        RequestHandle::Message
    }
}
<|endoftext|>


<commit_before>
extern crate num;
use num::complex::Complex;

#[derive(Debug)]
struct ModelSingle {
    num_kernels: usize,
    kernels: Vec<Vec<Complex<f64>>>,
}

#[derive(Debug)]
struct ModelArray {
    num_models: usize,
    models: Vec<ModelSingle>,
}

#[derive(Debug)]
struct KernelIterator<'a> {
    num_kernels: usize,
    state: (usize, usize),
    ref_mod: &'a ModelArray,
    full_rank: Vec<usize>,
    reduced_rank: Vec<usize>,
}

impl<'b> KernelIterator<'b> {
    fn new(num_kernels: usize, ref_mod: &'b ModelArray) -> KernelIterator {
        let num_models = ref_mod.num_models;
        let mut full_rank     = vec![0; num_models];
        let mut reduced_rank  = vec![0; num_models];
        let mut total_kernels = 0;
        for i in 0..num_models {
            let nkernels = ref_mod.models[i].num_kernels;
            full_rank[i] = nkernels;
            total_kernels += nkernels;
        }
        let delta = total_kernels - num_kernels;

        println!("total_kernels: {} num_kernels: {} delta: {}",
                 total_kernels, num_kernels, delta);

        for i in 0..num_models {
            let nkernels = full_rank[i];
            let fraction = nkernels as f64 / total_kernels as f64;
            reduced_rank[i] = nkernels - ((delta as f64 * fraction) as usize);
        }

        KernelIterator { num_kernels: num_kernels, state : (0, 0), ref_mod: ref_mod,
                         full_rank: full_rank, reduced_rank: reduced_rank}
    }

    fn valid_state(&self) -> bool {
        let (model_index, kernel_index) = self.state;
        println!("inside `valid_state` model_index: {} kernel_index: {}", model_index, kernel_index);
        println!("model_index  < self.ref_mod.num_models : {}", model_index  < self.ref_mod.num_models);
        model_index  < self.ref_mod.num_models //&&
        //kernel_index < self.reduced_rank[model_index]
    }

    fn advance_state(&mut self) -> bool {
        let (model_index, kernel_index) = self.state;
        if kernel_index+1 < self.reduced_rank[model_index] {
            self.state = (model_index, kernel_index + 1);
            true
        } else if model_index < self.ref_mod.num_models {
            self.state = (model_index + 1, 0);
            true
        } else {
            self.state = (model_index + 1, kernel_index);
            false
        }
    }
}

impl<'c> Iterator for KernelIterator<'c> {
    type Item = Vec<Complex<f64>>;

    fn next(&mut self) -> Option<Vec<Complex<f64>>> {
        let ret = if self.valid_state() {
            let (model_index, kernel_index) = self.state;
            let m = self.ref_mod;
            let ref model = m.models[model_index];
            println!("model_index: {} kernel_index: {}", model_index, kernel_index);
            self.advance_state();
            Some(model.kernels[kernel_index].clone())
        } else {
            None
        };
        ret
    }
}

fn main() {
    println!("This is a kernel_iterator test");

    let kq1 = ModelSingle{num_kernels: 21,  kernels: vec![Vec::new(); 21]};
    let kq2 = ModelSingle{num_kernels: 75,  kernels: vec![Vec::new(); 75]};
    let kq3 = ModelSingle{num_kernels: 121, kernels: vec![Vec::new(); 121]};

    let mut mod_array = ModelArray {num_models: 3, models: Vec::new()};
    mod_array.models.push(kq1);
    mod_array.models.push(kq2);
    mod_array.models.push(kq3);

    println!("mod_array is constructed and properly initialized: {:?}", mod_array);

    let ki = KernelIterator::new(17, &mod_array);

    println!("KernelIterator is constructed and properly initialized: {:?}", ki);

    for z in ki {
        let kernel: &Vec<Complex<f64>> = &z;
    }
}

<commit_msg>cleanup for kernel_iterator.<commit_after>
extern crate num;
use num::complex::Complex;

#[derive(Debug)]
struct ModelSingle {
    num_kernels: usize,
    kernels: Vec<Vec<Complex<f64>>>,
}

impl ModelSingle {
    // a convenient constructor
    fn new(num_kernels: usize) -> ModelSingle {
        ModelSingle{num_kernels: num_kernels,  kernels: vec![Vec::new(); num_kernels]}
    }
}

#[derive(Debug)]
struct ModelArray {
    num_models: usize,
    models: Vec<ModelSingle>,
}

#[derive(Debug)]
struct KernelIterator<'a> {
    num_kernels: usize,
    state: (usize, usize),
    ref_mod: &'a ModelArray,
    full_rank: Vec<usize>,
    reduced_rank: Vec<usize>,
}

impl<'b> KernelIterator<'b> {
    // a constructor
    fn new(num_kernels: usize, ref_mod: &'b ModelArray) -> KernelIterator {
        let num_models = ref_mod.num_models;
        let mut full_rank     = vec![0; num_models];
        let mut reduced_rank  = vec![0; num_models];
        let mut total_kernels = 0;
        for i in 0..num_models {
            let nkernels = ref_mod.models[i].num_kernels;
            full_rank[i] = nkernels;
            total_kernels += nkernels;
        }
        let delta = total_kernels - num_kernels;

        println!("total_kernels: {} num_kernels: {} delta: {}",
                 total_kernels, num_kernels, delta);

        for i in 0..num_models {
            let nkernels = full_rank[i];
            let fraction = nkernels as f64 / total_kernels as f64;
            reduced_rank[i] = nkernels - ((delta as f64 * fraction) as usize);
        }

        KernelIterator { num_kernels: num_kernels, state : (0, 0), ref_mod: ref_mod,
                         full_rank: full_rank, reduced_rank: reduced_rank}
    }

    fn valid_state(&self) -> bool {
        let (model_index, _) = self.state;
        model_index < self.ref_mod.num_models
    }

    fn advance_state(&mut self) {
        let (model_index, kernel_index) = self.state;
        if kernel_index+1 < self.reduced_rank[model_index] {
            self.state = (model_index, kernel_index + 1);
        } else {
            self.state = (model_index + 1, 0);
        }
    }
}

impl<'c> Iterator for KernelIterator<'c> {
    type Item = Vec<Complex<f64>>;

    fn next(&mut self) -> Option<Vec<Complex<f64>>> {
        let ret = if self.valid_state() {
            let (model_index, kernel_index) = self.state;
            let m = self.ref_mod;
            let ref model = m.models[model_index];
            println!("model_index: {} kernel_index: {}", model_index, kernel_index);
            self.advance_state();
            Some(model.kernels[kernel_index].clone())
        } else {
            None
        };
        ret
    }
}

fn main() {
    println!("This is a kernel_iterator test");

    let kq1 = ModelSingle::new(21);
    let kq2 = ModelSingle::new(75);
    let kq3 = ModelSingle::new(121);

    let mut mod_array = ModelArray {num_models: 3, models: Vec::new()};
    mod_array.models.push(kq1);
    mod_array.models.push(kq2);
    mod_array.models.push(kq3);

    println!("mod_array is constructed and properly initialized: {:?}", mod_array);

    let ki = KernelIterator::new(17, &mod_array);

    println!("KernelIterator is constructed and properly initialized: {:?}", ki);

    for z in ki {
        let kernel: &Vec<Complex<f64>> = &z;
    }
}

<|endoftext|>


<commit_before>extern crate crossbeam;
use std;
use decoders::NUM_CORES;
use std::mem;

extern crate byteorder;
use self::byteorder::{BigEndian, LittleEndian, ByteOrder};

#[derive(Debug, Copy, Clone)]
pub struct Endian {
  big: bool,
}

impl Endian {
  pub fn ru32(&self, buf: &[u8], pos: usize) -> u32 {
    if self.big {
      BEu32(buf,pos)
    } else {
      LEu32(buf,pos)
    }
  }

  pub fn ru16(&self, buf: &[u8], pos: usize) -> u16 {
    if self.big {
      BEu16(buf,pos)
    } else {
      LEu16(buf,pos)
    }
  }
}

pub static BIG_ENDIAN: Endian = Endian{big: true};
pub static LITTLE_ENDIAN: Endian = Endian{big: false};

#[allow(non_snake_case)] #[inline] pub fn BEu32(buf: &[u8], pos: usize) -> u32 {
  BigEndian::read_u32(&buf[pos..pos+4])
}

#[allow(non_snake_case)] #[inline] pub fn LEu32(buf: &[u8], pos: usize) -> u32 {
  LittleEndian::read_u32(&buf[pos..pos+4])
}

#[allow(non_snake_case)] #[inline] pub fn BEu16(buf: &[u8], pos: usize) -> u16 {
  BigEndian::read_u16(&buf[pos..pos+2])
}

#[allow(non_snake_case)] #[inline] pub fn LEu16(buf: &[u8], pos: usize) -> u16 {
  LittleEndian::read_u16(&buf[pos..pos+2])
}

pub fn decode_12be(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*12/8) as usize)..];

    for (o, i) in out.chunks_mut(2).zip(inb.chunks(3)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;
      let g3: u16 = i[2] as u16;

      o[0] = (g1 << 4) | (g2 >> 4);
      o[1] = ((g2 & 0x0f) << 8) | g3;
    }
  }))
}

pub fn decode_12le(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*12/8) as usize)..];

    for (o, i) in out.chunks_mut(2).zip(inb.chunks(3)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;
      let g3: u16 = i[2] as u16;

      o[0] = ((g2 & 0x0f) << 8) | g1;
      o[1] = (g3 << 4) | (g2 >> 4);
    }
  }))
}

pub fn decode_12be_unpacked(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*2) as usize)..];

    for (o, i) in out.chunks_mut(1).zip(inb.chunks(2)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;

      o[0] = ((g1 & 0x0f) << 8) | g2;
    }
  }))
}

pub fn decode_16le(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*2) as usize)..];

    for (o, i) in out.chunks_mut(1).zip(inb.chunks(2)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;

      o[0] = (g2 << 8) | g1;
    }
  }))
}

pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
  where F : Fn(&mut [u16], usize, usize, usize)+std::marker::Sync {
  let mut out: Vec<u16> = vec![0; (width*height) as usize];

  // Default to the number of available physical cores (no hyperthreading)
  let threads = *NUM_CORES;

  // If we're only using one thread do it all sequentially and be done with it
  if threads < 2 || height < threads {
    closure(&mut out, 0, width, height);
    return out
  }

  let mut split = height/threads;
  if split*threads < height { // Make sure the last split is always the smallest
    split += 1;
  }

  crossbeam::scope(|scope| {
    let mut handles = Vec::new();
    for (i,out_part) in (&mut out[..]).chunks_mut(split*width).enumerate() {
      let start = split*(i);
      let tall = out_part.len()/width;
      let handle = scope.spawn(move || {
        closure(out_part, start, width, tall);
      });
      handles.push(handle);
    }

    for h in handles { h.join() };
  });

  out
}

#[derive(Debug, Clone)]
pub struct LookupTable {
  table: Vec<(u16, u16, u16)>,
}

impl LookupTable {
  pub fn new(table: &[u16]) -> LookupTable {
    let mut tbl = vec![(0,0,0); table.len()];
    for i in 0..table.len() {
      let center = table[i];
      let lower = if i > 0 {table[i-1]} else {center};
      let upper = if i < (table.len()-1) {table[i+1]} else {center};
      let base = center - ((upper - lower + 2) / 4);
      let delta = upper - lower;
      tbl[i] = (center, base, delta);
    }
    LookupTable {
      table: tbl,
    }
  }

//  pub fn lookup(&self, value: u16) -> u16 {
//    let (val, _, _) = self.table[value as usize];
//    val
//  }

  pub fn dither(&self, value: u16, rand: &mut u32) -> u16 {
    let (_, sbase, sdelta) = self.table[value as usize];
    let base = sbase as u32;
    let delta = sdelta as u32;
    let pixel = base + ((delta * (*rand & 2047) + 1024) >> 12);
    *rand = 15700 * (*rand & 65535) + (*rand >> 16);
    pixel as u16
  }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PumpOrder {
  LSB,
  MSB,
}

pub struct BitPump<'a> {
  buffer: &'a [u8],
  pos: usize,
  bits: u64,
  nbits: u32,
  order: PumpOrder,
}

impl<'a> BitPump<'a> {
  pub fn new(src: &'a [u8], order: PumpOrder) -> BitPump {
    BitPump {
      buffer: src,
      pos: 0,
      bits: 0,
      nbits: 0,
      order: order,
    }
  }

  fn fill_bits(&mut self) {
    if self.order == PumpOrder::MSB {
      let mut bytes: [u8;8] = unsafe{mem::transmute(self.bits)};
      let fillbytes = (64-self.nbits) / 8;
      for i in 0..(8-fillbytes) {
        bytes[(7-i) as usize] = bytes[(7-i-fillbytes) as usize];
      }
      for i in 0..fillbytes {
        bytes[(fillbytes-i-1) as usize] = self.buffer[self.pos];
        self.pos += 1;
      }
      self.bits = unsafe{mem::transmute(bytes)};
      self.nbits += fillbytes*8;
    } else {
      let inbits: u64 = LEu32(self.buffer, self.pos) as u64;
      self.pos += 4;
      self.bits = ((inbits << 32) | (self.bits << (32-self.nbits))) >> (32-self.nbits);
      self.nbits += 32;
    }
  }

  pub fn peek_bits(&mut self, num: u32) -> u32 {
    if num > self.nbits {
      self.fill_bits();
    }
    if self.order == PumpOrder::MSB {
      (self.bits >> (self.nbits-num)) as u32
    } else {
      (self.bits & (0x0ffffffffu64 >> (32-num))) as u32
    }
  }

  pub fn get_bits(&mut self, num: u32) -> u32 {
    if num == 0 {
      return 0
    }

    let val = self.peek_bits(num);
    self.nbits -= num;
    if self.order == PumpOrder::MSB {
      if self.nbits > 0 {
        let erasebits = 64-self.nbits;
        self.bits = (self.bits << erasebits) >> erasebits;
      }
    } else {
      self.bits = self.bits >> num;
    }
    val
  }

  pub fn get_ibits(&mut self, num: u32) -> i32 {
    let val = self.get_bits(num);
    unsafe{mem::transmute(val)}
  }
}
<commit_msg>Simplify bit dumping in BitPump<commit_after>extern crate crossbeam;
use std;
use decoders::NUM_CORES;
use std::mem;

extern crate byteorder;
use self::byteorder::{BigEndian, LittleEndian, ByteOrder};

#[derive(Debug, Copy, Clone)]
pub struct Endian {
  big: bool,
}

impl Endian {
  pub fn ru32(&self, buf: &[u8], pos: usize) -> u32 {
    if self.big {
      BEu32(buf,pos)
    } else {
      LEu32(buf,pos)
    }
  }

  pub fn ru16(&self, buf: &[u8], pos: usize) -> u16 {
    if self.big {
      BEu16(buf,pos)
    } else {
      LEu16(buf,pos)
    }
  }
}

pub static BIG_ENDIAN: Endian = Endian{big: true};
pub static LITTLE_ENDIAN: Endian = Endian{big: false};

#[allow(non_snake_case)] #[inline] pub fn BEu32(buf: &[u8], pos: usize) -> u32 {
  BigEndian::read_u32(&buf[pos..pos+4])
}

#[allow(non_snake_case)] #[inline] pub fn LEu32(buf: &[u8], pos: usize) -> u32 {
  LittleEndian::read_u32(&buf[pos..pos+4])
}

#[allow(non_snake_case)] #[inline] pub fn BEu16(buf: &[u8], pos: usize) -> u16 {
  BigEndian::read_u16(&buf[pos..pos+2])
}

#[allow(non_snake_case)] #[inline] pub fn LEu16(buf: &[u8], pos: usize) -> u16 {
  LittleEndian::read_u16(&buf[pos..pos+2])
}

pub fn decode_12be(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*12/8) as usize)..];

    for (o, i) in out.chunks_mut(2).zip(inb.chunks(3)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;
      let g3: u16 = i[2] as u16;

      o[0] = (g1 << 4) | (g2 >> 4);
      o[1] = ((g2 & 0x0f) << 8) | g3;
    }
  }))
}

pub fn decode_12le(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*12/8) as usize)..];

    for (o, i) in out.chunks_mut(2).zip(inb.chunks(3)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;
      let g3: u16 = i[2] as u16;

      o[0] = ((g2 & 0x0f) << 8) | g1;
      o[1] = (g3 << 4) | (g2 >> 4);
    }
  }))
}

pub fn decode_12be_unpacked(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*2) as usize)..];

    for (o, i) in out.chunks_mut(1).zip(inb.chunks(2)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;

      o[0] = ((g1 & 0x0f) << 8) | g2;
    }
  }))
}

pub fn decode_16le(buf: &[u8], width: usize, height: usize) -> Vec<u16> {
  decode_threaded(width, height, &(|out: &mut [u16], start, width, _| {
    let inb = &buf[((start*width*2) as usize)..];

    for (o, i) in out.chunks_mut(1).zip(inb.chunks(2)) {
      let g1: u16 = i[0] as u16;
      let g2: u16 = i[1] as u16;

      o[0] = (g2 << 8) | g1;
    }
  }))
}

pub fn decode_threaded<F>(width: usize, height: usize, closure: &F) -> Vec<u16>
  where F : Fn(&mut [u16], usize, usize, usize)+std::marker::Sync {
  let mut out: Vec<u16> = vec![0; (width*height) as usize];

  // Default to the number of available physical cores (no hyperthreading)
  let threads = *NUM_CORES;

  // If we're only using one thread do it all sequentially and be done with it
  if threads < 2 || height < threads {
    closure(&mut out, 0, width, height);
    return out
  }

  let mut split = height/threads;
  if split*threads < height { // Make sure the last split is always the smallest
    split += 1;
  }

  crossbeam::scope(|scope| {
    let mut handles = Vec::new();
    for (i,out_part) in (&mut out[..]).chunks_mut(split*width).enumerate() {
      let start = split*(i);
      let tall = out_part.len()/width;
      let handle = scope.spawn(move || {
        closure(out_part, start, width, tall);
      });
      handles.push(handle);
    }

    for h in handles { h.join() };
  });

  out
}

#[derive(Debug, Clone)]
pub struct LookupTable {
  table: Vec<(u16, u16, u16)>,
}

impl LookupTable {
  pub fn new(table: &[u16]) -> LookupTable {
    let mut tbl = vec![(0,0,0); table.len()];
    for i in 0..table.len() {
      let center = table[i];
      let lower = if i > 0 {table[i-1]} else {center};
      let upper = if i < (table.len()-1) {table[i+1]} else {center};
      let base = center - ((upper - lower + 2) / 4);
      let delta = upper - lower;
      tbl[i] = (center, base, delta);
    }
    LookupTable {
      table: tbl,
    }
  }

//  pub fn lookup(&self, value: u16) -> u16 {
//    let (val, _, _) = self.table[value as usize];
//    val
//  }

  pub fn dither(&self, value: u16, rand: &mut u32) -> u16 {
    let (_, sbase, sdelta) = self.table[value as usize];
    let base = sbase as u32;
    let delta = sdelta as u32;
    let pixel = base + ((delta * (*rand & 2047) + 1024) >> 12);
    *rand = 15700 * (*rand & 65535) + (*rand >> 16);
    pixel as u16
  }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PumpOrder {
  LSB,
  MSB,
}

pub struct BitPump<'a> {
  buffer: &'a [u8],
  pos: usize,
  bits: u64,
  nbits: u32,
  order: PumpOrder,
}

impl<'a> BitPump<'a> {
  pub fn new(src: &'a [u8], order: PumpOrder) -> BitPump {
    BitPump {
      buffer: src,
      pos: 0,
      bits: 0,
      nbits: 0,
      order: order,
    }
  }

  fn fill_bits(&mut self) {
    if self.order == PumpOrder::MSB {
      let mut bytes: [u8;8] = unsafe{mem::transmute(self.bits)};
      let fillbytes = (64-self.nbits) / 8;
      for i in 0..(8-fillbytes) {
        bytes[(7-i) as usize] = bytes[(7-i-fillbytes) as usize];
      }
      for i in 0..fillbytes {
        bytes[(fillbytes-i-1) as usize] = self.buffer[self.pos];
        self.pos += 1;
      }
      self.bits = unsafe{mem::transmute(bytes)};
      self.nbits += fillbytes*8;
    } else {
      let inbits: u64 = LEu32(self.buffer, self.pos) as u64;
      self.pos += 4;
      self.bits = ((inbits << 32) | (self.bits << (32-self.nbits))) >> (32-self.nbits);
      self.nbits += 32;
    }
  }

  pub fn peek_bits(&mut self, num: u32) -> u32 {
    if num > self.nbits {
      self.fill_bits();
    }
    if self.order == PumpOrder::MSB {
      (self.bits >> (self.nbits-num)) as u32
    } else {
      (self.bits & (0x0ffffffffu64 >> (32-num))) as u32
    }
  }

  pub fn get_bits(&mut self, num: u32) -> u32 {
    if num == 0 {
      return 0
    }

    let val = self.peek_bits(num);
    self.nbits -= num;
    if self.order == PumpOrder::MSB {
      self.bits &= (1 << self.nbits) - 1;
    } else {
      self.bits >>= num;
    }
    val
  }

  pub fn get_ibits(&mut self, num: u32) -> i32 {
    let val = self.get_bits(num);
    unsafe{mem::transmute(val)}
  }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate lazy_static;
extern crate quickcheck;
extern crate fail;
extern crate rand;
extern crate sled;
extern crate pagecache;

use std::collections::{BTreeMap, HashSet};
use std::sync::Mutex;

use quickcheck::{Arbitrary, Gen, QuickCheck, StdGen};

use sled::*;

#[derive(Debug, Clone)]
enum Op {
    Set,
    Del(u8),
    Restart,
    FailPoint(&'static str),
}

use Op::*;

impl Arbitrary for Op {
    fn arbitrary<G: Gen>(g: &mut G) -> Op {
        let fail_points = vec![
            "initial allocation",
            "initial allocation post",
            "zero segment",
            "zero segment post",
            "zero garbage segment",
            "zero garbage segment post",
            "buffer write",
            "buffer write post",
            "write_config bytes",
            "write_config crc",
            "write_config post",
            "trailer write",
            "trailer write post",
            "snap write",
            "snap write len",
            "snap write crc",
            "snap write post",
            "snap write mv",
            "snap write mv post",
            "snap write rm old",
        ];

        if g.gen_weighted_bool(30) {
            return FailPoint(*g.choose(&fail_points).unwrap());
        }

        if g.gen_weighted_bool(10) {
            return Restart;
        }

        let choice = g.gen_range(0, 2);

        match choice {
            0 => Set,
            1 => Del(g.gen::<u8>()),
            _ => panic!("impossible choice"),
        }
    }

    fn shrink(&self) -> Box<Iterator<Item = Op>> {
        match *self {
            Op::Del(ref lid) if *lid > 0 => Box::new(
                vec![Op::Del(*lid - 1)].into_iter(),
            ),
            _ => Box::new(vec![].into_iter()),
        }
    }
}

fn v(b: &Vec<u8>) -> u16 {
    assert_eq!(b.len(), 2);
    ((b[0] as u16) << 8) + b[1] as u16
}

fn prop_tree_crashes_nicely(ops: Vec<Op>, flusher: bool) -> bool {
    lazy_static! {
        // forces quickcheck to run one thread at a time
        static ref M: Mutex<()> = Mutex::new(());
    }

    let _lock = M.lock().unwrap();

    // clear all failpoints that may be left over from the last run
    fail::teardown();

    let config = ConfigBuilder::new()
        .temporary(true)
        .snapshot_after_ops(1)
        .flush_every_ms(if flusher { Some(1) } else {None})
        .io_buf_size(300)
        .min_items_per_segment(1)
        .blink_fanout(2) // smol pages for smol buffers
        .cache_capacity(40)
        .build();

    let mut tree = sled::Tree::start(config.clone()).unwrap();
    let mut reference = BTreeMap::new();
    let mut fail_points = HashSet::new();

    macro_rules! restart {
        () => {
            drop(tree);
            let tree_res = sled::Tree::start(config.clone());
            if let Err(ref e) = tree_res {
                if e == &Error::FailPoint {
                    return true;
                }

                println!("could not start database: {}", e);
                return false;
            }

            tree = tree_res.unwrap();

            let tree_iter = tree.iter().map(|res| {
                let (ref tk, _) = res.unwrap();
                v(tk)
            });
            let mut ref_iter = reference.iter().map(|(ref rk, ref rv)| (**rk, **rv));
            for t in tree_iter {
                // make sure the tree value is in there
                while let Some((r, (_rv, certainty))) = ref_iter.next() {
                    if certainty {
                        // tree MUST match reference if we have a certain reference
                        if t != r {
                            println!("expected to iterate over {:?} but got {:?} instead", r, t);
                            return false;
                        }
                        break;
                    } else {
                        // we have an uncertain reference, so we iterate through
                        // it and guarantee the reference is never higher than
                        // the tree value.
                        if t == r {
                            // we can move on to the next tree item
                            break;
                        }

                        if r > t {
                            // we have a bug, the reference iterator should always be <= tree
                            println!("tree verification failed: expected {:?} got {:?}", r, t);
                            return false;
                        }

                        // we are iterating through the reference until we have a certain
                        // item or an item that matches the tree's real item anyway
                    }
                }
            }
        }
    }

    macro_rules! fp_crash {
        ($e:expr) => {
            match $e {
                Ok(thing) => thing,
                Err(Error::FailPoint) => {
                    fail::teardown();
                    restart!();
                    continue;
                }
                other => {
                    println!("got non-failpoint err: {:?}", other);
                    return false;
                },
            }
        }
    }

    // we always increase set_counter because
    let mut set_counter = 0u16;

    for op in ops.into_iter() {
        match op {
            Set => {
                let hi = (set_counter >> 8) as u8;
                let lo = set_counter as u8;

                // insert false certainty until it fully completes
                reference.insert(set_counter, (set_counter, false));

                fp_crash!(tree.set(vec![hi, lo], vec![hi, lo]));

                // make sure we keep the disk and reference in-sync
                // maybe in the future put pending things in their own
                // reference and have a Flush op that syncs them.
                // just because the set above didn't hit a failpoint,
                // it doesn't mean this flush won't hit one, so we
                // also use the fp_crash macro here for handling it.
                fp_crash!(tree.flush());

                // now we should be certain the thing is in there, set certainty to true
                reference.insert(set_counter, (set_counter, true));

                set_counter += 1;
            }
            Del(k) => {
                // insert false certainty before completes
                reference.insert(k as u16, (k as u16, false));

                fp_crash!(tree.del(&*vec![0, k]));
                tree.flush().unwrap();

                reference.remove(&(k as u16));
            }
            Restart => {
                restart!();
            }
            FailPoint(fp) => {
                fail_points.insert(fp.clone());
                fail::cfg(&*fp, "return").unwrap();
            }
        }
    }

    fail::teardown();

    true
}

#[test]
fn quickcheck_tree_with_failpoints() {
    // use fewer tests for travis OSX builds that stall out all the time
    #[cfg(target_os = "macos")]
    let n_tests = 50;

    #[cfg(not(target_os = "macos"))]
    let n_tests = 100;

    QuickCheck::new()
        .gen(StdGen::new(rand::thread_rng(), 100))
        .tests(n_tests)
        .max_tests(10000)
        .quickcheck(prop_tree_crashes_nicely as fn(Vec<Op>, bool) -> bool);
}

#[test]
fn failpoints_bug_1() {
    // postmortem 1: model did not account for proper reasons to fail to start
    assert!(prop_tree_crashes_nicely(
        vec![FailPoint("snap write"), Restart],
        false,
    ));
}

#[test]
fn failpoints_bug_2() {
    // postmortem 1: the system was assuming the happy path across failpoints
    assert!(prop_tree_crashes_nicely(
        vec![FailPoint("buffer write post"), Set, Set, Restart],
        false,
    ))
}

#[test]
fn failpoints_bug_3() {
    // postmortem 1: this was a regression that happened because we
    // chose to eat errors about advancing snapshots, which trigger
    // log flushes. We should not trigger flushes from snapshots,
    // but first we need to make sure we are better about detecting
    // tears, by not also using 0 as a failed flush signifier.
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            Set,
            Set,
            Set,
            FailPoint("trailer write"),
            Set,
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}

#[test]
fn failpoints_bug_4() {
    // postmortem 1: the test model was not properly accounting for
    // writes that may-or-may-not be present due to an error.
    assert!(prop_tree_crashes_nicely(
        vec![Set, FailPoint("snap write"), Del(0), Set, Restart],
        false,
    ))
}

#[test]
fn failpoints_bug_5() {
    // postmortem 1:
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            FailPoint("snap write mv post"),
            Set,
            FailPoint("snap write"),
            Set,
            Set,
            Set,
            Restart,
            FailPoint("zero segment"),
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}

#[test]
fn failpoints_bug_6() {
    // postmortem 1:
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            Del(0),
            Set,
            Set,
            Set,
            Restart,
            FailPoint("zero segment post"),
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}
<commit_msg>catch panics when testing failpoints so that we can continue to shrink our failing cases<commit_after>#[macro_use]
extern crate lazy_static;
extern crate quickcheck;
extern crate fail;
extern crate rand;
extern crate sled;
extern crate pagecache;

use std::collections::{BTreeMap, HashSet};
use std::sync::Mutex;

use quickcheck::{Arbitrary, Gen, QuickCheck, StdGen};

use sled::*;

#[derive(Debug, Clone)]
enum Op {
    Set,
    Del(u8),
    Restart,
    FailPoint(&'static str),
}

use Op::*;

impl Arbitrary for Op {
    fn arbitrary<G: Gen>(g: &mut G) -> Op {
        let fail_points = vec![
            "initial allocation",
            "initial allocation post",
            "zero segment",
            "zero segment post",
            "zero garbage segment",
            "zero garbage segment post",
            "buffer write",
            "buffer write post",
            "write_config bytes",
            "write_config crc",
            "write_config post",
            "trailer write",
            "trailer write post",
            "snap write",
            "snap write len",
            "snap write crc",
            "snap write post",
            "snap write mv",
            "snap write mv post",
            "snap write rm old",
        ];

        if g.gen_weighted_bool(30) {
            return FailPoint(*g.choose(&fail_points).unwrap());
        }

        if g.gen_weighted_bool(10) {
            return Restart;
        }

        let choice = g.gen_range(0, 2);

        match choice {
            0 => Set,
            1 => Del(g.gen::<u8>()),
            _ => panic!("impossible choice"),
        }
    }

    fn shrink(&self) -> Box<Iterator<Item = Op>> {
        match *self {
            Op::Del(ref lid) if *lid > 0 => Box::new(
                vec![Op::Del(*lid - 1)].into_iter(),
            ),
            _ => Box::new(vec![].into_iter()),
        }
    }
}

fn v(b: &Vec<u8>) -> u16 {
    assert_eq!(b.len(), 2);
    ((b[0] as u16) << 8) + b[1] as u16
}

fn prop_tree_crashes_nicely(ops: Vec<Op>, flusher: bool) -> bool {
    lazy_static! {
        // forces quickcheck to run one thread at a time
        static ref M: Mutex<()> = Mutex::new(());
    }

    let _lock = M.lock().expect("our test lock should not be poisoned");

    // clear all failpoints that may be left over from the last run
    fail::teardown();

    let res = std::panic::catch_unwind(
        || run_tree_crashes_nicely(ops.clone(), flusher),
    );

    fail::teardown();

    match res {
        Err(e) => {
            println!(
                "failed with {:?} on ops {:?} flusher {}",
                e,
                ops,
                flusher
            );
            false
        }
        Ok(res) => {
            if !res {
                println!("failed with ops {:?} flusher: {}", ops, flusher);
            }
            res
        }
    }
}

fn run_tree_crashes_nicely(ops: Vec<Op>, flusher: bool) -> bool {
    let config = ConfigBuilder::new()
        .temporary(true)
        .snapshot_after_ops(1)
        .flush_every_ms(if flusher { Some(1) } else {None})
        .io_buf_size(300)
        .min_items_per_segment(1)
        .blink_fanout(2) // smol pages for smol buffers
        .cache_capacity(40)
        .cache_bits(2)
        .build();

    let mut tree = sled::Tree::start(config.clone()).unwrap();
    let mut reference = BTreeMap::new();
    let mut fail_points = HashSet::new();

    macro_rules! restart {
        () => {
            drop(tree);
            let tree_res = sled::Tree::start(config.clone());
            if let Err(ref e) = tree_res {
                if e == &Error::FailPoint {
                    return true;
                }

                println!("could not start database: {}", e);
                return false;
            }

            tree = tree_res.unwrap();

            let tree_iter = tree.iter().map(|res| {
                let (ref tk, _) = res.unwrap();
                v(tk)
            });
            let mut ref_iter = reference.iter().map(|(ref rk, ref rv)| (**rk, **rv));
            for t in tree_iter {
                // make sure the tree value is in there
                while let Some((r, (_rv, certainty))) = ref_iter.next() {
                    if certainty {
                        // tree MUST match reference if we have a certain reference
                        if t != r {
                            println!("expected to iterate over {:?} but got {:?} instead", r, t);
                            return false;
                        }
                        break;
                    } else {
                        // we have an uncertain reference, so we iterate through
                        // it and guarantee the reference is never higher than
                        // the tree value.
                        if t == r {
                            // we can move on to the next tree item
                            break;
                        }

                        if r > t {
                            // we have a bug, the reference iterator should always be <= tree
                            println!("tree verification failed: expected {:?} got {:?}", r, t);
                            return false;
                        }

                        // we are iterating through the reference until we have a certain
                        // item or an item that matches the tree's real item anyway
                    }
                }
            }
        }
    }

    macro_rules! fp_crash {
        ($e:expr) => {
            match $e {
                Ok(thing) => thing,
                Err(Error::FailPoint) => {
                    fail::teardown();
                    restart!();
                    continue;
                }
                other => {
                    println!("got non-failpoint err: {:?}", other);
                    return false;
                },
            }
        }
    }

    // we always increase set_counter because
    let mut set_counter = 0u16;

    for op in ops.into_iter() {
        match op {
            Set => {
                let hi = (set_counter >> 8) as u8;
                let lo = set_counter as u8;

                // insert false certainty until it fully completes
                reference.insert(set_counter, (set_counter, false));

                fp_crash!(tree.set(vec![hi, lo], vec![hi, lo]));

                // make sure we keep the disk and reference in-sync
                // maybe in the future put pending things in their own
                // reference and have a Flush op that syncs them.
                // just because the set above didn't hit a failpoint,
                // it doesn't mean this flush won't hit one, so we
                // also use the fp_crash macro here for handling it.
                fp_crash!(tree.flush());

                // now we should be certain the thing is in there, set certainty to true
                reference.insert(set_counter, (set_counter, true));

                set_counter += 1;
            }
            Del(k) => {
                // insert false certainty before completes
                reference.insert(k as u16, (k as u16, false));

                fp_crash!(tree.del(&*vec![0, k]));
                tree.flush().unwrap();

                reference.remove(&(k as u16));
            }
            Restart => {
                restart!();
            }
            FailPoint(fp) => {
                fail_points.insert(fp.clone());
                fail::cfg(&*fp, "return").unwrap();
            }
        }
    }

    true
}

#[test]
fn quickcheck_tree_with_failpoints() {
    // use fewer tests for travis OSX builds that stall out all the time
    #[cfg(target_os = "macos")]
    let n_tests = 50;

    #[cfg(not(target_os = "macos"))]
    let n_tests = 100;

    QuickCheck::new()
        .gen(StdGen::new(rand::thread_rng(), 100))
        .tests(n_tests)
        .max_tests(10000)
        .quickcheck(prop_tree_crashes_nicely as fn(Vec<Op>, bool) -> bool);
}

#[test]
fn failpoints_bug_1() {
    // postmortem 1: model did not account for proper reasons to fail to start
    assert!(prop_tree_crashes_nicely(
        vec![FailPoint("snap write"), Restart],
        false,
    ));
}

#[test]
fn failpoints_bug_2() {
    // postmortem 1: the system was assuming the happy path across failpoints
    assert!(prop_tree_crashes_nicely(
        vec![FailPoint("buffer write post"), Set, Set, Restart],
        false,
    ))
}

#[test]
fn failpoints_bug_3() {
    // postmortem 1: this was a regression that happened because we
    // chose to eat errors about advancing snapshots, which trigger
    // log flushes. We should not trigger flushes from snapshots,
    // but first we need to make sure we are better about detecting
    // tears, by not also using 0 as a failed flush signifier.
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            Set,
            Set,
            Set,
            FailPoint("trailer write"),
            Set,
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}

#[test]
fn failpoints_bug_4() {
    // postmortem 1: the test model was not properly accounting for
    // writes that may-or-may-not be present due to an error.
    assert!(prop_tree_crashes_nicely(
        vec![Set, FailPoint("snap write"), Del(0), Set, Restart],
        false,
    ))
}

#[test]
fn failpoints_bug_5() {
    // postmortem 1:
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            FailPoint("snap write mv post"),
            Set,
            FailPoint("snap write"),
            Set,
            Set,
            Set,
            Restart,
            FailPoint("zero segment"),
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}

#[test]
fn failpoints_bug_6() {
    // postmortem 1:
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            Del(0),
            Set,
            Set,
            Set,
            Restart,
            FailPoint("zero segment post"),
            Set,
            Set,
            Set,
            Restart,
        ],
        false,
    ))
}

#[test]
fn failpoints_bug_7() {
    // postmortem 1:
    assert!(prop_tree_crashes_nicely(
        vec![
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Set,
            Del(17),
            Del(29),
            Del(246),
            Del(248),
            Set,
        ],
        false,
    ))
}
<|endoftext|>