<commit_before>
pub const CIPHERKEYLEN : usize = 32;
pub const TAGLEN : usize = 16;

/* TODO: replace with associated constants once that Rust feature is stable */
pub const MAXHASHLEN : usize = 64;
pub const MAXBLOCKLEN : usize = 128;
pub const DHLEN : usize = 32; /* TODO: generalize for Curve448, but annoying without prev item */

pub fn copy_memory(data: &[u8], out: &mut [u8]) -> usize {
    for count in 0..data.len() {out[count] = data[count];}
    data.len()
}

pub trait Random {
    fn new() -> Self;
    fn fill_bytes(&mut self, out: &mut [u8]);
}

pub trait Dh {
    fn name(out: &mut [u8]) -> usize;
    fn new(privkey: &[u8], pubkey: &[u8]) -> Self;
    fn generate<R: Random>(rng: &mut R) -> Self;

    fn pubkey(&self) -> &[u8];
    fn dh(&self, pubkey: &[u8]) -> [u8; DHLEN];
}

pub trait Cipher {
    fn name(out: &mut [u8]) -> usize;
    fn new(key: &[u8]) -> Self;

    fn encrypt(&mut self, nonce: u64, authtext: &[u8], plaintext: &[u8], out: &mut[u8]);
    fn decrypt(&mut self, nonce: u64, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool;
}

pub trait Hash : Sized {
    fn name(out: &mut [u8]) -> usize;
    fn new() -> Self;

    fn block_len() -> usize; /* see TODO at top */
    fn hash_len() -> usize; /* see TODO at top */

    fn input(&mut self, data: &[u8]);
    fn result(&mut self, out: &mut [u8]);

    fn hmac(key: &[u8], data: &[u8], out: &mut [u8]) {
        assert!(key.len() <= Self::block_len());
        let mut ipad = [0x36u8; MAXBLOCKLEN];
        let mut opad = [0x5cu8; MAXBLOCKLEN];
        for count in 0..key.len() {
            ipad[count] ^= key[count];
            opad[count] ^= key[count];
        }
        let mut inner_hasher = Self::new();
        inner_hasher.input(&ipad[..Self::block_len()]);
        inner_hasher.input(data);
        let mut inner_output = [0u8; MAXHASHLEN];
        inner_hasher.result(&mut inner_output);
        let mut outer_hasher = Self::new();
        outer_hasher.input(&opad[..Self::block_len()]);
        outer_hasher.input(&inner_output[..Self::hash_len()]);
        outer_hasher.result(out);
    }

    fn hkdf(chaining_key: &[u8], input_key_material: &[u8], out1: &mut [u8], out2: & mut[u8]) {
        let mut temp_key = [0u8; MAXHASHLEN];
        let mut in2 = [0u8; MAXHASHLEN+1];
        Self::hmac(chaining_key, input_key_material, &mut temp_key);
        Self::hmac(&temp_key, &[1u8], out1);
        copy_memory(&out1[0..Self::hash_len()], &mut in2);
        in2[Self::hash_len()] = 2;
        Self::hmac(&temp_key, &in2[..Self::hash_len()+1], out2);
    }
}

pub struct CipherState<C: Cipher> {
    cipher : C,
    pub n : u64,
    good : bool
}

impl<C: Cipher> CipherState<C> {

    pub fn new(key: &[u8], nonce: u64) -> CipherState<C> {
        CipherState{cipher : C::new(key), n: nonce, good : true}
    }

    pub fn encrypt_ad(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        assert!(self.good);
        self.cipher.encrypt(self.n, authtext, plaintext, out);
        self.n += 1;
    }

    pub fn decrypt_ad(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        assert!(self.good);
        self.good = self.cipher.decrypt(self.n, authtext, ciphertext, out);
        self.n += 1;
        self.good
    }

    pub fn encrypt(&mut self, plaintext: &[u8], out: &mut[u8]) {
        assert!(self.good);
        self.cipher.encrypt(self.n, &[0u8;0], plaintext, out);
        self.n += 1;
    }

    pub fn decrypt(&mut self, ciphertext: &[u8], out: &mut[u8]) -> bool {
        assert!(self.good);
        self.good = self.cipher.decrypt(self.n, &[0u8;0], ciphertext, out);
        self.n += 1;
        self.good
    }
}

<commit_msg>Adding integer overflow check.<commit_after>
pub const CIPHERKEYLEN : usize = 32;
pub const TAGLEN : usize = 16;

/* TODO: replace with associated constants once that Rust feature is stable */
pub const MAXHASHLEN : usize = 64;
pub const MAXBLOCKLEN : usize = 128;
pub const DHLEN : usize = 32; /* TODO: generalize for Curve448, but annoying without prev item */

pub fn copy_memory(data: &[u8], out: &mut [u8]) -> usize {
    for count in 0..data.len() {out[count] = data[count];}
    data.len()
}

pub trait Random {
    fn new() -> Self;
    fn fill_bytes(&mut self, out: &mut [u8]);
}

pub trait Dh {
    fn name(out: &mut [u8]) -> usize;
    fn new(privkey: &[u8], pubkey: &[u8]) -> Self;
    fn generate<R: Random>(rng: &mut R) -> Self;

    fn pubkey(&self) -> &[u8];
    fn dh(&self, pubkey: &[u8]) -> [u8; DHLEN];
}

pub trait Cipher {
    fn name(out: &mut [u8]) -> usize;
    fn new(key: &[u8]) -> Self;

    fn encrypt(&mut self, nonce: u64, authtext: &[u8], plaintext: &[u8], out: &mut[u8]);
    fn decrypt(&mut self, nonce: u64, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool;
}

pub trait Hash : Sized {
    fn name(out: &mut [u8]) -> usize;
    fn new() -> Self;

    fn block_len() -> usize; /* see TODO at top */
    fn hash_len() -> usize; /* see TODO at top */

    fn input(&mut self, data: &[u8]);
    fn result(&mut self, out: &mut [u8]);

    fn hmac(key: &[u8], data: &[u8], out: &mut [u8]) {
        assert!(key.len() <= Self::block_len());
        let mut ipad = [0x36u8; MAXBLOCKLEN];
        let mut opad = [0x5cu8; MAXBLOCKLEN];
        for count in 0..key.len() {
            ipad[count] ^= key[count];
            opad[count] ^= key[count];
        }
        let mut inner_hasher = Self::new();
        inner_hasher.input(&ipad[..Self::block_len()]);
        inner_hasher.input(data);
        let mut inner_output = [0u8; MAXHASHLEN];
        inner_hasher.result(&mut inner_output);
        let mut outer_hasher = Self::new();
        outer_hasher.input(&opad[..Self::block_len()]);
        outer_hasher.input(&inner_output[..Self::hash_len()]);
        outer_hasher.result(out);
    }

    fn hkdf(chaining_key: &[u8], input_key_material: &[u8], out1: &mut [u8], out2: & mut[u8]) {
        let mut temp_key = [0u8; MAXHASHLEN];
        let mut in2 = [0u8; MAXHASHLEN+1];
        Self::hmac(chaining_key, input_key_material, &mut temp_key);
        Self::hmac(&temp_key, &[1u8], out1);
        copy_memory(&out1[0..Self::hash_len()], &mut in2);
        in2[Self::hash_len()] = 2;
        Self::hmac(&temp_key, &in2[..Self::hash_len()+1], out2);
    }
}

pub struct CipherState<C: Cipher> {
    cipher : C,
    pub n : u64,
    good : bool
}

impl<C: Cipher> CipherState<C> {

    pub fn new(key: &[u8], nonce: u64) -> CipherState<C> {
        CipherState{cipher : C::new(key), n: nonce, good : true}
    }

    pub fn encrypt_ad(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        assert!(self.good);
        self.cipher.encrypt(self.n, authtext, plaintext, out);
        self.n += 1;
        self.good = self.n != 0; // check for wrap-around
    }

    pub fn decrypt_ad(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        assert!(self.good);
        let result = self.cipher.decrypt(self.n, authtext, ciphertext, out);
        self.n += 1;
        self.good = self.n != 0; // check for wrap-around
        result
    }

    pub fn encrypt(&mut self, plaintext: &[u8], out: &mut[u8]) {
        assert!(self.good);
        self.cipher.encrypt(self.n, &[0u8;0], plaintext, out);
        self.n += 1;
        self.good = self.n != 0; // check for wrap-around
    }

    pub fn decrypt(&mut self, ciphertext: &[u8], out: &mut[u8]) -> bool {
        assert!(self.good);
        let result = self.cipher.decrypt(self.n, &[0u8;0], ciphertext, out);
        self.n += 1;
        self.good = self.n != 0; // check for wrap-around
        result
    }
}

<|endoftext|>


<commit_before>use super::cache::Cache;
use super::errors::CacheError;
use super::key::Key;
use super::typedefs::CacheResult;
use super::value::Value;

// helper func since assert_eq!(rv.unwrap(), err) does not work
fn assert_rv_eq<T>(rv: CacheResult<T>, err: CacheError) {
    // First of all it's supposed to be an error
    assert!(rv.is_err());

    // Now check if the error constructor is the right one
    match rv {
        Ok(_) => (),
        Err(e) => { assert_eq!(e, err); }
    };
}

#[test]
fn test_set_one_key() {
    let mut cache = Cache::with_defaults(1);

    let key = key!(1, 2, 3);
    let value = value!(4, 5, 6);

    // First set it
    let rv = cache.set(key.clone(), value.clone());
    assert!(rv.is_ok());

    // Then test for it
    let rv = cache.contains_key(&key);
    assert_eq!(rv.unwrap(), true);

    // Test for a key that was not set
    let rv = cache.contains_key(&key!(9, 8));
    assert_eq!(rv.unwrap(), false);

    // Now fetch it
    let value_found = cache.get(&key).unwrap();

    assert_eq!(value, *value_found);
}

#[test]
fn test_key_not_found() {
    let mut cache = Cache::with_defaults(1);

    // Set a key
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // Retrieve a different key
    let rv = cache.get(&key!(2));
    assert_rv_eq(rv, CacheError::KeyNotFound);
}

#[test]
fn test_store_beyond_capacity() {
    let mut cache = Cache::with_defaults(1);

    // we've now reached capacity
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // overwriting is ok
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // but we cannot store a new key
    let rv = cache.set(key!(2), value!(9));
    assert_rv_eq(rv, CacheError::CapacityExceeded);
}

#[test]
fn test_exceed_item_size_limits() {
    let mut cache = Cache::new(1, 1, 1);

    // set: use a key that is too long
    let rv = cache.set(key!(1, 2), value!(9));
    assert_rv_eq(rv, CacheError::KeyTooLong);

    // set: use a value that is too long
    let rv = cache.set(key!(1), value!(9, 8));
    assert_rv_eq(rv, CacheError::ValueTooLong);

    // get: use a key that is too long
    let rv = cache.get(&key!(1, 2));
    assert_rv_eq(rv, CacheError::KeyTooLong);

    // contains_key: use a key that is too long
    let rv = cache.contains_key(&key!(1, 2));
    assert_rv_eq(rv, CacheError::KeyTooLong);
}
<commit_msg>simplify imports<commit_after>use super::Cache;
use super::CacheError;
use super::Key;
use super::CacheResult;
use super::Value;

// helper func since assert_eq!(rv.unwrap(), err) does not work
fn assert_rv_eq<T>(rv: CacheResult<T>, err: CacheError) {
    // First of all it's supposed to be an error
    assert!(rv.is_err());

    // Now check if the error constructor is the right one
    match rv {
        Ok(_) => (),
        Err(e) => { assert_eq!(e, err); }
    };
}

#[test]
fn test_set_one_key() {
    let mut cache = Cache::with_defaults(1);

    let key = key!(1, 2, 3);
    let value = value!(4, 5, 6);

    // First set it
    let rv = cache.set(key.clone(), value.clone());
    assert!(rv.is_ok());

    // Then test for it
    let rv = cache.contains_key(&key);
    assert_eq!(rv.unwrap(), true);

    // Test for a key that was not set
    let rv = cache.contains_key(&key!(9, 8));
    assert_eq!(rv.unwrap(), false);

    // Now fetch it
    let value_found = cache.get(&key).unwrap();

    assert_eq!(value, *value_found);
}

#[test]
fn test_key_not_found() {
    let mut cache = Cache::with_defaults(1);

    // Set a key
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // Retrieve a different key
    let rv = cache.get(&key!(2));
    assert_rv_eq(rv, CacheError::KeyNotFound);
}

#[test]
fn test_store_beyond_capacity() {
    let mut cache = Cache::with_defaults(1);

    // we've now reached capacity
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // overwriting is ok
    let rv = cache.set(key!(1), value!(9));
    assert!(rv.is_ok());

    // but we cannot store a new key
    let rv = cache.set(key!(2), value!(9));
    assert_rv_eq(rv, CacheError::CapacityExceeded);
}

#[test]
fn test_exceed_item_size_limits() {
    let mut cache = Cache::new(1, 1, 1);

    // set: use a key that is too long
    let rv = cache.set(key!(1, 2), value!(9));
    assert_rv_eq(rv, CacheError::KeyTooLong);

    // set: use a value that is too long
    let rv = cache.set(key!(1), value!(9, 8));
    assert_rv_eq(rv, CacheError::ValueTooLong);

    // get: use a key that is too long
    let rv = cache.get(&key!(1, 2));
    assert_rv_eq(rv, CacheError::KeyTooLong);

    // contains_key: use a key that is too long
    let rv = cache.contains_key(&key!(1, 2));
    assert_rv_eq(rv, CacheError::KeyTooLong);
}
<|endoftext|>


<commit_before>use dispatch_table::{DispatchTable, DispatchFn};
use spin::RwLock;
use isr;

#[derive(Copy, Clone, Default)]
#[repr(packed)]
struct IDTEntry {
    base_low: u16,
    selector: u16,
    always0: u8,
    flags: u8,
    base_high: u16,
    base_highest: u32,
    _reserved: u32,
}

pub struct IDT([IDTEntry; 256]);

lazy_static! {
    static ref INTERRUPT_TABLE: RwLock<DispatchTable<u64, InterruptCPUState>> =
        RwLock::new(DispatchTable::new(256));
    static ref DESCRIPTOR_TABLE: RwLock<IDT> = RwLock::new(IDT([Default::default(); 256]));
}

impl IDT {
    fn set_entry(&mut self, num: usize, base: u64, selector: u16, flags: u8) {
        self.0[num].base_low = (base & 0xffff) as u16;
        self.0[num].base_high = ((base >> 16) & 0xffff) as u16;
        self.0[num].base_highest = (base >> 32) as u32;
        self.0[num].always0 = 0;
        self.0[num].selector = selector;
        self.0[num].flags = flags;
    }
}

#[repr(C)]
pub struct InterruptCPUState {
    ds: u64,
    r15: u64,
    r14: u64,
    r13: u64,
    r12: u64,
    r11: u64,
    r10: u64,
    r9: u64,
    r8: u64,
    rdi: u64,
    rsi: u64,
    rbp: u64,
    rdx: u64,
    rcx: u64,
    rbx: u64,
    rax: u64,
    interrupt_number: u64,
    error_code: u64,
    rip: u64,
    cs: u64,
    rflags: u64,
    rsp: u64,
    ss: u64,
}

pub fn register_interrupt_handler(interrupt: u64,
                                  stub: isr::InterruptServiceRoutine,
                                  handler: DispatchFn<u64, InterruptCPUState>) {
    INTERRUPT_TABLE.write().register(interrupt, handler);
    DESCRIPTOR_TABLE.write().set_entry(interrupt as usize, stub as u64, 0x08, 0x8e);
}

pub fn unregister_interrupt_handler(interrupt: u64) {
    INTERRUPT_TABLE.write().unregister(interrupt);
}

#[no_mangle]
pub extern "C" fn dispatch_interrupt(state: &mut InterruptCPUState) {
    INTERRUPT_TABLE.read().dispatch(state.interrupt_number, state);
}

fn default_handler(_interrupt_number: u64, _regs: &mut InterruptCPUState) -> bool {
    true
}

pub fn init_interrupt_handlers() {
    for i in 0..20 {
        register_interrupt_handler(i, isr::ISR[i as usize], default_handler);
    }
}

#[cfg(feature = "runtime_tests")]
pub mod runtime_tests {

    use interrupts;
    use vmx;
    use cli;
    use core::mem;

    fn new_host_idt_descriptor() -> vmx::CPUTableDescriptor {
        let base = (*interrupts::DESCRIPTOR_TABLE.write()).0.as_ptr() as u64;
        vmx::CPUTableDescriptor {
            base: base,
            limit: (mem::size_of::<interrupts::IDT>() - 1) as u16,
        }
    }

    pub fn run() {
        test_load_and_restore_idt();
    }

    fn test_load_and_restore_idt() {
        cli::ClearLocalInterruptsGuard::new();
        let idt_desc = new_host_idt_descriptor();
        let mut orig_idt_desc: vmx::CPUTableDescriptor = Default::default();
        vmx::sidt(&mut orig_idt_desc);
        vmx::lidt(&idt_desc);
        vmx::lidt(&orig_idt_desc);
    }
}
<commit_msg>Smaller IDT<commit_after>use dispatch_table::{DispatchTable, DispatchFn};
use spin::RwLock;
use isr;

#[derive(Copy, Clone, Default)]
#[repr(packed)]
struct IDTEntry {
    base_low: u16,
    selector: u16,
    always0: u8,
    flags: u8,
    base_high: u16,
    base_highest: u32,
    _reserved: u32,
}

pub struct IDT([IDTEntry; 20]);

lazy_static! {
    static ref INTERRUPT_TABLE: RwLock<DispatchTable<u64, InterruptCPUState>> =
        RwLock::new(DispatchTable::new(25));
    static ref DESCRIPTOR_TABLE: RwLock<IDT> = RwLock::new(IDT([Default::default(); 20]));
}

impl IDT {
    fn set_entry(&mut self, num: usize, base: u64, selector: u16, flags: u8) {
        self.0[num].base_low = (base & 0xffff) as u16;
        self.0[num].base_high = ((base >> 16) & 0xffff) as u16;
        self.0[num].base_highest = (base >> 32) as u32;
        self.0[num].always0 = 0;
        self.0[num].selector = selector;
        self.0[num].flags = flags;
    }
}

#[repr(C)]
pub struct InterruptCPUState {
    ds: u64,
    r15: u64,
    r14: u64,
    r13: u64,
    r12: u64,
    r11: u64,
    r10: u64,
    r9: u64,
    r8: u64,
    rdi: u64,
    rsi: u64,
    rbp: u64,
    rdx: u64,
    rcx: u64,
    rbx: u64,
    rax: u64,
    interrupt_number: u64,
    error_code: u64,
    rip: u64,
    cs: u64,
    rflags: u64,
    rsp: u64,
    ss: u64,
}

pub fn register_interrupt_handler(interrupt: u64,
                                  stub: isr::InterruptServiceRoutine,
                                  handler: DispatchFn<u64, InterruptCPUState>) {
    INTERRUPT_TABLE.write().register(interrupt, handler);
    DESCRIPTOR_TABLE.write().set_entry(interrupt as usize, stub as u64, 0x08, 0x8e);
}

pub fn unregister_interrupt_handler(interrupt: u64) {
    INTERRUPT_TABLE.write().unregister(interrupt);
}

#[no_mangle]
pub extern "C" fn dispatch_interrupt(state: &mut InterruptCPUState) {
    INTERRUPT_TABLE.read().dispatch(state.interrupt_number, state);
}

fn default_handler(_interrupt_number: u64, _regs: &mut InterruptCPUState) -> bool {
    true
}

pub fn init_interrupt_handlers() {
    for i in 0..20 {
        register_interrupt_handler(i, isr::ISR[i as usize], default_handler);
    }
}

#[cfg(feature = "runtime_tests")]
pub mod runtime_tests {

    use interrupts;
    use vmx;
    use cli;
    use core::mem;

    fn new_host_idt_descriptor() -> vmx::CPUTableDescriptor {
        let base = (*interrupts::DESCRIPTOR_TABLE.write()).0.as_ptr() as u64;
        vmx::CPUTableDescriptor {
            base: base,
            limit: (mem::size_of::<interrupts::IDT>() - 1) as u16,
        }
    }

    pub fn run() {
        test_load_and_restore_idt();
    }

    fn test_load_and_restore_idt() {
        cli::ClearLocalInterruptsGuard::new();
        let idt_desc = new_host_idt_descriptor();
        let mut orig_idt_desc: vmx::CPUTableDescriptor = Default::default();
        vmx::sidt(&mut orig_idt_desc);
        vmx::lidt(&idt_desc);
        vmx::lidt(&orig_idt_desc);
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for discrete probability distributions in terms of cumulative distribution
//! functions (CDF).

use std::f64;
use std::iter;
use std::slice;
use std::ops::Range;

use num::traits::{cast, NumCast};
use itertools::Itertools;

use stats::LogProb;

#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
pub struct Entry<T: Ord> {
    pub value: T,
    pub prob: LogProb
}

impl<T: Ord> Entry<T> {
    pub fn new(value: T, prob: LogProb) -> Self {
        Entry { value: value, prob: prob }
    }
}

/// Implementation of a cumulative distribution function.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
pub struct CDF<T: Ord> {
    inner: Vec<Entry<T>>
}

impl<T: Ord> CDF<T> {
    /// Create CDF from given probability mass function (PMF). The PMF may contain duplicate values
    /// the probabilities of which are summed during generation of the CDF.
    ///
    /// # Arguments
    ///
    /// * `pmf` - the PMF as a vector of `Entry` objects
    pub fn from_pmf(mut entries: Vec<Entry<T>>) -> Self {
        entries.sort_by(|a, b| a.value.cmp(&b.value));
        let mut inner: Vec<Entry<T>> = Vec::new();
        for mut e in entries {
            let p = inner.last().map_or(LogProb::ln_zero(), |e| e.prob).ln_add_exp(e.prob);
            if !inner.is_empty() && inner.last().unwrap().value == e.value {
                inner.last_mut().unwrap().prob = p;
            }
            else {
                e.prob = p;
                inner.push(e);
            }
        }
        let mut cdf = CDF {
            inner: inner
        };

        if relative_eq!(*cdf.total_prob(), *LogProb::ln_one()) && *cdf.total_prob() > *LogProb::ln_one() {
            cdf.inner.last_mut().unwrap().prob = LogProb::ln_one();
        }

        cdf
    }

    /// Create CDF from iterator. This can be used to replace the values of a CDF.
    pub fn from_cdf<I: Iterator<Item = Entry<T>>>(entries: I) -> Self {
        CDF { inner: entries.collect_vec() }
    }

    /// Reduce CDF by omitting values with zero probability.
    pub fn reduce(self) -> Self {
        let mut inner = Vec::new();
        let mut last = LogProb::ln_zero();
        for e in self.inner {
            if last != e.prob {
                last = e.prob;
                inner.push(e);
            }
        }
        CDF { inner: inner }
    }

    /// Downsample CDF to n entries. Panics if n <= 1 and returns identity if n is greater
    /// than the number of entries.
    pub fn sample(mut self, n: usize) -> Self {
        assert!(n > 1);
        if self.inner.len() <= n {
            self
        }
        else {
            let s = self.inner.len() / (n - 1);
            let last = self.inner.pop().unwrap();
            let mut inner = self.inner.into_iter().step(s).collect_vec();
            inner.push(last);
            CDF {
                inner: inner
            }
        }
    }

    /// Provide iterator.
    pub fn iter(&self) -> slice::Iter<Entry<T>>{
        self.inner.iter()
    }

    /// Iterator over corresponding PMF.
    pub fn iter_pmf<'a>(&'a self) -> CDFPMFIter<'a, T> {
        fn cdf_to_pmf<'a, G: Ord>(last_prob: &mut LogProb, e: &'a Entry<G>) -> Option<Entry<&'a G>> {
            let prob = e.prob.ln_sub_exp(*last_prob);
            *last_prob = e.prob;
            Some(Entry::new(&e.value, prob))
        }
        self.inner.iter().scan(LogProb::ln_zero(), cdf_to_pmf)
    }

    /// Get cumulative probability for a given value. If the value is not present,
    /// return the probability of the previous value. Complexity O(log n).
    pub fn get(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {
            Some(match self.inner.binary_search_by(|e| e.value.partial_cmp(value).unwrap()) {
                Ok(i) => self.inner[i].prob,
                Err(i) => if i > 0 { self.inner[i - 1].prob } else { LogProb::ln_zero() }
            })
        }
    }

    /// Get probability (i.e. probability mass) for a given value. Complexity O(log n).
    pub fn get_pmf(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {
            Some(match self.inner.binary_search_by(|e| e.value.partial_cmp(value).unwrap()) {
                Ok(i) => if i > 0 { self.inner[i].prob.ln_sub_exp(self.inner[i - 1].prob) } else { self.inner[0].prob },
                Err(i) => if i > 0 { self.inner[i - 1].prob } else { LogProb::ln_zero() }
            })
        }
    }

    /// Return total probability.
    pub fn total_prob(&self) -> LogProb {
        self.inner.last().map_or(LogProb::ln_zero(), |e| e.prob)
    }

    /// Return maximum a posteriori probability estimate (MAP).
    pub fn map(&self) -> Option<&T> {
        if let Some(mut max) = self.iter_pmf().next() {
            for e in self.iter_pmf() {
                if e.prob >= max.prob {
                    max = e;
                }
            }
            Some(&max.value)
        } else {
            None
        }
    }

    /// Return 95% credible interval.
    pub fn credible_interval(&self) -> Range<&T> {
        let p_lower = LogProb(0.025f64.ln());
        let p_upper = LogProb(0.095f64.ln());
        let lower = self.inner.binary_search_by(|e| e.prob.partial_cmp(&p_lower).unwrap()).unwrap_or_else(|i| i);
        let upper = self.inner.binary_search_by(|e| e.prob.partial_cmp(&p_upper).unwrap()).unwrap_or_else(|i| i - 1);

        &self.inner[lower].value..&self.inner[upper].value
    }

    /// Number of entries in the CDF.
    pub fn len(&self) -> usize {
        self.inner.len()
    }
}

impl<T: NumCast + Clone + Ord> CDF<T> {
    /// Calculate expected value.
    pub fn expected_value(&self) -> f64 {
        self.iter_pmf().map(|e| {
            cast::<T, f64>(e.value.clone()).unwrap() * e.prob.exp()
        }).fold(0.0f64, |s, e| s + e)
    }

    /// Calculate variance.
    pub fn variance(&self) -> f64 {
        let ev = self.expected_value();
        self.iter_pmf().map(|e| {
                (cast::<T, f64>(e.value.clone()).unwrap() - ev).powi(2) * e.prob.exp()
        }).fold(0.0, |s, e| s + e)
    }

    /// Calculate standard deviation.
    pub fn standard_deviation(&self) -> f64 {
        self.variance().sqrt()
    }
}

pub type CDFPMFIter<'a, T> = iter::Scan<slice::Iter<'a, Entry<T>>, LogProb, fn(&mut LogProb, &'a Entry<T>) -> Option<Entry<&'a T>>>;

#[cfg(test)]
mod test {
    use super::*;
    use stats::LogProb;
    use ordered_float::NotNaN;

    #[test]
    fn test_cdf() {
        let mut pmf = vec![Entry::new(NotNaN::new(0.0).unwrap(), LogProb(0.1f64.ln()))];
        for i in 0..9 {
            pmf.push(Entry::new(NotNaN::new(i as f64).unwrap(), LogProb(0.1f64.ln())));
        }
        println!("{:?}", pmf);

        let cdf = CDF::from_pmf(pmf.clone());
        println!("{:?}", cdf);
        for e in pmf.iter().skip(2) {
            assert_ulps_eq!(*e.prob, *cdf.get_pmf(&e.value).unwrap(), epsilon = 0.0000000000001);
        }
        assert_relative_eq!(*cdf.total_prob(), 1.0f64.ln());
        assert_relative_eq!(*cdf.get(&NotNaN::new(1.0).unwrap()).unwrap(), 0.3f64.ln(), epsilon = 0.00000001);
    }
}
<commit_msg>Generalize credible interval implementation.<commit_after>// Copyright 2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for discrete probability distributions in terms of cumulative distribution
//! functions (CDF).

use std::f64;
use std::iter;
use std::slice;
use std::ops::Range;

use num::traits::{cast, NumCast};
use itertools::Itertools;
use ordered_float::OrderedFloat;

use stats::LogProb;

#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
pub struct Entry<T: Ord> {
    pub value: T,
    pub prob: LogProb
}

impl<T: Ord> Entry<T> {
    pub fn new(value: T, prob: LogProb) -> Self {
        Entry { value: value, prob: prob }
    }
}

/// Implementation of a cumulative distribution function.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
pub struct CDF<T: Ord> {
    inner: Vec<Entry<T>>
}

impl<T: Ord> CDF<T> {
    /// Create CDF from given probability mass function (PMF). The PMF may contain duplicate values
    /// the probabilities of which are summed during generation of the CDF.
    ///
    /// # Arguments
    ///
    /// * `pmf` - the PMF as a vector of `Entry` objects
    pub fn from_pmf(mut entries: Vec<Entry<T>>) -> Self {
        entries.sort_by(|a, b| a.value.cmp(&b.value));
        let mut inner: Vec<Entry<T>> = Vec::new();
        for mut e in entries {
            let p = inner.last().map_or(LogProb::ln_zero(), |e| e.prob).ln_add_exp(e.prob);
            if !inner.is_empty() && inner.last().unwrap().value == e.value {
                inner.last_mut().unwrap().prob = p;
            }
            else {
                e.prob = p;
                inner.push(e);
            }
        }
        let mut cdf = CDF {
            inner: inner
        };

        if relative_eq!(*cdf.total_prob(), *LogProb::ln_one()) && *cdf.total_prob() > *LogProb::ln_one() {
            cdf.inner.last_mut().unwrap().prob = LogProb::ln_one();
        }

        cdf
    }

    /// Create CDF from iterator. This can be used to replace the values of a CDF.
    pub fn from_cdf<I: Iterator<Item = Entry<T>>>(entries: I) -> Self {
        CDF { inner: entries.collect_vec() }
    }

    /// Reduce CDF by omitting values with zero probability.
    pub fn reduce(self) -> Self {
        let mut inner = Vec::new();
        let mut last = LogProb::ln_zero();
        for e in self.inner {
            if last != e.prob {
                last = e.prob;
                inner.push(e);
            }
        }
        CDF { inner: inner }
    }

    /// Downsample CDF to n entries. Panics if n <= 1 and returns identity if n is greater
    /// than the number of entries.
    pub fn sample(mut self, n: usize) -> Self {
        assert!(n > 1);
        if self.inner.len() <= n {
            self
        }
        else {
            let s = self.inner.len() / (n - 1);
            let last = self.inner.pop().unwrap();
            let mut inner = self.inner.into_iter().step(s).collect_vec();
            inner.push(last);
            CDF {
                inner: inner
            }
        }
    }

    /// Provide iterator.
    pub fn iter(&self) -> slice::Iter<Entry<T>>{
        self.inner.iter()
    }

    /// Iterator over corresponding PMF.
    pub fn iter_pmf<'a>(&'a self) -> CDFPMFIter<'a, T> {
        fn cdf_to_pmf<'a, G: Ord>(last_prob: &mut LogProb, e: &'a Entry<G>) -> Option<Entry<&'a G>> {
            let prob = e.prob.ln_sub_exp(*last_prob);
            *last_prob = e.prob;
            Some(Entry::new(&e.value, prob))
        }
        self.inner.iter().scan(LogProb::ln_zero(), cdf_to_pmf)
    }

    /// Get cumulative probability for a given value. If the value is not present,
    /// return the probability of the previous value. Complexity O(log n).
    pub fn get(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {
            Some(match self.inner.binary_search_by(|e| e.value.cmp(value)) {
                Ok(i) => self.inner[i].prob,
                Err(i) => if i > 0 { self.inner[i - 1].prob } else { LogProb::ln_zero() }
            })
        }
    }

    /// Get probability (i.e. probability mass) for a given value. Complexity O(log n).
    pub fn get_pmf(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {
            Some(match self.inner.binary_search_by(|e| e.value.cmp(value)) {
                Ok(i) => if i > 0 { self.inner[i].prob.ln_sub_exp(self.inner[i - 1].prob) } else { self.inner[0].prob },
                Err(i) => if i > 0 { self.inner[i - 1].prob } else { LogProb::ln_zero() }
            })
        }
    }

    /// Return total probability.
    pub fn total_prob(&self) -> LogProb {
        self.inner.last().map_or(LogProb::ln_zero(), |e| e.prob)
    }

    /// Return maximum a posteriori probability estimate (MAP).
    pub fn map(&self) -> Option<&T> {
        if let Some(mut max) = self.iter_pmf().next() {
            for e in self.iter_pmf() {
                if e.prob >= max.prob {
                    max = e;
                }
            }
            Some(&max.value)
        } else {
            None
        }
    }

    /// Return w%-credible interval. The width w is a float between 0 and 1. Panics otherwise.
    /// E.g. provide `width=0.95` for the 95% credible interval.
    pub fn credible_interval(&self, width: f64) -> Range<&T> {
        assert!(width >= 0.0 && width <= 1.0);
        let margin = 1.0 - width;
        let p_lower = OrderedFloat((margin / 2.0).ln());
        let p_upper = OrderedFloat((1.0 - margin / 2.0).ln());
        let lower = self.inner.binary_search_by(|e| OrderedFloat(*e.prob).cmp(&p_lower)).unwrap_or_else(|i| i);
        let upper = self.inner.binary_search_by(|e| OrderedFloat(*e.prob).cmp(&p_upper)).unwrap_or_else(|i| i - 1);

        &self.inner[lower].value..&self.inner[upper].value
    }

    /// Number of entries in the CDF.
    pub fn len(&self) -> usize {
        self.inner.len()
    }
}

impl<T: NumCast + Clone + Ord> CDF<T> {
    /// Calculate expected value.
    pub fn expected_value(&self) -> f64 {
        self.iter_pmf().map(|e| {
            cast::<T, f64>(e.value.clone()).unwrap() * e.prob.exp()
        }).fold(0.0f64, |s, e| s + e)
    }

    /// Calculate variance.
    pub fn variance(&self) -> f64 {
        let ev = self.expected_value();
        self.iter_pmf().map(|e| {
                (cast::<T, f64>(e.value.clone()).unwrap() - ev).powi(2) * e.prob.exp()
        }).fold(0.0, |s, e| s + e)
    }

    /// Calculate standard deviation.
    pub fn standard_deviation(&self) -> f64 {
        self.variance().sqrt()
    }
}

pub type CDFPMFIter<'a, T> = iter::Scan<slice::Iter<'a, Entry<T>>, LogProb, fn(&mut LogProb, &'a Entry<T>) -> Option<Entry<&'a T>>>;

#[cfg(test)]
mod test {
    use super::*;
    use stats::LogProb;
    use ordered_float::NotNaN;

    #[test]
    fn test_cdf() {
        let mut pmf = vec![Entry::new(NotNaN::new(0.0).unwrap(), LogProb(0.1f64.ln()))];
        for i in 0..9 {
            pmf.push(Entry::new(NotNaN::new(i as f64).unwrap(), LogProb(0.1f64.ln())));
        }
        println!("{:?}", pmf);

        let cdf = CDF::from_pmf(pmf.clone());
        println!("{:?}", cdf);
        for e in pmf.iter().skip(2) {
            assert_ulps_eq!(*e.prob, *cdf.get_pmf(&e.value).unwrap(), epsilon = 0.0000000000001);
        }
        assert_relative_eq!(*cdf.total_prob(), 1.0f64.ln());
        assert_relative_eq!(*cdf.get(&NotNaN::new(1.0).unwrap()).unwrap(), 0.3f64.ln(), epsilon = 0.00000001);
        let ci = cdf.credible_interval(0.95);
        assert_relative_eq!(**ci.start, 0.0);
        assert_relative_eq!(**ci.end, 7.0);
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::new(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    pub fn iter_ids(&self, m: &Module) -> Result<IntoIter<FileID>, StorageBackendError>
    {
        let globstr = self.prefix_of_files_for_module(m) + "*.imag";
        debug!("Globstring = {}", globstr);
        glob(&globstr[..])
            .and_then(|globlist| {
                debug!("Iterating over globlist");
                Ok(globlist.filter_map(Result::ok)
                           .map(|pbuf| FileID::from(&pbuf))
                           .collect::<Vec<FileID>>()
                           .into_iter())
            })
            .map_err(|e| {
                debug!("glob() returned error: {:?}", e);
                let serr = StorageBackendError::new(
                        "iter_ids()",
                        "Cannot iter on file ids",
                        None);
                // Why the hack is Error not implemented for glob::PatternError
                // serr.caused_by = Some(Box::new(e));
                serr
            })
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Result<IntoIter<File<'a>>, StorageBackendError>
        where HP: FileHeaderParser
    {
        self.iter_ids(m)
            .and_then(|ids| {
                debug!("Iterating ids and building files from them");
                debug!("  number of ids = {}", ids.len());
                Ok(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                        .collect::<Vec<File>>()
                        .into_iter())
            })
            .map_err(|e| {
                debug!("StorageBackend::iter_ids() returned error = {:?}", e);
                let mut serr = StorageBackendError::new("iter_files()",
                                           "Cannot iter on files",
                                           None);
                serr.caused_by = Some(Box::new(e));
                serr
            })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        use std::ops::Index;

        debug!("Searching for file with id '{}'", id);

        if id.get_type() == FileIDType::NONE {
            // We don't know the hash type, so we glob() around a bit.

            let globstr = self.prefix_of_files_for_module(m) + "*" + ".imag";
            glob(&globstr[..]).map(|globlist| {
                let mut vec = globlist.filter_map(Result::ok)
                                      .map(|pbuf| FileID::from(&pbuf))
                                      .filter_map(|id| self.get_file_by_id(m, &id, p))
                                      .collect::<Vec<File>>();
                vec.reverse();
                vec.pop()
            }).map_err(|e| e).unwrap()
        } else {
            // The (hash)type is already in the FileID object, so we can just
            // build a path from the information we already have
            if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
                let mut s = String::new();
                fs.read_to_string(&mut s);
                debug!("Success opening file with id '{}'", id);
                debug!("Parsing to internal structure now");
                p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
            } else {
                debug!("No file with id '{}'", id);
                None
            }
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::new(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        let idstr   : String        = id.clone().into();
        let idtype  : FileIDType    = id.into();
        let typestr : String        = idtype.into();

        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("        id: '{}'", idstr);
        debug!("      type: '{}'", typestr);

        self.prefix_of_files_for_module(owner) +
            "-" + &typestr[..] +
            "-" + &idstr[..] +
            ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {

    fn new<S>(action: S, desc: S, data: Option<String>) -> StorageBackendError
        where S: Into<String>
    {
        StorageBackendError {
            action:         action.into(),
            desc:           desc.into(),
            data_dump:      data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::new(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Outsource globlist -> Vec<FileID> transformation code snippet<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::new(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    pub fn iter_ids(&self, m: &Module) -> Result<IntoIter<FileID>, StorageBackendError>
    {
        let globstr = self.prefix_of_files_for_module(m) + "*.imag";
        debug!("Globstring = {}", globstr);
        glob(&globstr[..])
            .and_then(|globlist| {
                debug!("Iterating over globlist");
                Ok(globlist_to_file_id_vec(globlist).into_iter())
            })
            .map_err(|e| {
                debug!("glob() returned error: {:?}", e);
                let serr = StorageBackendError::new(
                        "iter_ids()",
                        "Cannot iter on file ids",
                        None);
                // Why the hack is Error not implemented for glob::PatternError
                // serr.caused_by = Some(Box::new(e));
                serr
            })
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Result<IntoIter<File<'a>>, StorageBackendError>
        where HP: FileHeaderParser
    {
        self.iter_ids(m)
            .and_then(|ids| {
                debug!("Iterating ids and building files from them");
                debug!("  number of ids = {}", ids.len());
                Ok(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                        .collect::<Vec<File>>()
                        .into_iter())
            })
            .map_err(|e| {
                debug!("StorageBackend::iter_ids() returned error = {:?}", e);
                let mut serr = StorageBackendError::new("iter_files()",
                                           "Cannot iter on files",
                                           None);
                serr.caused_by = Some(Box::new(e));
                serr
            })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        use std::ops::Index;

        debug!("Searching for file with id '{}'", id);

        if id.get_type() == FileIDType::NONE {
            // We don't know the hash type, so we glob() around a bit.

            let globstr = self.prefix_of_files_for_module(m) + "*" + ".imag";
            glob(&globstr[..]).map(|globlist| {
                let mut vec = globlist_to_file_id_vec(globlist).into_iter()
                                .filter_map(|id| self.get_file_by_id(m, &id, p))
                                .collect::<Vec<File>>();
                vec.reverse();
                vec.pop()
            }).map_err(|e| e).unwrap()
        } else {
            // The (hash)type is already in the FileID object, so we can just
            // build a path from the information we already have
            if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
                let mut s = String::new();
                fs.read_to_string(&mut s);
                debug!("Success opening file with id '{}'", id);
                debug!("Parsing to internal structure now");
                p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
            } else {
                debug!("No file with id '{}'", id);
                None
            }
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::new(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        let idstr   : String        = id.clone().into();
        let idtype  : FileIDType    = id.into();
        let typestr : String        = idtype.into();

        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("        id: '{}'", idstr);
        debug!("      type: '{}'", typestr);

        self.prefix_of_files_for_module(owner) +
            "-" + &typestr[..] +
            "-" + &idstr[..] +
            ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {

    fn new<S>(action: S, desc: S, data: Option<String>) -> StorageBackendError
        where S: Into<String>
    {
        StorageBackendError {
            action:         action.into(),
            desc:           desc.into(),
            data_dump:      data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::new(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}

fn globlist_to_file_id_vec(globlist: Paths) -> Vec<FileID> {
    globlist.filter_map(Result::ok)
            .map(|pbuf| FileID::from(&pbuf))
            .collect::<Vec<FileID>>()
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.storepath.clone() + owner.name() + "-" + &id[..] + ".imag"
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
             data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Add helper for building file prefix<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
             data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult = Result<(), StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
}

impl StorageBackend {

    pub fn new(basepath: String) -> StorageBackend {
        StorageBackend {
            basepath: basepath,
        }
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self) -> Option<Vec<FileID>> {
        debug!("Getting files from {}", self.basepath);
        let list = glob(&self.basepath[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, id: FileID, p: &Parser<HP>) -> Option<File>
        where HP: FileHeaderParser<'a>
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Fixup: get_file_by_id() gets &Module as well, so it can use build_filepath_with_id()<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult = Result<(), StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
}

impl StorageBackend {

    pub fn new(basepath: String) -> StorageBackend {
        StorageBackend {
            basepath: basepath,
        }
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self) -> Option<Vec<FileID>> {
        debug!("Getting files from {}", self.basepath);
        let list = glob(&self.basepath[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(id, h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate diesel;

use diesel::*;

table! {
    int_primary_key {
        id -> Serial,
    }
}

table! {
    string_primary_key {
        id -> VarChar,
    }
}

fn main() {
    let connection = Connection::establish("").unwrap();
    let one = connection.find(int_primary_key::table, "1".to_string()).unwrap();
    //~^ ERROR E0277
    //~| ERROR E0277
    let string = connection.find(string_primary_key::table, 1).unwrap();
    //~^ ERROR E0277
    //~| ERROR E0277
}
<commit_msg>Add extra compile fail error<commit_after>#[macro_use]
extern crate diesel;

use diesel::*;

table! {
    int_primary_key {
        id -> Serial,
    }
}

table! {
    string_primary_key {
        id -> VarChar,
    }
}

fn main() {
    let connection = Connection::establish("").unwrap();
    let one = connection.find(int_primary_key::table, "1".to_string()).unwrap();
    //~^ ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
    let string = connection.find(string_primary_key::table, 1).unwrap();
    //~^ ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
    //~| ERROR E0277
}
<|endoftext|>