<commit_before>use std::vec;
use std::slice;
use std::borrow::Borrow;
use std::default::Default;
use std::hash::Hash;
use std::hash::Hasher;
use std::iter::FromIterator;
use std::iter::IntoIterator;
use std::ops::Index;
use std::ops::IndexMut;
use std::ops::Deref;
use std::ops::DerefMut;
use std::cmp::Ordering;
use std::fmt;

use clear::Clear;

pub struct RepeatedField<T> {
    vec: Vec<T>,
    len: usize,
}

impl<T> RepeatedField<T> {
    #[inline]
    pub fn len(&self) -> usize {
        self.len
    }

    #[inline]
    pub fn clear(&mut self) {
        self.len = 0;
    }
}

impl<T> Clear for RepeatedField<T> {
    #[inline]
    fn clear(&mut self) {
        self.len = 0;
    }
}

impl<T> Default for RepeatedField<T> {
    #[inline]
    fn default() -> RepeatedField<T> {
        RepeatedField {
            vec: Vec::new(),
            len: 0,
        }
    }
}

impl<T> RepeatedField<T> {
    #[inline]
    pub fn new() -> RepeatedField<T> {
        Default::default()
    }

    #[inline]
    pub fn from_vec(vec: Vec<T>) -> RepeatedField<T> {
        let len = vec.len();
        RepeatedField { vec: vec, len: len }
    }

    #[inline]
    pub fn into_vec(self) -> Vec<T> {
        let mut vec = self.vec;
        vec.truncate(self.len);
        vec
    }

    #[inline]
    pub fn capacity(&self) -> usize {
        self.vec.capacity()
    }

    #[inline]
    pub fn as_mut_slice<'a>(&'a mut self) -> &'a mut [T] {
        &mut self.vec[..self.len]
    }

    #[inline]
    pub fn slice<'a>(&'a self, start: usize, end: usize) -> &'a [T] {
        &self.as_ref()[start..end]
    }

    #[inline]
    pub fn slice_mut<'a>(&'a mut self, start: usize, end: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[start..end]
    }

    #[inline]
    pub fn slice_from<'a>(&'a self, start: usize) -> &'a [T] {
        &self.as_ref()[start..]
    }

    #[inline]
    pub fn slice_from_mut<'a>(&'a mut self, start: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[start..]
    }

    #[inline]
    pub fn slice_to<'a>(&'a self, end: usize) -> &'a [T] {
        &self.as_ref()[..end]
    }

    #[inline]
    pub fn slice_to_mut<'a>(&'a mut self, end: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[..end]
    }

    #[inline]
    pub fn split_at<'a>(&'a self, mid: usize) -> (&'a [T], &'a [T]) {
        self.as_ref().split_at(mid)
    }

    #[inline]
    pub fn split_at_mut<'a>(&'a mut self, mid: usize) -> (&'a mut [T], &'a mut [T]) {
        self.as_mut_slice().split_at_mut(mid)
    }

    #[inline]
    pub fn tail<'a>(&'a self) -> &'a [T] {
        &self.as_ref()[1..]
    }

    #[inline]
    pub fn last<'a>(&'a self) -> Option<&'a T> {
        self.as_ref().last()
    }

    #[inline]
    pub fn last_mut<'a>(&'a mut self) -> Option<&'a mut T> {
        self.as_mut_slice().last_mut()
    }

    #[inline]
    pub fn init<'a>(&'a self) -> &'a [T] {
        let s = self.as_ref();
        &s[0..s.len() - 1]
    }

    #[inline]
    pub fn push(&mut self, value: T) {
        if self.len == self.vec.len() {
            self.vec.push(value);
        } else {
            self.vec[self.len] = value;
        }
        self.len += 1;
    }

    #[inline]
    pub fn pop(&mut self) -> Option<T> {
        if self.len == 0 {
            None
        } else {
            self.vec.truncate(self.len);
            self.len -= 1;
            self.vec.pop()
        }
    }

    #[inline]
    pub fn insert(&mut self, index: usize, value: T) {
        assert!(index <= self.len);
        self.vec.insert(index, value);
        self.len += 1;
    }

    #[inline]
    pub fn remove(&mut self, index: usize) -> T {
        assert!(index < self.len);
        self.len -= 1;
        self.vec.remove(index)
    }

    #[inline]
    pub fn truncate(&mut self, len: usize) {
        if self.len > len {
            self.len = len;
        }
    }

    #[inline]
    pub fn reverse(&mut self) {
        self.as_mut_slice().reverse()
    }

    #[inline]
    pub fn into_iter(mut self) -> vec::IntoIter<T> {
        self.vec.truncate(self.len);
        self.vec.into_iter()
    }

    #[inline]
    pub fn iter<'a>(&'a self) -> slice::Iter<'a, T> {
        self.as_ref().iter()
    }

    #[inline]
    pub fn iter_mut<'a>(&'a mut self) -> slice::IterMut<'a, T> {
        self.as_mut_slice().iter_mut()
    }

    #[inline]
    pub fn sort_by<F>(&mut self, compare: F)
    where
        F : Fn(&T, &T) -> Ordering,
    {
        self.as_mut_slice().sort_by(compare)
    }

    #[inline]
    pub fn as_ptr(&self) -> *const T {
        self.vec.as_ptr()
    }

    #[inline]
    pub fn as_mut_ptr(&mut self) -> *mut T {
        self.vec.as_mut_ptr()
    }
}

impl<T : Default + Clear> RepeatedField<T> {
    pub fn push_default<'a>(&'a mut self) -> &'a mut T {
        if self.len == self.vec.len() {
            self.vec.push(Default::default());
        } else {
            self.vec[self.len].clear();
        }
        self.len += 1;
        self.last_mut().unwrap()
    }
}

impl<T : Clone> RepeatedField<T> {
    // TODO: implement to_vec()
    #[inline]
    pub fn from_slice(values: &[T]) -> RepeatedField<T> {
        RepeatedField::from_vec(values.to_vec())
    }
}

impl<T : Clone> Clone for RepeatedField<T> {
    #[inline]
    fn clone(&self) -> RepeatedField<T> {
        RepeatedField {
            vec: self.to_vec(),
            len: self.len(),
        }
    }
}

impl<T> FromIterator<T> for RepeatedField<T> {
    #[inline]
    fn from_iter<I : IntoIterator<Item = T>>(iter: I) -> RepeatedField<T> {
        RepeatedField::from_vec(FromIterator::from_iter(iter))
    }
}

impl<'a, T> IntoIterator for &'a RepeatedField<T> {
    type Item = &'a T;
    type IntoIter = slice::Iter<'a, T>;

    fn into_iter(self) -> slice::Iter<'a, T> {
        self.iter()
    }
}

impl<T : PartialEq> PartialEq for RepeatedField<T> {
    #[inline]
    fn eq(&self, other: &RepeatedField<T>) -> bool {
        self.as_ref() == other.as_ref()
    }
}

impl<T : Eq> Eq for RepeatedField<T> {}

impl<T : PartialEq> RepeatedField<T> {
    #[inline]
    pub fn contains(&self, value: &T) -> bool {
        self.as_ref().contains(value)
    }
}

impl<T : Hash> Hash for RepeatedField<T> {
    fn hash<H : Hasher>(&self, state: &mut H) {
        self.as_ref().hash(state);
    }
}

impl<T> AsRef<[T]> for RepeatedField<T> {
    #[inline]
    fn as_ref<'a>(&'a self) -> &'a [T] {
        &self.vec[..self.len]
    }
}

impl<T> Borrow<[T]> for RepeatedField<T> {
    #[inline]
    fn borrow(&self) -> &[T] {
        &self.vec[..self.len]
    }
}

impl<T> Deref for RepeatedField<T> {
    type Target = [T];
    #[inline]
    fn deref(&self) -> &[T] {
        &self.vec[..self.len]
    }
}

impl<T> DerefMut for RepeatedField<T> {
    #[inline]
    fn deref_mut(&mut self) -> &mut [T] {
        &mut self.vec[..self.len]
    }
}

impl<T> Index<usize> for RepeatedField<T> {
    type Output = T;

    #[inline]
    fn index<'a>(&'a self, index: usize) -> &'a T {
        &self.as_ref()[index]
    }
}

impl<T> IndexMut<usize> for RepeatedField<T> {
    #[inline]
    fn index_mut<'a>(&'a mut self, index: usize) -> &'a mut T {
        &mut self.as_mut_slice()[index]
    }
}

impl<T : fmt::Debug> fmt::Debug for RepeatedField<T> {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.as_ref().fmt(f)
    }
}

#[cfg(test)]
mod test {
    use super::RepeatedField;

    #[test]
    fn as_mut_slice() {
        let mut v = RepeatedField::new();
        v.push(10);
        v.push(20);
        v.clear();
        assert_eq!(v.as_mut_slice(), &mut []);
        v.push(30);
        assert_eq!(v.as_mut_slice(), &mut [30]);
    }

    #[test]
    fn push_default() {
        let mut v = RepeatedField::new();
        v.push("aa".to_string());
        v.push("bb".to_string());
        v.clear();
        assert_eq!("".to_string(), *v.push_default());
    }
}
<commit_msg>Added a few new traits to repeated<commit_after>use std::vec;
use std::slice;
use std::borrow::Borrow;
use std::default::Default;
use std::hash::Hash;
use std::hash::Hasher;
use std::iter::FromIterator;
use std::iter::IntoIterator;
use std::ops::Index;
use std::ops::IndexMut;
use std::ops::Deref;
use std::ops::DerefMut;
use std::cmp::Ordering;
use std::fmt;

use clear::Clear;

pub struct RepeatedField<T> {
    vec: Vec<T>,
    len: usize,
}

impl<T> RepeatedField<T> {
    #[inline]
    pub fn len(&self) -> usize {
        self.len
    }

    #[inline]
    pub fn clear(&mut self) {
        self.len = 0;
    }
}

impl<T> Clear for RepeatedField<T> {
    #[inline]
    fn clear(&mut self) {
        self.len = 0;
    }
}

impl<T> Default for RepeatedField<T> {
    #[inline]
    fn default() -> RepeatedField<T> {
        RepeatedField {
            vec: Vec::new(),
            len: 0,
        }
    }
}

impl<T> RepeatedField<T> {
    #[inline]
    pub fn new() -> RepeatedField<T> {
        Default::default()
    }

    #[inline]
    pub fn from_vec(vec: Vec<T>) -> RepeatedField<T> {
        let len = vec.len();
        RepeatedField { vec: vec, len: len }
    }

    #[inline]
    pub fn into_vec(self) -> Vec<T> {
        let mut vec = self.vec;
        vec.truncate(self.len);
        vec
    }

    #[inline]
    pub fn capacity(&self) -> usize {
        self.vec.capacity()
    }

    #[inline]
    pub fn as_mut_slice<'a>(&'a mut self) -> &'a mut [T] {
        &mut self.vec[..self.len]
    }

    #[inline]
    pub fn slice<'a>(&'a self, start: usize, end: usize) -> &'a [T] {
        &self.as_ref()[start..end]
    }

    #[inline]
    pub fn slice_mut<'a>(&'a mut self, start: usize, end: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[start..end]
    }

    #[inline]
    pub fn slice_from<'a>(&'a self, start: usize) -> &'a [T] {
        &self.as_ref()[start..]
    }

    #[inline]
    pub fn slice_from_mut<'a>(&'a mut self, start: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[start..]
    }

    #[inline]
    pub fn slice_to<'a>(&'a self, end: usize) -> &'a [T] {
        &self.as_ref()[..end]
    }

    #[inline]
    pub fn slice_to_mut<'a>(&'a mut self, end: usize) -> &'a mut [T] {
        &mut self.as_mut_slice()[..end]
    }

    #[inline]
    pub fn split_at<'a>(&'a self, mid: usize) -> (&'a [T], &'a [T]) {
        self.as_ref().split_at(mid)
    }

    #[inline]
    pub fn split_at_mut<'a>(&'a mut self, mid: usize) -> (&'a mut [T], &'a mut [T]) {
        self.as_mut_slice().split_at_mut(mid)
    }

    #[inline]
    pub fn tail<'a>(&'a self) -> &'a [T] {
        &self.as_ref()[1..]
    }

    #[inline]
    pub fn last<'a>(&'a self) -> Option<&'a T> {
        self.as_ref().last()
    }

    #[inline]
    pub fn last_mut<'a>(&'a mut self) -> Option<&'a mut T> {
        self.as_mut_slice().last_mut()
    }

    #[inline]
    pub fn init<'a>(&'a self) -> &'a [T] {
        let s = self.as_ref();
        &s[0..s.len() - 1]
    }

    #[inline]
    pub fn push(&mut self, value: T) {
        if self.len == self.vec.len() {
            self.vec.push(value);
        } else {
            self.vec[self.len] = value;
        }
        self.len += 1;
    }

    #[inline]
    pub fn pop(&mut self) -> Option<T> {
        if self.len == 0 {
            None
        } else {
            self.vec.truncate(self.len);
            self.len -= 1;
            self.vec.pop()
        }
    }

    #[inline]
    pub fn insert(&mut self, index: usize, value: T) {
        assert!(index <= self.len);
        self.vec.insert(index, value);
        self.len += 1;
    }

    #[inline]
    pub fn remove(&mut self, index: usize) -> T {
        assert!(index < self.len);
        self.len -= 1;
        self.vec.remove(index)
    }

    #[inline]
    pub fn truncate(&mut self, len: usize) {
        if self.len > len {
            self.len = len;
        }
    }

    #[inline]
    pub fn reverse(&mut self) {
        self.as_mut_slice().reverse()
    }

    #[inline]
    pub fn into_iter(mut self) -> vec::IntoIter<T> {
        self.vec.truncate(self.len);
        self.vec.into_iter()
    }

    #[inline]
    pub fn iter<'a>(&'a self) -> slice::Iter<'a, T> {
        self.as_ref().iter()
    }

    #[inline]
    pub fn iter_mut<'a>(&'a mut self) -> slice::IterMut<'a, T> {
        self.as_mut_slice().iter_mut()
    }

    #[inline]
    pub fn sort_by<F>(&mut self, compare: F)
    where
        F : Fn(&T, &T) -> Ordering,
    {
        self.as_mut_slice().sort_by(compare)
    }

    #[inline]
    pub fn as_ptr(&self) -> *const T {
        self.vec.as_ptr()
    }

    #[inline]
    pub fn as_mut_ptr(&mut self) -> *mut T {
        self.vec.as_mut_ptr()
    }
}

impl<T : Default + Clear> RepeatedField<T> {
    pub fn push_default<'a>(&'a mut self) -> &'a mut T {
        if self.len == self.vec.len() {
            self.vec.push(Default::default());
        } else {
            self.vec[self.len].clear();
        }
        self.len += 1;
        self.last_mut().unwrap()
    }
}

impl<T> From<Vec<T>> for RepeatedField<T> {
    #[inline]
    fn from(values: Vec<T>) -> RepeatedField<T> {
        RepeatedField::from_vec(values)
    }
}

impl<'a, T : Clone> From<&'a [T]> for RepeatedField<T> {
    #[inline]
    fn from(values: &'a [T]) -> RepeatedField<T> {
        RepeatedField::from_slice(values)
    }
}

impl<T> Into<Vec<T>> for RepeatedField<T> {
    #[inline]
    fn into(self) -> Vec<T> {
        self.into_vec()
    }
}

impl<T : Clone> RepeatedField<T> {

    #[inline]
    pub fn from_slice(values: &[T]) -> RepeatedField<T> {
        RepeatedField::from_vec(values.to_vec())
    }

    #[inline]
    pub fn from_ref<X: AsRef<[T]>>(values: X) -> RepeatedField<T> {
        RepeatedField::from_slice(values.as_ref())
    }

    #[inline]
    pub fn to_vec(&self) -> Vec<T> {
        self.as_ref().to_vec()
    }
}

impl<T : Clone> Clone for RepeatedField<T> {
    #[inline]
    fn clone(&self) -> RepeatedField<T> {
        RepeatedField {
            vec: self.to_vec(),
            len: self.len(),
        }
    }
}

impl<T> FromIterator<T> for RepeatedField<T> {
    #[inline]
    fn from_iter<I : IntoIterator<Item = T>>(iter: I) -> RepeatedField<T> {
        RepeatedField::from_vec(FromIterator::from_iter(iter))
    }
}

impl<'a, T> IntoIterator for &'a RepeatedField<T> {
    type Item = &'a T;
    type IntoIter = slice::Iter<'a, T>;

    fn into_iter(self) -> slice::Iter<'a, T> {
        self.iter()
    }
}

impl<T : PartialEq> PartialEq for RepeatedField<T> {
    #[inline]
    fn eq(&self, other: &RepeatedField<T>) -> bool {
        self.as_ref() == other.as_ref()
    }
}

impl<T : Eq> Eq for RepeatedField<T> {}

impl<T : PartialEq> RepeatedField<T> {
    #[inline]
    pub fn contains(&self, value: &T) -> bool {
        self.as_ref().contains(value)
    }
}

impl<T : Hash> Hash for RepeatedField<T> {
    fn hash<H : Hasher>(&self, state: &mut H) {
        self.as_ref().hash(state);
    }
}

impl<T> AsRef<[T]> for RepeatedField<T> {
    #[inline]
    fn as_ref<'a>(&'a self) -> &'a [T] {
        &self.vec[..self.len]
    }
}

impl<T> Borrow<[T]> for RepeatedField<T> {
    #[inline]
    fn borrow(&self) -> &[T] {
        &self.vec[..self.len]
    }
}

impl<T> Deref for RepeatedField<T> {
    type Target = [T];
    #[inline]
    fn deref(&self) -> &[T] {
        &self.vec[..self.len]
    }
}

impl<T> DerefMut for RepeatedField<T> {
    #[inline]
    fn deref_mut(&mut self) -> &mut [T] {
        &mut self.vec[..self.len]
    }
}

impl<T> Index<usize> for RepeatedField<T> {
    type Output = T;

    #[inline]
    fn index<'a>(&'a self, index: usize) -> &'a T {
        &self.as_ref()[index]
    }
}

impl<T> IndexMut<usize> for RepeatedField<T> {
    #[inline]
    fn index_mut<'a>(&'a mut self, index: usize) -> &'a mut T {
        &mut self.as_mut_slice()[index]
    }
}

impl<T : fmt::Debug> fmt::Debug for RepeatedField<T> {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.as_ref().fmt(f)
    }
}

#[cfg(test)]
mod test {
    use super::RepeatedField;

    #[test]
    fn as_mut_slice() {
        let mut v = RepeatedField::new();
        v.push(10);
        v.push(20);
        v.clear();
        assert_eq!(v.as_mut_slice(), &mut []);
        v.push(30);
        assert_eq!(v.as_mut_slice(), &mut [30]);
    }

    #[test]
    fn push_default() {
        let mut v = RepeatedField::new();
        v.push("aa".to_string());
        v.push("bb".to_string());
        v.clear();
        assert_eq!("".to_string(), *v.push_default());
    }
}
<|endoftext|>


<commit_before>extern crate easy_ll;
extern crate rustyline;
extern crate weld;

use rustyline::error::ReadlineError;
use rustyline::Editor;
use weld::ast::ExprKind::*;
use weld::llvm::LlvmGenerator;
use weld::macro_processor;
use weld::parser::*;
use weld::pretty_print::*;
use weld::transforms;
use weld::type_inference::*;

fn main() {
    let mut rl = Editor::<()>::new();
    if let Err(_) = rl.load_history("history.txt") {
        println!("No previous history.");
    }
    loop {
        let readline = rl.readline(">> ");
        match readline {
            Ok(line) => {
                rl.add_history_entry(&line);

                let trimmed = line.trim();
                if trimmed == "" {
                    continue;
                }

                let program = parse_program(trimmed);
                if let Err(ref e) = program {
                    println!("Error during parsing: {:?}", e);
                    continue;
                }
                let program = program.unwrap();
                println!("Raw structure:\n{:?}\n", program);

                let expr = macro_processor::process_program(&program);
                if let Err(ref e) = expr {
                    println!("Error during macro substitution: {}", e);
                    continue;
                }
                let mut expr = expr.unwrap();
                println!("After macro substitution:\n{}\n", print_expr(&expr));

                if let Err(ref e) = transforms::inline_apply(&mut expr) {
                    println!("Error during inlining applies: {}\n", e);
                }
                println!("After inlining applies:\n{}\n", print_expr(&expr));

                if let Err(ref e) = infer_types(&mut expr) {
                    println!("Error during type inference: {}\n", e);
                    println!("Partially inferred types:\n{}\n", print_typed_expr(&expr));
                    continue;
                }
                println!("After type inference:\n{}\n", print_typed_expr(&expr));
                println!("Expression type: {}\n", print_type(&expr.ty));

                let expr = expr.to_typed().unwrap();
                if let Lambda(ref args, ref body) = expr.kind {
                    let mut generator = LlvmGenerator::new();
                    if let Err(ref e) = generator.add_function_on_pointers("run", args, body) {
                        println!("Error during LLVM code gen:\n{}\n", e);
                        continue;
                    }
                    let llvm_code = generator.result();
                    println!("LLVM code:\n{}\n", llvm_code);

                    if let Err(ref e) = easy_ll::compile_module(&llvm_code) {
                        println!("Error during LLVM compilation:\n{}\n", e);
                        continue;
                    }
                    println!("LLVM module compiled successfully\n")
                } else {
                    println!("Expression is not a function, so not compiling to LLVM.\n")
                }
            },
            Err(ReadlineError::Interrupted) => {
                println!("Exiting!");
                break
            },
            Err(ReadlineError::Eof) => {
                println!("Exiting!");
                break
            },
            Err(err) => {
                println!("Error: {:?}", err);
                break
            }
        }
    }
}
<commit_msg>Default weld_history file to home directory, move code from within match<commit_after>extern crate easy_ll;
extern crate rustyline;
extern crate weld;

use rustyline::error::ReadlineError;
use rustyline::Editor;
use std::env;
use std::path::PathBuf;
use weld::ast::ExprKind::*;
use weld::llvm::LlvmGenerator;
use weld::macro_processor;
use weld::parser::*;
use weld::pretty_print::*;
use weld::transforms;
use weld::type_inference::*;

fn main() {
    let home_path = match env::home_dir() {
        Some(path) => path,
        None => PathBuf::new(),
    };
    let history_file_path = home_path.join(".weld_history");
    let history_file_path = match history_file_path.to_str() {
        Some(history_file_path) => history_file_path,
        None => ".weld_history",
    };

    let mut rl = Editor::<()>::new();
    if let Err(_) = rl.load_history(&history_file_path) {
        println!("No previous history.");
    }

    loop {
        let raw_readline = rl.readline(">> ");
        let readline;
        match raw_readline {
            Ok(raw_readline) => {
                rl.add_history_entry(&raw_readline);
                readline = raw_readline;
            },
            Err(ReadlineError::Interrupted) => {
                println!("Exiting!");
                break
            },
            Err(ReadlineError::Eof) => {
                println!("Exiting!");
                break
            },
            Err(err) => {
                println!("Error: {:?}", err);
                break
            }
        }

        let trimmed = readline.trim();
        if trimmed == "" {
            continue;
        }

        let program = parse_program(trimmed);
        if let Err(ref e) = program {
            println!("Error during parsing: {:?}", e);
            continue;
        }
        let program = program.unwrap();
        println!("Raw structure:\n{:?}\n", program);

        let expr = macro_processor::process_program(&program);
        if let Err(ref e) = expr {
            println!("Error during macro substitution: {}", e);
            continue;
        }
        let mut expr = expr.unwrap();
        println!("After macro substitution:\n{}\n", print_expr(&expr));

        if let Err(ref e) = transforms::inline_apply(&mut expr) {
            println!("Error during inlining applies: {}\n", e);
        }
        println!("After inlining applies:\n{}\n", print_expr(&expr));

        if let Err(ref e) = infer_types(&mut expr) {
            println!("Error during type inference: {}\n", e);
            println!("Partially inferred types:\n{}\n", print_typed_expr(&expr));
            continue;
        }
        println!("After type inference:\n{}\n", print_typed_expr(&expr));
        println!("Expression type: {}\n", print_type(&expr.ty));

        let expr = expr.to_typed().unwrap();
        if let Lambda(ref args, ref body) = expr.kind {
            let mut generator = LlvmGenerator::new();
            if let Err(ref e) = generator.add_function_on_pointers("run", args, body) {
                println!("Error during LLVM code gen:\n{}\n", e);
                continue;
            }
            let llvm_code = generator.result();
            println!("LLVM code:\n{}\n", llvm_code);

            if let Err(ref e) = easy_ll::compile_module(&llvm_code) {
                println!("Error during LLVM compilation:\n{}\n", e);
                continue;
            }
            println!("LLVM module compiled successfully\n")
        } else {
            println!("Expression is not a function, so not compiling to LLVM.\n")
        }
    }
    rl.save_history(&history_file_path).unwrap();
}
<|endoftext|>


<commit_before>//
// GLSL Mathematics for Rust.
//
// Copyright (c) 2015 The glm-rs authors.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use num::{ BaseFloat, One, Zero };
use vec::vec::{ Vector2, Vector3, Vector4 };
use super::tpe::{ GenMat, GenSquareMat };
use super::mat::*;
use std::num::Float;

impl<T: BaseFloat> One for Matrix2<T> {
    #[inline]
    fn one() -> Matrix2<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix2::new(
            Vector2::new(y, l),
            Vector2::new(l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector2<T>> for Matrix2<T> {
    #[inline(always)]
    fn determinant(&self) -> T {
        self[0][0] * self[1][1] - self[0][1] * self[1][0]
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix2<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let m = Matrix2::new(
                Vector2::new(self[1][1] * inv_det, -self[0][1] * inv_det),
                Vector2::new(-self[1][0] * inv_det, self[0][0] * inv_det)
            );
            Some(m)
        }
    }
}

impl<T: BaseFloat> One for Matrix3<T> {
    #[inline]
    fn one() -> Matrix3<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix3::new(
            Vector3::new(y, l, l),
            Vector3::new(l, y, l),
            Vector3::new(l, l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector3<T>> for Matrix3<T> {
    #[inline]
    fn determinant(&self) -> T {
        self[0][0] * (self[1][1] * self[2][2] - self[2][1] * self[1][2]) -
        self[1][0] * (self[0][1] * self[2][2] - self[2][1] * self[0][2]) +
        self[2][0] * (self[0][1] * self[1][2] - self[1][1] * self[0][2])
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix3<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let r11 = self[1][1] * self[2][2] - self[2][1] * self[1][2];
            let r12 = self[2][0] * self[1][2] - self[1][0] * self[2][2];
            let r13 = self[1][0] * self[2][1] - self[2][0] * self[1][1];
            let r21 = self[2][1] * self[0][2] - self[0][1] * self[2][2];
            let r22 = self[0][0] * self[2][2] - self[2][0] * self[0][2];
            let r23 = self[2][0] * self[0][1] - self[0][0] * self[2][1];
            let r31 = self[0][1] * self[1][2] - self[1][1] * self[0][2];
            let r32 = self[1][0] * self[0][2] - self[0][0] * self[1][2];
            let r33 = self[0][0] * self[1][1] - self[1][0] * self[0][1];
            let m = Matrix3::new(
                Vector3::new(r11 * inv_det, r21 * inv_det, r31 * inv_det),
                Vector3::new(r12 * inv_det, r22 * inv_det, r32 * inv_det),
                Vector3::new(r13 * inv_det, r23 * inv_det, r33 * inv_det)
            );
            Some(m)
        }
    }
}

impl<T: BaseFloat> One for Matrix4<T> {
    #[inline]
    fn one() -> Matrix4<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix4::new(
            Vector4::new(y, l, l, l),
            Vector4::new(l, y, l, l),
            Vector4::new(l, l, y, l),
            Vector4::new(l, l, l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector4<T>> for Matrix4<T> {
    #[inline]
    fn determinant(&self) -> T {
        self[0][0] * (
            self[1][1] * self[2][2] * self[3][3] +
            self[2][1] * self[3][2] * self[1][3] +
            self[3][1] * self[1][2] * self[2][3] -
            self[3][1] * self[2][2] * self[1][3] -
            self[1][1] * self[3][2] * self[2][3] -
            self[2][1] * self[1][2] * self[3][3]
        ) -
        self[1][0] * (
            self[0][1] * self[2][2] * self[3][3] +
            self[2][1] * self[3][2] * self[0][3] +
            self[3][1] * self[0][2] * self[2][3] -
            self[3][1] * self[2][2] * self[0][3] -
            self[0][1] * self[3][2] * self[2][3] -
            self[2][1] * self[0][2] * self[3][3]
        ) +
        self[2][0] * (
            self[0][1] * self[1][2] * self[3][3] +
            self[1][1] * self[3][2] * self[0][3] +
            self[3][1] * self[0][2] * self[1][3] -
            self[3][1] * self[1][2] * self[0][3] -
            self[0][1] * self[3][2] * self[1][3] -
            self[1][1] * self[0][2] * self[3][3]
        ) -
        self[3][0] * (
            self[0][1] * self[1][2] * self[2][3] +
            self[1][1] * self[2][2] * self[0][3] +
            self[2][1] * self[0][2] * self[1][3] -
            self[2][1] * self[1][2] * self[0][3] -
            self[0][1] * self[2][2] * self[1][3] -
            self[1][1] * self[0][2] * self[2][3]
        )
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix4<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let tr = self.transpose();
            let cf = |&: i, j| -> T {
                let mat = match i {
                    0 => Matrix3::new(
                        tr.c1.truncate(j),
                        tr.c2.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    1 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c2.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    2 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c1.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    3 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c1.truncate(j),
                        tr.c2.truncate(j)
                    ),
                    _ => unreachable!(),
                };
                let d = mat.determinant() * inv_det;
                if (i + j) & 1 == 1 {
                    -d
                } else {
                    d
                }
            };
            let m = Matrix4::new(
                Vector4::new(cf(0, 0), cf(0, 1), cf(0, 2), cf(0, 3)),
                Vector4::new(cf(1, 0), cf(1, 1), cf(1, 2), cf(1, 3)),
                Vector4::new(cf(2, 0), cf(2, 1), cf(2, 2), cf(2, 3)),
                Vector4::new(cf(3, 0), cf(3, 1), cf(3, 2), cf(3, 3))
            );
            Some(m)
        }
    }
}

#[cfg(test)]
mod test {

    use num::*;
    use mat::tpe::GenSquareMat;
    use mat::mat::*;
    use mat::ctor::*;

    #[test]
    fn test_determinant() {
        let m2 = mat2(4., 5., 6., 7.);
        assert_eq!(m2.determinant(), -2.);
        assert_eq!(<Mat3 as One>::one().determinant(), 1.);
        let m4 = mat4(
            1., 0., 4., 0.,
            2., 1., 2., 1.,
            3., 2., 3., 1.,
            4., 3., 0., 0.
        );
        assert_eq!(m4.determinant(), -7.);
        assert_eq!((m4 * m4).determinant(), 49.);
        assert_eq!(<Mat4 as One>::one().determinant(), 1.);
    }

    #[test]
    fn test_inverse_mat2() {
        let yi = <Mat2 as One>::one();
        assert!(yi.inverse().is_some());
        assert!(<DMat2 as Zero>::zero().inverse().is_none());
        let mat = mat2(1., 3., 2., 4.);
        let inv = mat.inverse().unwrap();
        assert_close_to!(mat * inv, yi, 0.000001);
        assert_close_to!(inv * mat, yi, 0.000001);
        let m2 = mat2(1., 2., 2., 4.);
        assert!(m2.inverse().is_none());
    }

    #[test]
    fn test_inverse_mat3() {
        let yi = <Mat3 as One>::one();
        assert!(yi.inverse().is_some());
        assert_eq!(yi.inverse().unwrap(), yi);
        assert!(<DMat3 as Zero>::zero().inverse().is_none());
        let mat = mat3(5., 7., 11., -6., 9., 2., 1., 13., 0.);
        let inv = mat.inverse().unwrap();
        assert_close_to!(mat * inv, yi, 0.000001);
        assert_close_to!(inv * mat, yi, 0.000001);
    }

    #[test]
    fn test_inverse_mat4() {
        let mat = mat4(
            1., 0., 4., 0.,
            2., 1., 2., 1.,
            3., 2., 3., 1.,
            4., 3., 0., 0.
        );
        let invm = mat4(
            3./7., 12./7., -12./7., 4./7.,
            -4./7., -16./7., 16./7., -3./7.,
            1./7., -3./7., 3./7., -1./7.,
            -4./7., 5./7., 2./7., -3./7.
        );
        assert_approx_eq!(mat.inverse().unwrap(), invm);
        assert_close_to!(mat.inverse().unwrap().inverse().unwrap(), mat, 0.000001);
        assert!(<Mat4 as One>::one().inverse().is_some());
    }
}
<commit_msg>Remove obsolete syntax.<commit_after>//
// GLSL Mathematics for Rust.
//
// Copyright (c) 2015 The glm-rs authors.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use num::{ BaseFloat, One, Zero };
use vec::vec::{ Vector2, Vector3, Vector4 };
use super::tpe::{ GenMat, GenSquareMat };
use super::mat::*;
use std::num::Float;

impl<T: BaseFloat> One for Matrix2<T> {
    #[inline]
    fn one() -> Matrix2<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix2::new(
            Vector2::new(y, l),
            Vector2::new(l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector2<T>> for Matrix2<T> {
    #[inline(always)]
    fn determinant(&self) -> T {
        self[0][0] * self[1][1] - self[0][1] * self[1][0]
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix2<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let m = Matrix2::new(
                Vector2::new(self[1][1] * inv_det, -self[0][1] * inv_det),
                Vector2::new(-self[1][0] * inv_det, self[0][0] * inv_det)
            );
            Some(m)
        }
    }
}

impl<T: BaseFloat> One for Matrix3<T> {
    #[inline]
    fn one() -> Matrix3<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix3::new(
            Vector3::new(y, l, l),
            Vector3::new(l, y, l),
            Vector3::new(l, l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector3<T>> for Matrix3<T> {
    #[inline]
    fn determinant(&self) -> T {
        self[0][0] * (self[1][1] * self[2][2] - self[2][1] * self[1][2]) -
        self[1][0] * (self[0][1] * self[2][2] - self[2][1] * self[0][2]) +
        self[2][0] * (self[0][1] * self[1][2] - self[1][1] * self[0][2])
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix3<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let r11 = self[1][1] * self[2][2] - self[2][1] * self[1][2];
            let r12 = self[2][0] * self[1][2] - self[1][0] * self[2][2];
            let r13 = self[1][0] * self[2][1] - self[2][0] * self[1][1];
            let r21 = self[2][1] * self[0][2] - self[0][1] * self[2][2];
            let r22 = self[0][0] * self[2][2] - self[2][0] * self[0][2];
            let r23 = self[2][0] * self[0][1] - self[0][0] * self[2][1];
            let r31 = self[0][1] * self[1][2] - self[1][1] * self[0][2];
            let r32 = self[1][0] * self[0][2] - self[0][0] * self[1][2];
            let r33 = self[0][0] * self[1][1] - self[1][0] * self[0][1];
            let m = Matrix3::new(
                Vector3::new(r11 * inv_det, r21 * inv_det, r31 * inv_det),
                Vector3::new(r12 * inv_det, r22 * inv_det, r32 * inv_det),
                Vector3::new(r13 * inv_det, r23 * inv_det, r33 * inv_det)
            );
            Some(m)
        }
    }
}

impl<T: BaseFloat> One for Matrix4<T> {
    #[inline]
    fn one() -> Matrix4<T> {
        let y = <T as One>::one();
        let l = <T as Zero>::zero();
        Matrix4::new(
            Vector4::new(y, l, l, l),
            Vector4::new(l, y, l, l),
            Vector4::new(l, l, y, l),
            Vector4::new(l, l, l, y)
        )
    }
}

impl<T: BaseFloat> GenSquareMat<T, Vector4<T>> for Matrix4<T> {
    #[inline]
    fn determinant(&self) -> T {
        self[0][0] * (
            self[1][1] * self[2][2] * self[3][3] +
            self[2][1] * self[3][2] * self[1][3] +
            self[3][1] * self[1][2] * self[2][3] -
            self[3][1] * self[2][2] * self[1][3] -
            self[1][1] * self[3][2] * self[2][3] -
            self[2][1] * self[1][2] * self[3][3]
        ) -
        self[1][0] * (
            self[0][1] * self[2][2] * self[3][3] +
            self[2][1] * self[3][2] * self[0][3] +
            self[3][1] * self[0][2] * self[2][3] -
            self[3][1] * self[2][2] * self[0][3] -
            self[0][1] * self[3][2] * self[2][3] -
            self[2][1] * self[0][2] * self[3][3]
        ) +
        self[2][0] * (
            self[0][1] * self[1][2] * self[3][3] +
            self[1][1] * self[3][2] * self[0][3] +
            self[3][1] * self[0][2] * self[1][3] -
            self[3][1] * self[1][2] * self[0][3] -
            self[0][1] * self[3][2] * self[1][3] -
            self[1][1] * self[0][2] * self[3][3]
        ) -
        self[3][0] * (
            self[0][1] * self[1][2] * self[2][3] +
            self[1][1] * self[2][2] * self[0][3] +
            self[2][1] * self[0][2] * self[1][3] -
            self[2][1] * self[1][2] * self[0][3] -
            self[0][1] * self[2][2] * self[1][3] -
            self[1][1] * self[0][2] * self[2][3]
        )
    }
    #[inline]
    fn inverse(&self) -> Option<Matrix4<T>> {
        let det = self.determinant();
        let ling = <T as Zero>::zero();
        if det.is_approx_eq(&ling) {
            None
        } else {
            let inv_det = det.recip();
            let tr = self.transpose();
            let cf = |i, j| -> T {
                let mat = match i {
                    0 => Matrix3::new(
                        tr.c1.truncate(j),
                        tr.c2.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    1 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c2.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    2 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c1.truncate(j),
                        tr.c3.truncate(j)
                    ),
                    3 => Matrix3::new(
                        tr.c0.truncate(j),
                        tr.c1.truncate(j),
                        tr.c2.truncate(j)
                    ),
                    _ => unreachable!(),
                };
                let d = mat.determinant() * inv_det;
                if (i + j) & 1 == 1 {
                    -d
                } else {
                    d
                }
            };
            let m = Matrix4::new(
                Vector4::new(cf(0, 0), cf(0, 1), cf(0, 2), cf(0, 3)),
                Vector4::new(cf(1, 0), cf(1, 1), cf(1, 2), cf(1, 3)),
                Vector4::new(cf(2, 0), cf(2, 1), cf(2, 2), cf(2, 3)),
                Vector4::new(cf(3, 0), cf(3, 1), cf(3, 2), cf(3, 3))
            );
            Some(m)
        }
    }
}

#[cfg(test)]
mod test {

    use num::*;
    use mat::tpe::GenSquareMat;
    use mat::mat::*;
    use mat::ctor::*;

    #[test]
    fn test_determinant() {
        let m2 = mat2(4., 5., 6., 7.);
        assert_eq!(m2.determinant(), -2.);
        assert_eq!(<Mat3 as One>::one().determinant(), 1.);
        let m4 = mat4(
            1., 0., 4., 0.,
            2., 1., 2., 1.,
            3., 2., 3., 1.,
            4., 3., 0., 0.
        );
        assert_eq!(m4.determinant(), -7.);
        assert_eq!((m4 * m4).determinant(), 49.);
        assert_eq!(<Mat4 as One>::one().determinant(), 1.);
    }

    #[test]
    fn test_inverse_mat2() {
        let yi = <Mat2 as One>::one();
        assert!(yi.inverse().is_some());
        assert!(<DMat2 as Zero>::zero().inverse().is_none());
        let mat = mat2(1., 3., 2., 4.);
        let inv = mat.inverse().unwrap();
        assert_close_to!(mat * inv, yi, 0.000001);
        assert_close_to!(inv * mat, yi, 0.000001);
        let m2 = mat2(1., 2., 2., 4.);
        assert!(m2.inverse().is_none());
    }

    #[test]
    fn test_inverse_mat3() {
        let yi = <Mat3 as One>::one();
        assert!(yi.inverse().is_some());
        assert_eq!(yi.inverse().unwrap(), yi);
        assert!(<DMat3 as Zero>::zero().inverse().is_none());
        let mat = mat3(5., 7., 11., -6., 9., 2., 1., 13., 0.);
        let inv = mat.inverse().unwrap();
        assert_close_to!(mat * inv, yi, 0.000001);
        assert_close_to!(inv * mat, yi, 0.000001);
    }

    #[test]
    fn test_inverse_mat4() {
        let mat = mat4(
            1., 0., 4., 0.,
            2., 1., 2., 1.,
            3., 2., 3., 1.,
            4., 3., 0., 0.
        );
        let invm = mat4(
            3./7., 12./7., -12./7., 4./7.,
            -4./7., -16./7., 16./7., -3./7.,
            1./7., -3./7., 3./7., -1./7.,
            -4./7., 5./7., 2./7., -3./7.
        );
        assert_approx_eq!(mat.inverse().unwrap(), invm);
        assert_close_to!(mat.inverse().unwrap().inverse().unwrap(), mat, 0.000001);
        assert!(<Mat4 as One>::one().inverse().is_some());
    }
}
<|endoftext|>


<commit_before>//! This module implements the in-memory database
//!
//! A support folder with SDK debug symbols can be processed into a
//! in-memory database format which is a flat file on the file system
//! that gets mmaped into the process.
use std::io::{Read, Write, Seek, SeekFrom, Stdout, ErrorKind as IoErrorKind};
use std::fs::File;
use std::mem;
use std::slice;
use std::path::Path;
use std::cell::RefCell;
use std::collections::HashMap;

use uuid::Uuid;
use pbr::ProgressBar;
use xz2::write::XzEncoder;
use tempfile::tempfile;

use super::Result;
use super::sdk::{SdkInfo, DumpOptions, Objects};
use super::dsym::{Object, Variant};
use super::memdbtypes::{IndexItem, StoredSlice, MemDbHeader, IndexedUuid};
use super::utils::file_size_format;

pub struct MemDbBuilder<W> {
    writer: RefCell<W>,
    tempfile: Option<RefCell<File>>,
    info: SdkInfo,
    symbols: Vec<String>,
    symbols_map: HashMap<String, u32>,
    object_names: Vec<String>,
    object_names_map: HashMap<String, u16>,
    object_uuid_mapping: Vec<(String, Uuid)>,
    variant_uuids: Vec<IndexedUuid>,
    variants: Vec<Vec<IndexItem>>,
    symbol_count: usize,
    state: RefCell<DumpState>,
    options: DumpOptions,
}

pub struct DumpState {
    pb: Option<ProgressBar<Stdout>>,
}

fn make_progress_bar(count: usize) -> ProgressBar<Stdout> {
    let mut pb = ProgressBar::new(count as u64);
    pb.tick_format("⠇⠋⠙⠸⠴⠦");
    pb.format("[■□□]");
    pb.show_tick = true;
    pb.show_speed = false;
    pb.show_percent = false;
    pb.show_counter = false;
    pb.show_time_left = false;
    pb
}

trait WriteSeek : Write + Seek {}
impl<T: Write+Seek> WriteSeek for T {}

impl<W: Write + Seek> MemDbBuilder<W> {

    pub fn new(writer: W, info: &SdkInfo, opts: DumpOptions, object_count: usize)
        -> Result<MemDbBuilder<W>>
    {
        let rv = MemDbBuilder {
            writer: RefCell::new(writer),
            tempfile: if opts.compress {
                Some(RefCell::new(tempfile()?))
            } else {
                None
            },
            info: info.clone(),
            symbols: vec![],
            symbols_map: HashMap::new(),
            object_names: vec![],
            object_names_map: HashMap::new(),
            object_uuid_mapping: vec![],
            variant_uuids: vec![],
            variants: vec![],
            symbol_count: 0,
            state: RefCell::new(DumpState {
                pb: if opts.show_progress_bar {
                    Some(make_progress_bar(object_count))
                } else {
                    None
                },
            }),
            options: opts,
        };
        let header = MemDbHeader { ..Default::default() };
        rv.write(&header)?;
        rv.set_progress_message("Initializing");
        Ok(rv)
    }

    fn with_file<T, F: FnOnce(&mut WriteSeek) -> T>(&self, f: F) -> T {
        if self.options.compress {
            f(&mut *self.tempfile.as_ref().unwrap().borrow_mut() as &mut WriteSeek)
        } else {
            f(&mut *self.writer.borrow_mut() as &mut WriteSeek)
        }
    }

    fn write_bytes(&self, x: &[u8]) -> Result<usize> {
        self.with_file(|mut w| {
            w.write_all(x)?;
            Ok(x.len())
        })
    }

    fn write<T>(&self, x: &T) -> Result<usize> {
        unsafe {
            let bytes : *const u8 = mem::transmute(x);
            let size = mem::size_of_val(x);
            self.with_file(|mut w| {
                w.write_all(slice::from_raw_parts(bytes, size))?;
                Ok(size)
            })
        }
    }

    fn seek(&self, new_pos: usize) -> Result<()> {
        self.with_file(|mut w| {
            w.seek(SeekFrom::Start(new_pos as u64))?;
            Ok(())
        })
    }

    fn tell(&self) -> Result<usize> {
        self.with_file(|mut w| {
            Ok(w.seek(SeekFrom::Current(0))? as usize)
        })
    }

    fn inc_progress_bar(&self, step: usize) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.add(step as u64);
        }
    }

    fn set_progress_message(&self, msg: &str) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.message(&format!("  ◦ {: <40}", msg));
            pb.tick();
        }
    }

    fn tick_progress_bar(&self, idx: usize) {
        if idx % 100 == 0 {
            if let Some(ref mut pb) = self.state.borrow_mut().pb {
                pb.tick();
            }
        }
    }

    fn progress_bar_finish(&self, msg: &str) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.finish_print(&format!("  ● {}", msg));
            println!("");
        }
    }

    fn new_progress_bar(&self, count: usize) {
        if !self.options.show_progress_bar {
            return;
        }
        self.state.borrow_mut().pb = Some(make_progress_bar(count));
    }

    fn add_symbol(&mut self, sym: &str) -> usize {
        if let Some(&sym_id) = self.symbols_map.get(sym) {
            return sym_id as usize;
        }
        let symbol_count = self.symbols.len();
        self.symbols.push(sym.to_string());
        self.symbols_map.insert(sym.to_string(), symbol_count as u32);
        symbol_count
    }

    fn add_object_name(&mut self, src: &str) -> usize {
        if let Some(&src_id) = self.object_names_map.get(src) {
            return src_id as usize;
        }
        let object_count = self.object_names.len();
        self.object_names.push(src.to_string());
        self.object_names_map.insert(src.to_string(), object_count as u16);
        object_count
    }

    pub fn write_object(&mut self, obj: &Object, filename: Option<&str>,
                        offset: usize) -> Result<()> {
        self.inc_progress_bar(offset);
        for variant in obj.variants() {
            let src = variant.name().or(filename).unwrap();
            if let Some(fname) = Path::new(src).file_name().and_then(|x| x.to_str()) {
                self.set_progress_message(fname);
            }
            self.write_object_variant(&obj, &variant, src)?;
        }
        Ok(())
    }

    pub fn write_object_variant(&mut self, obj: &Object, var: &Variant,
                                src: &str) -> Result<()> {
        let mut symbols = obj.symbols(var.arch())?;
        let src_id = self.add_object_name(src);

        // build symbol index
        let mut index = vec![];
        for (idx, (addr, sym)) in symbols.iter().enumerate() {
            let sym_id = self.add_symbol(sym);
            index.push(IndexItem::new(addr, src_id, sym_id));
            self.tick_progress_bar(idx);
            self.symbol_count += 1;
        }
        index.sort_by_key(|item| item.addr());

        // register variant and uuid
        self.variant_uuids.push(IndexedUuid::new(&var.uuid().unwrap(), self.variants.len()));
        self.object_uuid_mapping.push((
            format!("{}:{}", src, var.arch()),
            var.uuid().unwrap()
        ));
        self.variants.push(index);

        Ok(())
    }

    fn make_string_slices(&self, strings: &[String], _try_compress: bool) -> Result<Vec<StoredSlice>> {
        let mut slices = vec![];
        for (idx, string) in strings.iter().enumerate() {
            let offset = self.tell()?;
            let len = self.write_bytes(string.as_bytes())?;
            slices.push(StoredSlice::new(offset, len, false));
            self.tick_progress_bar(idx);
        }
        Ok(slices)
    }

    fn write_slices(&self, slices: &[StoredSlice], start: &mut u32, len: &mut u32) -> Result<()> {
        *start = self.tell()? as u32;
        for (idx, item) in slices.iter().enumerate() {
            self.write(item)?;
            self.tick_progress_bar(idx);
        }
        *len = slices.len() as u32;
        Ok(())
    }

    pub fn flush(&mut self) -> Result<()> {
        self.progress_bar_finish(&format!(
            "Processed {} symbols", self.symbol_count));
        self.new_progress_bar(3);

        let mut header = MemDbHeader { ..Default::default() };
        header.version = 1;
        header.sdk_info.set_from_sdk_info(&self.info);

        self.inc_progress_bar(1);
        self.set_progress_message("Writing metadata");

        // start by writing out the index of the variants and record the slices.
        let mut slices = vec![];
        let mut ii_idx = 0;
        for variant in self.variants.iter() {
            let offset = self.tell()?;
            for index_item in variant {
                self.write(index_item)?;
                ii_idx += 1;
                self.tick_progress_bar(ii_idx);
            }
            slices.push(StoredSlice::new(offset, (self.tell()? - offset), false));
        }
        self.write_slices(&slices[..], &mut header.variants_start,
                          &mut header.variants_count)?;

        // next write out the UUIDs.  Since these are fixed length we do not
        // need to use slices here.
        header.uuids_start = self.tell()? as u32;
        header.uuids_count = self.variant_uuids.len() as u32;
        self.variant_uuids.sort_by_key(|x| x.uuid);
        for (idx, indexed_uuid) in self.variant_uuids.iter().enumerate() {
            self.write(indexed_uuid)?;
            self.tick_progress_bar(idx);
        }

        // next we write out the name + arch -> uuid index mapping.  We also sort
        // this by uuid so that the index matches up.
        header.tagged_object_names_start = self.tell()? as u32;
        self.object_uuid_mapping.sort_by_key(|&(_, b)| b);
        for (idx, &(ref tagged_object, _)) in self.object_uuid_mapping.iter().enumerate() {
            self.write_bytes(format!("{}\x00", tagged_object).as_bytes())?;
            self.tick_progress_bar(idx);
        }
        header.tagged_object_names_end = self.tell()? as u32;

        // now write out all the object name sources
        let slices = self.make_string_slices(&self.object_names[..], true)?;
        self.write_slices(&slices[..], &mut header.object_names_start,
                          &mut header.object_names_count)?;

        self.inc_progress_bar(1);
        self.set_progress_message("Writing symbols");

        // now write out all the symbols
        let slices = self.make_string_slices(&self.symbols[..], true)?;
        self.write_slices(&slices[..], &mut header.symbols_start,
                          &mut header.symbols_count)?;

        self.inc_progress_bar(1);
        self.set_progress_message("Writing headers");

        let file_size = self.tell()?;

        // write the updated header
        self.seek(0)?;
        self.write(&header)?;

        self.progress_bar_finish(&format!(
            "Indexed {} variants", self.variant_uuids.len()));

        // compress if necessary
        if self.options.compress {
            let mut buf = [0; 4096];
            let steps = file_size / buf.len() + 1;
            self.new_progress_bar(steps);
            self.set_progress_message("Compressing");
            self.seek(0)?;
            let mut reader = self.tempfile.as_ref().unwrap().borrow_mut();
            let mut writer = self.writer.borrow_mut();
            {
                let mut writer = XzEncoder::new(&mut *writer, 9);
                loop {
                    let len = match reader.read(&mut buf) {
                        Ok(0) => { break },
                        Ok(len) => len,
                        Err(ref e) if e.kind() == IoErrorKind::Interrupted => continue,
                        Err(e) => return Err(e.into()),
                    };
                    self.inc_progress_bar(1);
                    writer.write_all(&buf[..len])?;
                }
            }
            let compressed_file_size = writer.seek(SeekFrom::Current(0))? as usize;
            let pct = (compressed_file_size * 100) / file_size;
            self.progress_bar_finish(&format!(
                "Compressed by {}% from {} to {}",
                pct,
                file_size_format(file_size),
                file_size_format(compressed_file_size)));
        }

        Ok(())
    }
}

pub fn dump_memdb<W: Write + Seek>(writer: W, info: &SdkInfo,
                                   opts: DumpOptions, objects: Objects)
    -> Result<()>
{
    let mut builder = MemDbBuilder::new(writer, info, opts, objects.file_count())?;
    for obj_res in objects {
        let (offset, filename, obj) = obj_res?;
        builder.write_object(&obj, Some(&filename), offset)?;
    }
    builder.flush()?;
    Ok(())
}
<commit_msg>Fixed percentage text<commit_after>//! This module implements the in-memory database
//!
//! A support folder with SDK debug symbols can be processed into a
//! in-memory database format which is a flat file on the file system
//! that gets mmaped into the process.
use std::io::{Read, Write, Seek, SeekFrom, Stdout, ErrorKind as IoErrorKind};
use std::fs::File;
use std::mem;
use std::slice;
use std::path::Path;
use std::cell::RefCell;
use std::collections::HashMap;

use uuid::Uuid;
use pbr::ProgressBar;
use xz2::write::XzEncoder;
use tempfile::tempfile;

use super::Result;
use super::sdk::{SdkInfo, DumpOptions, Objects};
use super::dsym::{Object, Variant};
use super::memdbtypes::{IndexItem, StoredSlice, MemDbHeader, IndexedUuid};
use super::utils::file_size_format;

pub struct MemDbBuilder<W> {
    writer: RefCell<W>,
    tempfile: Option<RefCell<File>>,
    info: SdkInfo,
    symbols: Vec<String>,
    symbols_map: HashMap<String, u32>,
    object_names: Vec<String>,
    object_names_map: HashMap<String, u16>,
    object_uuid_mapping: Vec<(String, Uuid)>,
    variant_uuids: Vec<IndexedUuid>,
    variants: Vec<Vec<IndexItem>>,
    symbol_count: usize,
    state: RefCell<DumpState>,
    options: DumpOptions,
}

pub struct DumpState {
    pb: Option<ProgressBar<Stdout>>,
}

fn make_progress_bar(count: usize) -> ProgressBar<Stdout> {
    let mut pb = ProgressBar::new(count as u64);
    pb.tick_format("⠇⠋⠙⠸⠴⠦");
    pb.format("[■□□]");
    pb.show_tick = true;
    pb.show_speed = false;
    pb.show_percent = false;
    pb.show_counter = false;
    pb.show_time_left = false;
    pb
}

trait WriteSeek : Write + Seek {}
impl<T: Write+Seek> WriteSeek for T {}

impl<W: Write + Seek> MemDbBuilder<W> {

    pub fn new(writer: W, info: &SdkInfo, opts: DumpOptions, object_count: usize)
        -> Result<MemDbBuilder<W>>
    {
        let rv = MemDbBuilder {
            writer: RefCell::new(writer),
            tempfile: if opts.compress {
                Some(RefCell::new(tempfile()?))
            } else {
                None
            },
            info: info.clone(),
            symbols: vec![],
            symbols_map: HashMap::new(),
            object_names: vec![],
            object_names_map: HashMap::new(),
            object_uuid_mapping: vec![],
            variant_uuids: vec![],
            variants: vec![],
            symbol_count: 0,
            state: RefCell::new(DumpState {
                pb: if opts.show_progress_bar {
                    Some(make_progress_bar(object_count))
                } else {
                    None
                },
            }),
            options: opts,
        };
        let header = MemDbHeader { ..Default::default() };
        rv.write(&header)?;
        rv.set_progress_message("Initializing");
        Ok(rv)
    }

    fn with_file<T, F: FnOnce(&mut WriteSeek) -> T>(&self, f: F) -> T {
        if self.options.compress {
            f(&mut *self.tempfile.as_ref().unwrap().borrow_mut() as &mut WriteSeek)
        } else {
            f(&mut *self.writer.borrow_mut() as &mut WriteSeek)
        }
    }

    fn write_bytes(&self, x: &[u8]) -> Result<usize> {
        self.with_file(|mut w| {
            w.write_all(x)?;
            Ok(x.len())
        })
    }

    fn write<T>(&self, x: &T) -> Result<usize> {
        unsafe {
            let bytes : *const u8 = mem::transmute(x);
            let size = mem::size_of_val(x);
            self.with_file(|mut w| {
                w.write_all(slice::from_raw_parts(bytes, size))?;
                Ok(size)
            })
        }
    }

    fn seek(&self, new_pos: usize) -> Result<()> {
        self.with_file(|mut w| {
            w.seek(SeekFrom::Start(new_pos as u64))?;
            Ok(())
        })
    }

    fn tell(&self) -> Result<usize> {
        self.with_file(|mut w| {
            Ok(w.seek(SeekFrom::Current(0))? as usize)
        })
    }

    fn inc_progress_bar(&self, step: usize) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.add(step as u64);
        }
    }

    fn set_progress_message(&self, msg: &str) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.message(&format!("  ◦ {: <40}", msg));
            pb.tick();
        }
    }

    fn tick_progress_bar(&self, idx: usize) {
        if idx % 100 == 0 {
            if let Some(ref mut pb) = self.state.borrow_mut().pb {
                pb.tick();
            }
        }
    }

    fn progress_bar_finish(&self, msg: &str) {
        if let Some(ref mut pb) = self.state.borrow_mut().pb {
            pb.finish_print(&format!("  ● {}", msg));
            println!("");
        }
    }

    fn new_progress_bar(&self, count: usize) {
        if !self.options.show_progress_bar {
            return;
        }
        self.state.borrow_mut().pb = Some(make_progress_bar(count));
    }

    fn add_symbol(&mut self, sym: &str) -> usize {
        if let Some(&sym_id) = self.symbols_map.get(sym) {
            return sym_id as usize;
        }
        let symbol_count = self.symbols.len();
        self.symbols.push(sym.to_string());
        self.symbols_map.insert(sym.to_string(), symbol_count as u32);
        symbol_count
    }

    fn add_object_name(&mut self, src: &str) -> usize {
        if let Some(&src_id) = self.object_names_map.get(src) {
            return src_id as usize;
        }
        let object_count = self.object_names.len();
        self.object_names.push(src.to_string());
        self.object_names_map.insert(src.to_string(), object_count as u16);
        object_count
    }

    pub fn write_object(&mut self, obj: &Object, filename: Option<&str>,
                        offset: usize) -> Result<()> {
        self.inc_progress_bar(offset);
        for variant in obj.variants() {
            let src = variant.name().or(filename).unwrap();
            if let Some(fname) = Path::new(src).file_name().and_then(|x| x.to_str()) {
                self.set_progress_message(fname);
            }
            self.write_object_variant(&obj, &variant, src)?;
        }
        Ok(())
    }

    pub fn write_object_variant(&mut self, obj: &Object, var: &Variant,
                                src: &str) -> Result<()> {
        let mut symbols = obj.symbols(var.arch())?;
        let src_id = self.add_object_name(src);

        // build symbol index
        let mut index = vec![];
        for (idx, (addr, sym)) in symbols.iter().enumerate() {
            let sym_id = self.add_symbol(sym);
            index.push(IndexItem::new(addr, src_id, sym_id));
            self.tick_progress_bar(idx);
            self.symbol_count += 1;
        }
        index.sort_by_key(|item| item.addr());

        // register variant and uuid
        self.variant_uuids.push(IndexedUuid::new(&var.uuid().unwrap(), self.variants.len()));
        self.object_uuid_mapping.push((
            format!("{}:{}", src, var.arch()),
            var.uuid().unwrap()
        ));
        self.variants.push(index);

        Ok(())
    }

    fn make_string_slices(&self, strings: &[String], _try_compress: bool) -> Result<Vec<StoredSlice>> {
        let mut slices = vec![];
        for (idx, string) in strings.iter().enumerate() {
            let offset = self.tell()?;
            let len = self.write_bytes(string.as_bytes())?;
            slices.push(StoredSlice::new(offset, len, false));
            self.tick_progress_bar(idx);
        }
        Ok(slices)
    }

    fn write_slices(&self, slices: &[StoredSlice], start: &mut u32, len: &mut u32) -> Result<()> {
        *start = self.tell()? as u32;
        for (idx, item) in slices.iter().enumerate() {
            self.write(item)?;
            self.tick_progress_bar(idx);
        }
        *len = slices.len() as u32;
        Ok(())
    }

    pub fn flush(&mut self) -> Result<()> {
        self.progress_bar_finish(&format!(
            "Processed {} symbols", self.symbol_count));
        self.new_progress_bar(3);

        let mut header = MemDbHeader { ..Default::default() };
        header.version = 1;
        header.sdk_info.set_from_sdk_info(&self.info);

        self.inc_progress_bar(1);
        self.set_progress_message("Writing metadata");

        // start by writing out the index of the variants and record the slices.
        let mut slices = vec![];
        let mut ii_idx = 0;
        for variant in self.variants.iter() {
            let offset = self.tell()?;
            for index_item in variant {
                self.write(index_item)?;
                ii_idx += 1;
                self.tick_progress_bar(ii_idx);
            }
            slices.push(StoredSlice::new(offset, (self.tell()? - offset), false));
        }
        self.write_slices(&slices[..], &mut header.variants_start,
                          &mut header.variants_count)?;

        // next write out the UUIDs.  Since these are fixed length we do not
        // need to use slices here.
        header.uuids_start = self.tell()? as u32;
        header.uuids_count = self.variant_uuids.len() as u32;
        self.variant_uuids.sort_by_key(|x| x.uuid);
        for (idx, indexed_uuid) in self.variant_uuids.iter().enumerate() {
            self.write(indexed_uuid)?;
            self.tick_progress_bar(idx);
        }

        // next we write out the name + arch -> uuid index mapping.  We also sort
        // this by uuid so that the index matches up.
        header.tagged_object_names_start = self.tell()? as u32;
        self.object_uuid_mapping.sort_by_key(|&(_, b)| b);
        for (idx, &(ref tagged_object, _)) in self.object_uuid_mapping.iter().enumerate() {
            self.write_bytes(format!("{}\x00", tagged_object).as_bytes())?;
            self.tick_progress_bar(idx);
        }
        header.tagged_object_names_end = self.tell()? as u32;

        // now write out all the object name sources
        let slices = self.make_string_slices(&self.object_names[..], true)?;
        self.write_slices(&slices[..], &mut header.object_names_start,
                          &mut header.object_names_count)?;

        self.inc_progress_bar(1);
        self.set_progress_message("Writing symbols");

        // now write out all the symbols
        let slices = self.make_string_slices(&self.symbols[..], true)?;
        self.write_slices(&slices[..], &mut header.symbols_start,
                          &mut header.symbols_count)?;

        self.inc_progress_bar(1);
        self.set_progress_message("Writing headers");

        let file_size = self.tell()?;

        // write the updated header
        self.seek(0)?;
        self.write(&header)?;

        self.progress_bar_finish(&format!(
            "Indexed {} variants", self.variant_uuids.len()));

        // compress if necessary
        if self.options.compress {
            let mut buf = [0; 4096];
            let steps = file_size / buf.len() + 1;
            self.new_progress_bar(steps);
            self.set_progress_message("Compressing");
            self.seek(0)?;
            let mut reader = self.tempfile.as_ref().unwrap().borrow_mut();
            let mut writer = self.writer.borrow_mut();
            {
                let mut writer = XzEncoder::new(&mut *writer, 9);
                loop {
                    let len = match reader.read(&mut buf) {
                        Ok(0) => { break },
                        Ok(len) => len,
                        Err(ref e) if e.kind() == IoErrorKind::Interrupted => continue,
                        Err(e) => return Err(e.into()),
                    };
                    self.inc_progress_bar(1);
                    writer.write_all(&buf[..len])?;
                }
            }
            let compressed_file_size = writer.seek(SeekFrom::Current(0))? as usize;
            let pct = (compressed_file_size * 100) / file_size;
            self.progress_bar_finish(&format!(
                "Compressed from {} to {} ({}% of original size)",
                file_size_format(file_size),
                file_size_format(compressed_file_size),
                pct));
        }

        Ok(())
    }
}

pub fn dump_memdb<W: Write + Seek>(writer: W, info: &SdkInfo,
                                   opts: DumpOptions, objects: Objects)
    -> Result<()>
{
    let mut builder = MemDbBuilder::new(writer, info, opts, objects.file_count())?;
    for obj_res in objects {
        let (offset, filename, obj) = obj_res?;
        builder.write_object(&obj, Some(&filename), offset)?;
    }
    builder.flush()?;
    Ok(())
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::{SeekFrom, Seek, Read, Write};
use std::fs::OpenOptions;
use std::fmt;
use num::iter::range_step_inclusive;
use std::cell::RefCell;
use std::io::Error;

extern crate libc;
use self::libc::funcs::posix01::unistd::ftruncate;
use std::os::unix::prelude::AsRawFd;

use super::header::{ Header, read_header };
use super::write_op::WriteOp;
use super::archive_info::{ ArchiveInfo };
use super::metadata::{Metadata, AggregationType};
use whisper::schema::Schema;

use whisper::point;

pub struct WhisperFile<'a> {
    pub path: &'a str,
    pub handle: RefCell<File>,
    pub header: Header
}

impl<'a> fmt::Debug for WhisperFile<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path:  {:?}, header: {:?}", self.path, self.header)
    }
}

impl<'a> fmt::Display for WhisperFile<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ref metadata = self.header.metadata;
        let ref archive_infos = self.header.archive_infos;

        try!(write!(f, "whisper file ({})\n", self.path));
        try!(write!(f, "  metadata\n"));
        try!(write!(f, "    aggregation method: {:?}\n", metadata.aggregation_type));
        try!(write!(f, "    max retention: {:?}\n", metadata.max_retention));
        try!(write!(f, "    xff: {:?}\n", metadata.x_files_factor));
        let mut index = 0;
        for archive_info in archive_infos.iter() {
        try!(write!(f, "  archive {}\n", index));
        try!(write!(f, "    seconds per point: {}\n", archive_info.seconds_per_point));
        try!(write!(f, "    points: {}\n", archive_info.points));
        try!(write!(f, "    retention: {} (s)\n", archive_info.retention));
        try!(write!(f, "    size: {} (bytes)\n", archive_info.size_in_bytes));

            index = index+1;
        };
        write!(f,"") // make the types happy
    }
}

pub fn open(path: &str) -> Result<WhisperFile, Error> {
    let file = try!(OpenOptions::new().read(true).write(true).create(false).open(path));
    let header = try!(read_header(&file));
    let whisper_file = WhisperFile { path: path, header: header, handle: RefCell::new(file) };
    Ok(whisper_file)
}

impl<'a> WhisperFile<'a> {

    pub fn new(path: &str, schema: Schema /* , _: Metadata */) -> Result<WhisperFile, Error> {
        let size_needed = schema.size_on_disk();
        let opened_file = try!(OpenOptions::new().read(true).write(true).create(true).open(path));

        // Allocate the room necessary
        debug!("allocating...");
        {
            let raw_fd = opened_file.as_raw_fd();
            let retval = unsafe {
                // TODO skip to fallocate-like behavior. Will need wrapper for OSX.
                ftruncate(raw_fd, size_needed as i64)
            };
            if retval != 0 {
                return Err(Error::last_os_error());
            }
        }
        debug!("done allocating");

        let metadata = {
            // TODO make agg_t, max_r options from the command line.
            let aggregation_type = AggregationType::Average;
            let x_files_factor = 0.5;
            Metadata {
                aggregation_type: aggregation_type,
                max_retention: schema.max_retention() as u32,
                x_files_factor: x_files_factor,
                archive_count: schema.retention_policies.len() as u32
            }
        };

        // Piggy back on moving file write forward
        metadata.write(&opened_file);

        let initial_archive_offset = schema.header_size_on_disk();
        schema.retention_policies.iter().fold(initial_archive_offset, |offset, &rp| {
            debug!("sup");
            rp.write(&opened_file, offset);
            offset + rp.size_on_disk()
        });

        let new_whisper_file = WhisperFile {
            path: path,
            handle: RefCell::new(opened_file),
            header: Header {
                metadata: metadata,
                archive_infos: vec![]
            }
        };
        Ok(new_whisper_file)
    }

    // TODO: Result<usize> return how many write ops were done
    pub fn write(&mut self, current_time: u64, point: point::Point) {

        match self.split(current_time, point.timestamp) {
            Some( (high_precision_archive, rest) ) => {
                let base_point = self.read_point(high_precision_archive.offset);
                let base_timestamp = base_point.timestamp;

                self.write_archives(
                    (high_precision_archive, rest),
                    point,
                    base_timestamp
                );
            },
            None => {
                panic!("no archives satisfy current time")
            }
        }

    }

    fn perform_write_op(&self, write_op: &WriteOp) {
        let mut handle = self.handle.borrow_mut();
        handle.seek(write_op.seek).unwrap();
        handle.write_all(&(write_op.bytes)).unwrap();
    }

    fn read_point(&self, offset: u64) -> point::Point {
        let mut file = self.handle.borrow_mut();
        file.seek(SeekFrom::Start(offset)).unwrap();

        let mut points_buf : [u8; 12] = [0; 12];
        let mut buf_ref : &mut [u8] = &mut points_buf;
        file.read(buf_ref).unwrap();

        point::buf_to_point(buf_ref)
    }

    pub fn write_archives(&self, (ai,rest): (&ArchiveInfo, Vec<&ArchiveInfo>), point: point::Point, base_timestamp: u64) {
        {
            let write_op = build_write_op( ai, &point, base_timestamp );
            self.perform_write_op(&write_op);
        }

        if rest.len() > 1 {
            self.downsample(ai, rest[0], point.timestamp).map(|write_op| self.perform_write_op(&write_op) );

            let high_res_iter = rest[0..rest.len()-1].into_iter();
            let low_res_iter = rest[1..].into_iter();
            let _ : Vec<()> = high_res_iter.
                zip(low_res_iter).
                take_while(|&(h,l)| {
                    match self.downsample(h, l, point.timestamp) {
                        Some(write_op) => {
                            self.perform_write_op(&write_op);
                            true
                        },
                        None => false
                    }
                }).
                map(|_| ()).
                collect();
        }
    }

    // The most expensive IO functionality
    // Reads many samples from high-res archive and
    // aggregates to lower-res archive. Schemas could do well to avoid
    // aggregation unless disk space is truly at a premium.
    //
    // A cache for each archive would do well here. `memmap` would be awesomesauce.
    fn downsample(&self, h_res_archive: &ArchiveInfo, l_res_archive: &ArchiveInfo, base_timestamp: u64) -> Option<WriteOp> {
        assert!(h_res_archive.seconds_per_point < l_res_archive.seconds_per_point);

        let l_interval_start = l_res_archive.interval_ceiling(base_timestamp);
        // debug!("base_timestamp: {}, l_interval_start: {}", base_timestamp, l_interval_start);

        let h_base_timestamp = self.read_point(h_res_archive.offset).timestamp;
        let h_res_start_offset : u64 = if h_base_timestamp == 0 {
            h_res_archive.offset
        } else {
            // TODO: this can be negative. Does that change timestamp understanding?
            let timespan  = l_interval_start as i64 - h_base_timestamp as i64;
            // debug!("timespan {}", timespan);
            let points = timespan / h_res_archive.seconds_per_point as i64;
            // debug!("points {}", points);
            let bytes = points * point::POINT_SIZE as i64;

            // TODO: OMG, move this craziness somewhere else
            let wrapped_index = {
                let remainder = bytes % h_res_archive.size_in_bytes as i64;
                if remainder < 0 {
                    h_res_archive.size_in_bytes as i64 + remainder
                } else {
                    remainder
                }
            };
            // debug!("offset: {}, wrapped_index: {}, bytes: {}, points: {}", h_res_archive.offset, wrapped_index, bytes, points);
            (h_res_archive.offset as i64 + wrapped_index) as u64
        };

        let h_res_points_needed = l_res_archive.seconds_per_point / h_res_archive.seconds_per_point;
        let h_res_bytes_needed = h_res_points_needed * point::POINT_SIZE as u64;

        let h_res_end_offset = {
            let rel_first_offset = h_res_start_offset - h_res_archive.offset;
            let rel_second_offset = (rel_first_offset + h_res_bytes_needed) % h_res_archive.size_in_bytes;
            h_res_archive.offset + rel_second_offset
        };

        let mut h_res_read_buf : Vec<u8> = Vec::with_capacity(h_res_bytes_needed as usize);

        // Subroutine for filling in the buffer
        {
            let mut handle = self.handle.borrow_mut();

            // TODO: must be a better way of zeroing out a buffer to fill in the vector
            for _ in 0..h_res_bytes_needed {
                h_res_read_buf.push(0);
            }

            // TODO: refactor in to function which
            // returns ((Seek,BytesRead),Option<(Seek,BytesRead)>)
            // so this code can be refactored and unit tested...
            if h_res_start_offset < h_res_end_offset {
                // No wrap situation
                let seek = SeekFrom::Start(h_res_start_offset);

                let mut read_buf : &mut [u8] = &mut h_res_read_buf[..];
                handle.seek(seek).unwrap();
                // debug!("READ FROM {} to {} (contiguous read)", h_res_start_offset, h_res_start_offset + read_buf.len() as u64);
                handle.read(read_buf).unwrap();
            } else {
                let high_res_abs_end = h_res_archive.offset + h_res_archive.size_in_bytes;
                let first_seek = SeekFrom::Start(h_res_start_offset);
                let first_seek_bytes = high_res_abs_end - h_res_start_offset;

                // debug!("READ FROM {} to {} (wrap-around read 1)", h_res_start_offset, h_res_start_offset+first_seek_bytes);

                let (first_buf, second_buf) = h_res_read_buf.split_at_mut(first_seek_bytes as usize);

                handle.seek(first_seek).unwrap();
                handle.read(first_buf).unwrap();

                let second_seek = SeekFrom::Start(h_res_archive.offset);
                // debug!("READ FROM {} to {} (wrap-around read 2)", h_res_archive.offset, h_res_archive.offset + second_buf.len() as u64);

                handle.seek(second_seek).unwrap();
                handle.read(second_buf).unwrap();
            }

        }

        let low_res_aggregate = {
            let points : Vec<point::Point> = h_res_read_buf.chunks(point::POINT_SIZE).map(|chunk| {
                point::buf_to_point(chunk)

            }).collect();

            let timestamp_start = l_interval_start;
            let timestamp_stop = l_interval_start + (h_res_points_needed as u64)*h_res_archive.seconds_per_point;
            let step = h_res_archive.seconds_per_point;

            let expected_timestamps = range_step_inclusive(timestamp_start, timestamp_stop, step);
            let valid_points : Vec<&point::Point> = expected_timestamps.
                zip(points.iter()).
                map(|(ts, p)| {
                    if p.timestamp == ts {
                        // debug!("comparing {} and {}/{} (MATCH!)", ts, p.timestamp, p.value);
                        Some(p)
                    } else {
                        // debug!("comparing {} and {}/{}", ts, p.timestamp, p.value);
                        None
                    }
                }).filter(|agg| !agg.is_none()).map(|agg| agg.unwrap()).collect();
            self.aggregate_samples(valid_points, h_res_points_needed)
        };

        low_res_aggregate.map(|aggregate| {
            let l_res_base_point = self.read_point(l_res_archive.offset);
            let l_res_point = point::Point{ timestamp: l_interval_start, value: aggregate };
            build_write_op(l_res_archive, &l_res_point, l_res_base_point.timestamp)
        })
    }

    fn aggregate_samples(&self, points: Vec<&point::Point>, points_possible: u64) -> Option<f64>{
        // debug!("points: {:?}", points);
        let valid_points : Vec<&&point::Point> = points.iter().filter(|p| p.timestamp != 0).map(|p| p).collect();

        let ratio : f32 = valid_points.len() as f32 / points_possible as f32;
        // debug!("valid_points: {}, len: {}, ratio {} vs xff {}", valid_points.len(), points.len(), ratio, self.header.metadata.x_files_factor);
        if ratio < self.header.metadata.x_files_factor {
            // debug!("not enough data to propagate");
            return None;
        }

        // TODO: we only do aggregation right now!
        match self.header.metadata.aggregation_type {
            AggregationType::Average => {
                let sum = points.iter().map(|p| p.value).fold(0.0, |l, r| l + r);
                Some(sum / points.len() as f64)
            },
            _ => { Some(0.0) }
        }
    }

    fn split(&self, current_time: u64, point_timestamp: u64) -> Option<(&ArchiveInfo, Vec<&ArchiveInfo>)>  {
        let mut archive_iter = self.header.archive_infos.iter();

        let high_precision_archive_option = archive_iter.find(|ai|
            ai.retention > (current_time - point_timestamp)
        );

        match high_precision_archive_option {
            Some(ai) => {
                let rest_of_archives = archive_iter;
                let low_res_archives : Vec<&ArchiveInfo> = rest_of_archives.collect();

                Some((ai, low_res_archives))
            },
            None => {
                None
            }
        }
    }
}

fn build_write_op(archive_info: &ArchiveInfo, point: &point::Point, base_timestamp: u64) -> WriteOp {
    let mut output_data = [0; 12];
    let interval_ceiling = archive_info.interval_ceiling(point.timestamp);
    // debug!("interval_ceiling: {}", interval_ceiling);

    {
        let point_value = point.value;
        let buf : &mut [u8] = &mut output_data;
        point::fill_buf(buf, interval_ceiling, point_value);
    }

    let seek_info = archive_info.calculate_seek(&point,  base_timestamp);

    return WriteOp {
        seek: seek_info,
        bytes: output_data
    }
}
<commit_msg>More pretty-er print<commit_after>use std::fs::File;
use std::io::{SeekFrom, Seek, Read, Write};
use std::fs::OpenOptions;
use std::fmt;
use num::iter::range_step_inclusive;
use std::cell::RefCell;
use std::io::Error;

extern crate libc;
use self::libc::funcs::posix01::unistd::ftruncate;
use std::os::unix::prelude::AsRawFd;

use super::header::{ Header, read_header };
use super::write_op::WriteOp;
use super::archive_info::{ ArchiveInfo };
use super::metadata::{Metadata, AggregationType};
use whisper::schema::Schema;

use whisper::point;

pub struct WhisperFile<'a> {
    pub path: &'a str,
    pub handle: RefCell<File>,
    pub header: Header
}

impl<'a> fmt::Debug for WhisperFile<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path:  {:?}, header: {:?}", self.path, self.header)
    }
}

impl<'a> fmt::Display for WhisperFile<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ref metadata = self.header.metadata;
        let ref archive_infos = self.header.archive_infos;

        try!(writeln!(f, "whisper file ({})", self.path));
        try!(writeln!(f, "  metadata"));
        try!(writeln!(f, "    aggregation method: {:?}", metadata.aggregation_type));
        try!(writeln!(f, "    max retention: {:?}", metadata.max_retention));
        try!(writeln!(f, "    xff: {:?}", metadata.x_files_factor));

        let mut index = 0;
        for archive_info in archive_infos.iter() {
        // Archive details
        try!(writeln!(f, "  archive {}", index));
        try!(writeln!(f, "    seconds per point: {}", archive_info.seconds_per_point));
        try!(writeln!(f, "    points: {}", archive_info.points));
        try!(writeln!(f, "    retention: {} (s)", archive_info.retention));
        try!(write!(f, "    size: {} (bytes)\n", archive_info.size_in_bytes));

        // Print out all the data from this archive
        let points : Vec<point::Point> = Vec::with_capacity(archive_info.points as usize);
        try!(writeln!(f, "{:?}", points));

        if index != archive_infos.len() - 1 {
            try!(writeln!(f, ""));
        }
        index = index+1;
        }
        write!(f,"") // make the types happy
    }
}

pub fn open(path: &str) -> Result<WhisperFile, Error> {
    let file = try!(OpenOptions::new().read(true).write(true).create(false).open(path));
    let header = try!(read_header(&file));
    let whisper_file = WhisperFile { path: path, header: header, handle: RefCell::new(file) };
    Ok(whisper_file)
}

impl<'a> WhisperFile<'a> {

    pub fn new(path: &str, schema: Schema /* , _: Metadata */) -> Result<WhisperFile, Error> {
        let size_needed = schema.size_on_disk();
        let opened_file = try!(OpenOptions::new().read(true).write(true).create(true).open(path));

        // Allocate the room necessary
        debug!("allocating...");
        {
            let raw_fd = opened_file.as_raw_fd();
            let retval = unsafe {
                // TODO skip to fallocate-like behavior. Will need wrapper for OSX.
                ftruncate(raw_fd, size_needed as i64)
            };
            if retval != 0 {
                return Err(Error::last_os_error());
            }
        }
        debug!("done allocating");

        let metadata = {
            // TODO make agg_t, max_r options from the command line.
            let aggregation_type = AggregationType::Average;
            let x_files_factor = 0.5;
            Metadata {
                aggregation_type: aggregation_type,
                max_retention: schema.max_retention() as u32,
                x_files_factor: x_files_factor,
                archive_count: schema.retention_policies.len() as u32
            }
        };

        // Piggy back on moving file write forward
        metadata.write(&opened_file);

        let initial_archive_offset = schema.header_size_on_disk();
        schema.retention_policies.iter().fold(initial_archive_offset, |offset, &rp| {
            debug!("sup");
            rp.write(&opened_file, offset);
            offset + rp.size_on_disk()
        });

        let new_whisper_file = WhisperFile {
            path: path,
            handle: RefCell::new(opened_file),
            header: Header {
                metadata: metadata,
                archive_infos: vec![]
            }
        };
        Ok(new_whisper_file)
    }

    // TODO: Result<usize> return how many write ops were done
    pub fn write(&mut self, current_time: u64, point: point::Point) {

        match self.split(current_time, point.timestamp) {
            Some( (high_precision_archive, rest) ) => {
                let base_point = self.read_point(high_precision_archive.offset);
                let base_timestamp = base_point.timestamp;

                self.write_archives(
                    (high_precision_archive, rest),
                    point,
                    base_timestamp
                );
            },
            None => {
                panic!("no archives satisfy current time")
            }
        }

    }

    fn perform_write_op(&self, write_op: &WriteOp) {
        let mut handle = self.handle.borrow_mut();
        handle.seek(write_op.seek).unwrap();
        handle.write_all(&(write_op.bytes)).unwrap();
    }

    fn read_point(&self, offset: u64) -> point::Point {
        let mut file = self.handle.borrow_mut();
        file.seek(SeekFrom::Start(offset)).unwrap();

        let mut points_buf : [u8; 12] = [0; 12];
        let mut buf_ref : &mut [u8] = &mut points_buf;
        file.read(buf_ref).unwrap();

        point::buf_to_point(buf_ref)
    }

    pub fn write_archives(&self, (ai,rest): (&ArchiveInfo, Vec<&ArchiveInfo>), point: point::Point, base_timestamp: u64) {
        {
            let write_op = build_write_op( ai, &point, base_timestamp );
            self.perform_write_op(&write_op);
        }

        if rest.len() > 1 {
            self.downsample(ai, rest[0], point.timestamp).map(|write_op| self.perform_write_op(&write_op) );

            let high_res_iter = rest[0..rest.len()-1].into_iter();
            let low_res_iter = rest[1..].into_iter();
            let _ : Vec<()> = high_res_iter.
                zip(low_res_iter).
                take_while(|&(h,l)| {
                    match self.downsample(h, l, point.timestamp) {
                        Some(write_op) => {
                            self.perform_write_op(&write_op);
                            true
                        },
                        None => false
                    }
                }).
                map(|_| ()).
                collect();
        }
    }

    // The most expensive IO functionality
    // Reads many samples from high-res archive and
    // aggregates to lower-res archive. Schemas could do well to avoid
    // aggregation unless disk space is truly at a premium.
    //
    // A cache for each archive would do well here. `memmap` would be awesomesauce.
    fn downsample(&self, h_res_archive: &ArchiveInfo, l_res_archive: &ArchiveInfo, base_timestamp: u64) -> Option<WriteOp> {
        assert!(h_res_archive.seconds_per_point < l_res_archive.seconds_per_point);

        let l_interval_start = l_res_archive.interval_ceiling(base_timestamp);
        // debug!("base_timestamp: {}, l_interval_start: {}", base_timestamp, l_interval_start);

        let h_base_timestamp = self.read_point(h_res_archive.offset).timestamp;
        let h_res_start_offset : u64 = if h_base_timestamp == 0 {
            h_res_archive.offset
        } else {
            // TODO: this can be negative. Does that change timestamp understanding?
            let timespan  = l_interval_start as i64 - h_base_timestamp as i64;
            // debug!("timespan {}", timespan);
            let points = timespan / h_res_archive.seconds_per_point as i64;
            // debug!("points {}", points);
            let bytes = points * point::POINT_SIZE as i64;

            // TODO: OMG, move this craziness somewhere else
            let wrapped_index = {
                let remainder = bytes % h_res_archive.size_in_bytes as i64;
                if remainder < 0 {
                    h_res_archive.size_in_bytes as i64 + remainder
                } else {
                    remainder
                }
            };
            // debug!("offset: {}, wrapped_index: {}, bytes: {}, points: {}", h_res_archive.offset, wrapped_index, bytes, points);
            (h_res_archive.offset as i64 + wrapped_index) as u64
        };

        let h_res_points_needed = l_res_archive.seconds_per_point / h_res_archive.seconds_per_point;
        let h_res_bytes_needed = h_res_points_needed * point::POINT_SIZE as u64;

        let h_res_end_offset = {
            let rel_first_offset = h_res_start_offset - h_res_archive.offset;
            let rel_second_offset = (rel_first_offset + h_res_bytes_needed) % h_res_archive.size_in_bytes;
            h_res_archive.offset + rel_second_offset
        };

        let mut h_res_read_buf : Vec<u8> = Vec::with_capacity(h_res_bytes_needed as usize);

        // Subroutine for filling in the buffer
        {
            let mut handle = self.handle.borrow_mut();

            // TODO: must be a better way of zeroing out a buffer to fill in the vector
            for _ in 0..h_res_bytes_needed {
                h_res_read_buf.push(0);
            }

            // TODO: refactor in to function which
            // returns ((Seek,BytesRead),Option<(Seek,BytesRead)>)
            // so this code can be refactored and unit tested...
            if h_res_start_offset < h_res_end_offset {
                // No wrap situation
                let seek = SeekFrom::Start(h_res_start_offset);

                let mut read_buf : &mut [u8] = &mut h_res_read_buf[..];
                handle.seek(seek).unwrap();
                // debug!("READ FROM {} to {} (contiguous read)", h_res_start_offset, h_res_start_offset + read_buf.len() as u64);
                handle.read(read_buf).unwrap();
            } else {
                let high_res_abs_end = h_res_archive.offset + h_res_archive.size_in_bytes;
                let first_seek = SeekFrom::Start(h_res_start_offset);
                let first_seek_bytes = high_res_abs_end - h_res_start_offset;

                // debug!("READ FROM {} to {} (wrap-around read 1)", h_res_start_offset, h_res_start_offset+first_seek_bytes);

                let (first_buf, second_buf) = h_res_read_buf.split_at_mut(first_seek_bytes as usize);

                handle.seek(first_seek).unwrap();
                handle.read(first_buf).unwrap();

                let second_seek = SeekFrom::Start(h_res_archive.offset);
                // debug!("READ FROM {} to {} (wrap-around read 2)", h_res_archive.offset, h_res_archive.offset + second_buf.len() as u64);

                handle.seek(second_seek).unwrap();
                handle.read(second_buf).unwrap();
            }

        }

        let low_res_aggregate = {
            let points : Vec<point::Point> = h_res_read_buf.chunks(point::POINT_SIZE).map(|chunk| {
                point::buf_to_point(chunk)

            }).collect();

            let timestamp_start = l_interval_start;
            let timestamp_stop = l_interval_start + (h_res_points_needed as u64)*h_res_archive.seconds_per_point;
            let step = h_res_archive.seconds_per_point;

            let expected_timestamps = range_step_inclusive(timestamp_start, timestamp_stop, step);
            let valid_points : Vec<&point::Point> = expected_timestamps.
                zip(points.iter()).
                map(|(ts, p)| {
                    if p.timestamp == ts {
                        // debug!("comparing {} and {}/{} (MATCH!)", ts, p.timestamp, p.value);
                        Some(p)
                    } else {
                        // debug!("comparing {} and {}/{}", ts, p.timestamp, p.value);
                        None
                    }
                }).filter(|agg| !agg.is_none()).map(|agg| agg.unwrap()).collect();
            self.aggregate_samples(valid_points, h_res_points_needed)
        };

        low_res_aggregate.map(|aggregate| {
            let l_res_base_point = self.read_point(l_res_archive.offset);
            let l_res_point = point::Point{ timestamp: l_interval_start, value: aggregate };
            build_write_op(l_res_archive, &l_res_point, l_res_base_point.timestamp)
        })
    }

    fn aggregate_samples(&self, points: Vec<&point::Point>, points_possible: u64) -> Option<f64>{
        // debug!("points: {:?}", points);
        let valid_points : Vec<&&point::Point> = points.iter().filter(|p| p.timestamp != 0).map(|p| p).collect();

        let ratio : f32 = valid_points.len() as f32 / points_possible as f32;
        // debug!("valid_points: {}, len: {}, ratio {} vs xff {}", valid_points.len(), points.len(), ratio, self.header.metadata.x_files_factor);
        if ratio < self.header.metadata.x_files_factor {
            // debug!("not enough data to propagate");
            return None;
        }

        // TODO: we only do aggregation right now!
        match self.header.metadata.aggregation_type {
            AggregationType::Average => {
                let sum = points.iter().map(|p| p.value).fold(0.0, |l, r| l + r);
                Some(sum / points.len() as f64)
            },
            _ => { Some(0.0) }
        }
    }

    fn split(&self, current_time: u64, point_timestamp: u64) -> Option<(&ArchiveInfo, Vec<&ArchiveInfo>)>  {
        let mut archive_iter = self.header.archive_infos.iter();

        let high_precision_archive_option = archive_iter.find(|ai|
            ai.retention > (current_time - point_timestamp)
        );

        match high_precision_archive_option {
            Some(ai) => {
                let rest_of_archives = archive_iter;
                let low_res_archives : Vec<&ArchiveInfo> = rest_of_archives.collect();

                Some((ai, low_res_archives))
            },
            None => {
                None
            }
        }
    }
}

fn build_write_op(archive_info: &ArchiveInfo, point: &point::Point, base_timestamp: u64) -> WriteOp {
    let mut output_data = [0; 12];
    let interval_ceiling = archive_info.interval_ceiling(point.timestamp);
    // debug!("interval_ceiling: {}", interval_ceiling);

    {
        let point_value = point.value;
        let buf : &mut [u8] = &mut output_data;
        point::fill_buf(buf, interval_ceiling, point_value);
    }

    let seek_info = archive_info.calculate_seek(&point,  base_timestamp);

    return WriteOp {
        seek: seek_info,
        bytes: output_data
    }
}
<|endoftext|>