<commit_before>use std::str::FromStr;
use common::Atom;

#[derive(Debug, PartialEq)]
enum LexerErrorCode {
    InvalidSyntax,
    TrailingCharacters,
    EOFWhileReadingToken,
}

#[derive(Debug, PartialEq)]
enum LexerError {
    SyntaxError {
        code: LexerErrorCode,
        line: usize,
        col: usize,
    }
}

#[derive(Debug, PartialEq)]
enum ParseAtomError {
    IncorrectSymbolName
}

impl FromStr for Atom {
    type Err = ParseAtomError;

    fn from_str(s: &str) -> Result<Atom, ParseAtomError> {
        match s.parse::<f64>() {
            Ok(f) => {
                Ok(Atom::Number(f))
            },
            _ => {
                match s.chars().next() {
                    Some(c) if !c.is_numeric() => {
                        Ok(Atom::Symbol(s.to_string()))
                    },
                    _ => {
                        Err(ParseAtomError::IncorrectSymbolName)
                    }
                }
            }
        }
    }
}

#[derive(Debug, PartialEq)]
enum Token {
    Nil,
    Number(f64),
    String(String),
    Error(LexerError),
}

enum LexerState {
    Start,
    BeforeFinish,
    Finish,
}

pub struct Lexer<T> {
    reader: T,
    cur_char: Option<char>,
    line: usize,
    col: usize,
    state: LexerState
}

impl<T: Iterator<Item=char>> Iterator for Lexer<T> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.state {
            LexerState::Finish => {
                None
            },
            LexerState::BeforeFinish => {
                self.read_whitespaces();
                self.cur_char
                    .and(Some(self.error_token(LexerErrorCode::TrailingCharacters)))
                    .or_else(|| { self.state = LexerState::Finish; None })
            },
            _ => {
                Some(self.read())
            }
        }
    }
}

impl<T: Iterator<Item=char>> Lexer<T> {
    fn new(reader: T) -> Lexer<T> {
        let mut l = Lexer {
            reader: reader,
            cur_char: None,
            line: 1,
            col: 0,
            state: LexerState::Start
        };
        l.bump();
        l
    }

    fn read(&mut self) -> Token {
        loop {
            self.read_whitespaces();
            match self.state {
                LexerState::Start => {
                    return self.read_start()
                },
                _ => {
                    return self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        }
    }

    fn read_start(&mut self) -> Token {
        let token = self.read_token();
        self.state = match token {
            Token::Error(_) => {
                LexerState::Finish
            },
            _ => LexerState::BeforeFinish,
        };
        token
    }

    fn read_token(&mut self) -> Token {
        if let Some(c) = self.cur_char {
            match c {
                'n' => {
                    self.read_ident("il", Token::Nil)
                },
                '0' ... '9' | '-' => {
                    self.read_number()
                },
                '"' => {
                    self.read_string()
                },
                _ => {
                    self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        } else {
            self.error_token(LexerErrorCode::EOFWhileReadingToken)
        }
    }

    fn read_ident(&mut self, ident: &str, token: Token) -> Token {
        if ident.chars().all(|c| { self.bump(); Some(c) == self.cur_char }) {
            self.bump();
            token
        } else {
            self.error_token(LexerErrorCode::InvalidSyntax)
        }
    }

    fn read_number(&mut self) -> Token {
        let mut neg = false;

        if let Some('-') = self.cur_char {
            neg = true;
            self.bump();
        }

        let mut accum = 0_f64;

        while let Some(c) = self.cur_char {
            match c {
                '0' ... '9' => {
                    let c_as_usize = (c as usize) - ('0' as usize);
                    accum *= 10_f64;
                    accum += c_as_usize as f64;
                    self.bump()
                },
                '.' => {
                    self.bump();

                    let mut dec = 1.0;
                    while let Some(c) = self.cur_char {
                        match c {
                            '0' ... '9' => {
                                let c_as_usize = (c as usize) - ('0' as usize);
                                dec /= 10.0;
                                accum += (c_as_usize as f64) * dec;
                                self.bump();
                            },
                            _ => {
                                break
                            }
                        }
                    }
                }
                _ => {
                    break
                }
            }
        }

        if neg {
            accum *= -1_f64;
        }

        Token::Number(accum)
    }

    fn read_string(&mut self) -> Token {
        let mut res = String::new();

        self.bump();

        while let Some(c) = self.cur_char {
            match c {
                '"' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        self.bump();

        Token::String(res)
    }

    fn read_whitespaces(&mut self) {
        while let Some(ch) = self.cur_char {
            if [' ', '\n', '\r', '\t'].contains(&ch) {
                self.bump()
            } else {
                break
            }
        }
    }

    fn bump(&mut self) {
        self.cur_char = self.reader.next();
        if Some('\n') == self.cur_char {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
    }

    fn error_token(&mut self, ec: LexerErrorCode) -> Token {
        self.state = LexerState::Finish;
        Token::Error(LexerError::SyntaxError {
            code: ec,
            line: self.line,
            col: self.col
        })
    }
}

fn tokenize(s: &str) -> Vec<String> {
    s.replace("("," ( ")
        .replace(")", " ) ")
        .replace("\n", " ")
        .split(' ')
        .filter(|s| { !s.is_empty() })
        .map(|s| { s.to_string() })
        .collect()
}

#[cfg(test)]
mod tests {
    use common::Atom::{self, Symbol, Number};
    use super::{Lexer, Token};
    use super::ParseAtomError::IncorrectSymbolName;
    use super::tokenize;

    #[test]
    fn test_read_nil() {
        let mut lexer = Lexer::new("nil".chars());
        assert_eq!(Some(Token::Nil), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_integer_as_float() {
        let mut lexer = Lexer::new("64".chars());
        assert_eq!(Some(Token::Number(64_f64)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_float() {
        let mut lexer = Lexer::new("64.5".chars());
        assert_eq!(Some(Token::Number(64.5)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_string() {
        let s = "rust is beautiful".to_string();
        let actual_input = format!(r#""{}""#, s);
        let mut lexer = Lexer::new(actual_input.chars());
        assert_eq!(Some(Token::String(s)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_symbol() {
        assert_eq!(Symbol("name".to_string()), "name".parse::<Atom>().ok().unwrap())
    }

    #[test]
    fn test_read_incorrect_symbol_starting_with_digit() {
        assert_eq!(IncorrectSymbolName, "6name".parse::<Atom>().err().unwrap())
    }

    #[test]
    fn test_read_dense_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize("(def a 1)"))
    }

    #[test]
    fn test_read_sparse_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize(" ( \n def a\n1)   \n"))
    }
}
<commit_msg>Generalize symbol reading for custom and predefined symbols<commit_after>use std::str::FromStr;
use common::Atom;

#[derive(Debug, PartialEq)]
enum LexerErrorCode {
    InvalidSyntax,
    TrailingCharacters,
    EOFWhileReadingToken,
}

#[derive(Debug, PartialEq)]
enum LexerError {
    SyntaxError {
        code: LexerErrorCode,
        line: usize,
        col: usize,
    }
}

#[derive(Debug, PartialEq)]
enum ParseAtomError {
    IncorrectSymbolName
}

impl FromStr for Atom {
    type Err = ParseAtomError;

    fn from_str(s: &str) -> Result<Atom, ParseAtomError> {
        match s.parse::<f64>() {
            Ok(f) => {
                Ok(Atom::Number(f))
            },
            _ => {
                match s.chars().next() {
                    Some(c) if !c.is_numeric() => {
                        Ok(Atom::Symbol(s.to_string()))
                    },
                    _ => {
                        Err(ParseAtomError::IncorrectSymbolName)
                    }
                }
            }
        }
    }
}

#[derive(Debug, PartialEq)]
enum Token {
    Number(f64),
    String(String),
    Symbol(String),
    Error(LexerError),
}

enum LexerState {
    Start,
    BeforeFinish,
    Finish,
}

pub struct Lexer<T> {
    reader: T,
    cur_char: Option<char>,
    line: usize,
    col: usize,
    state: LexerState
}

impl<T: Iterator<Item=char>> Iterator for Lexer<T> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.state {
            LexerState::Finish => {
                None
            },
            LexerState::BeforeFinish => {
                self.read_whitespaces();
                self.cur_char
                    .and(Some(self.error_token(LexerErrorCode::TrailingCharacters)))
                    .or_else(|| { self.state = LexerState::Finish; None })
            },
            _ => {
                Some(self.read())
            }
        }
    }
}

impl<T: Iterator<Item=char>> Lexer<T> {
    fn new(reader: T) -> Lexer<T> {
        let mut l = Lexer {
            reader: reader,
            cur_char: None,
            line: 1,
            col: 0,
            state: LexerState::Start
        };
        l.bump();
        l
    }

    fn read(&mut self) -> Token {
        loop {
            self.read_whitespaces();
            match self.state {
                LexerState::Start => {
                    return self.read_start()
                },
                _ => {
                    return self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        }
    }

    fn read_start(&mut self) -> Token {
        let token = self.read_token();
        self.state = match token {
            Token::Error(_) => {
                LexerState::Finish
            },
            _ => LexerState::BeforeFinish,
        };
        token
    }

    fn read_token(&mut self) -> Token {
        if let Some(c) = self.cur_char {
            match c {
                'a' ... 'z' | 'A' ... 'Z' => {
                    self.read_symbol()
                },
                '0' ... '9' | '-' => {
                    self.read_number()
                },
                '"' => {
                    self.read_string()
                },
                _ => {
                    self.error_token(LexerErrorCode::InvalidSyntax)
                }
            }
        } else {
            self.error_token(LexerErrorCode::EOFWhileReadingToken)
        }
    }

    fn read_symbol(&mut self) -> Token {
        let mut res = String::new();

        while let Some(c) = self.cur_char {
            match c {
                ' ' | '\t' | '\r' | '\n' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        Token::Symbol(res)
    }

    fn read_number(&mut self) -> Token {
        let mut neg = false;

        if let Some('-') = self.cur_char {
            neg = true;
            self.bump();
        }

        let mut accum = 0_f64;

        while let Some(c) = self.cur_char {
            match c {
                '0' ... '9' => {
                    let c_as_usize = (c as usize) - ('0' as usize);
                    accum *= 10_f64;
                    accum += c_as_usize as f64;
                    self.bump()
                },
                '.' => {
                    self.bump();

                    let mut dec = 1.0;
                    while let Some(c) = self.cur_char {
                        match c {
                            '0' ... '9' => {
                                let c_as_usize = (c as usize) - ('0' as usize);
                                dec /= 10.0;
                                accum += (c_as_usize as f64) * dec;
                                self.bump();
                            },
                            _ => {
                                break
                            }
                        }
                    }
                }
                _ => {
                    break
                }
            }
        }

        if neg {
            accum *= -1_f64;
        }

        Token::Number(accum)
    }

    fn read_string(&mut self) -> Token {
        let mut res = String::new();

        self.bump();

        while let Some(c) = self.cur_char {
            match c {
                '"' => {
                    break
                },
                _ => {
                    res.push(c);
                }
            }
            self.bump();
        }

        self.bump();

        Token::String(res)
    }

    fn read_whitespaces(&mut self) {
        while let Some(ch) = self.cur_char {
            if [' ', '\n', '\r', '\t'].contains(&ch) {
                self.bump()
            } else {
                break
            }
        }
    }

    fn bump(&mut self) {
        self.cur_char = self.reader.next();
        if Some('\n') == self.cur_char {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
    }

    fn error_token(&mut self, ec: LexerErrorCode) -> Token {
        self.state = LexerState::Finish;
        Token::Error(LexerError::SyntaxError {
            code: ec,
            line: self.line,
            col: self.col
        })
    }
}

fn tokenize(s: &str) -> Vec<String> {
    s.replace("("," ( ")
        .replace(")", " ) ")
        .replace("\n", " ")
        .split(' ')
        .filter(|s| { !s.is_empty() })
        .map(|s| { s.to_string() })
        .collect()
}

#[cfg(test)]
mod tests {
    use common::Atom::{self, Symbol, Number};
    use super::{Lexer, Token};
    use super::ParseAtomError::IncorrectSymbolName;
    use super::tokenize;

    #[test]
    fn test_read_nil() {
        let sym_name = "nil".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_integer_as_float() {
        let mut lexer = Lexer::new("64".chars());
        assert_eq!(Some(Token::Number(64_f64)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_float() {
        let mut lexer = Lexer::new("64.5".chars());
        assert_eq!(Some(Token::Number(64.5)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_string() {
        let s = "rust is beautiful".to_string();
        let actual_input = format!(r#""{}""#, s);
        let mut lexer = Lexer::new(actual_input.chars());
        assert_eq!(Some(Token::String(s)), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_symbol() {
        let sym_name = "my-symbol".to_string();
        let mut lexer = Lexer::new(sym_name.chars());
        assert_eq!(Some(Token::Symbol(sym_name.clone())), lexer.next());
        assert_eq!(None, lexer.next());
    }

    #[test]
    fn test_read_incorrect_symbol_starting_with_digit() {
        assert_eq!(IncorrectSymbolName, "6name".parse::<Atom>().err().unwrap())
    }

    #[test]
    fn test_read_dense_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize("(def a 1)"))
    }

    #[test]
    fn test_read_sparse_expression() {
        let expected_result = ["(", "def", "a", "1", ")"]
                                .iter()
                                .map(|s| { s.to_string() })
                                .collect();
        assert_eq!(expected_result, tokenize(" ( \n def a\n1)   \n"))
    }
}
<|endoftext|>


<commit_before>
mod single;

use ::prelude::*;
use ::core::ops::{ Deref };
use ::mem::paging::PAGE_SIZE;
use ::utility::log2_ceil;
use ::mem::memory_map::{ MMAP_MAX_LEN as MAX_FRAMES_CNT, MemoryMap };
use self::single::Single;

#[derive(PartialEq, Eq)]
pub struct BuddyBox {
    single_num: usize,
    pointer: NonZero<*mut u8>,
}

/// Similar to std::boxed::Box.
/// Provides ownership for an allocation, and drop its content when it go out of scope.
impl Deref for BuddyBox {
    type Target = *mut u8;
    fn deref(&self) -> &Self::Target {
        &(*self.pointer)
    }
}

impl Drop for BuddyBox {
    fn drop(&mut self) {
        unsafe { BuddyAllocator::get_instance().deallocate(self) };
    }
}

pub struct BuddyAllocator {
    singles: [Option<Shared<Single>>; MAX_FRAMES_CNT],
}

static mut INSTANCE: BuddyAllocator = BuddyAllocator { singles: [None; MAX_FRAMES_CNT] };
static mut INITIALIZED: bool = false;

impl BuddyAllocator {

    /// unsafe because it depends on mmap correctness
    pub unsafe fn init_default(mmap: &MemoryMap) {
        if !INITIALIZED {
            let mut cnt = 0;
            for entry in mmap.iter() {
                if let Some(single_ref) = Single::new(entry) {
                    INSTANCE.singles[cnt] = Some(Shared::new(single_ref));
                    cnt += 1;
                }
            }
            INITIALIZED = true;
        }
    }

    pub fn get_instance() -> &'static mut Self {
        unsafe { &mut INSTANCE }
    }

    pub fn allocate_level(&mut self, level: usize) -> Option<BuddyBox> {
        let mut num = 0;
        while let Some(ptr) = self.singles[num] {
            if let Some(address) = unsafe{ (**ptr).allocate(level) } {
                return Some(BuddyBox { single_num: num, pointer: address });
            }
            num += 1;
        }

        None
    }

    pub fn allocate(&mut self, size: usize) -> Option<BuddyBox> {
        self.allocate_level(Self::size_to_level(size))
    }

    pub fn size_to_level(size: usize) -> usize {
        debug_assert!(size > 0);
        log2_ceil((size + PAGE_SIZE - 1) / PAGE_SIZE)
    }

    unsafe fn deallocate(&mut self, bbox: &BuddyBox) {
        (**(self.singles[bbox.single_num].unwrap())).deallocate(bbox.pointer);
    }
}

#[cfg(os_test)]
pub mod buddy_tests {
    use super::*;

    fn size_to_level_test() {
        assert_eq!(0, BuddyAllocator::size_to_level(1));
        assert_eq!(0, BuddyAllocator::size_to_level(4000));
        assert_eq!(0, BuddyAllocator::size_to_level(4096));
        assert_eq!(1, BuddyAllocator::size_to_level(4097));
        assert_eq!(1, BuddyAllocator::size_to_level(4096 * 2));
        assert_eq!(2, BuddyAllocator::size_to_level(4096 * 2 + 1));
        assert_eq!(5, BuddyAllocator::size_to_level(4096 * (1 << 5)));
        assert_eq!(6, BuddyAllocator::size_to_level(4096 * (1 << 6) - 1));
    }

    fn allocate_test() {
        let allocator = BuddyAllocator::get_instance();
        let page1 = allocator.allocate(123).unwrap();
        let page2 = allocator.allocate(123).unwrap();
        let page3 = allocator.allocate(4096 * 2).unwrap();
        let page4 = allocator.allocate(4096 * 10).unwrap();

        assert!(*page1 as usize % 4096 == 0);
        assert!(*page2 as usize % 4096 == 0);
        assert!(*page3 as usize % 4096 == 0);
        assert!(*page4 as usize % 4096 == 0);

        assert!(page1 != page2);
        assert!(page1 != page3);
        assert!(page1 != page4);
        assert!(page2 != page3);
        assert!(page2 != page4);
        assert!(page3 != page4);
    }

    fn allocate_big_twice_test() {
        const N: usize = 32;

        let allocate_max = || generate![max_possible_page(); N];

        // assert allocate_max correctness
        {
            let _foo = allocate_max();
            let bar = allocate_max();  // must be empty
            for x in bar.iter() {
                assert!(x.is_none());
            }
        }

        let _page = BuddyAllocator::get_instance().allocate_level(0);
        let allocate_max_levels = || {
            let pages = allocate_max();
            let mut levels = [0; N];
            for (i, item) in pages.iter().enumerate() {
                levels[i] = match item.as_ref() {
                    Some(pair) => pair.1 as isize,
                    None => -1,
                };
            }
            levels
        };

        {
            let levels1 = allocate_max_levels();
            let levels2 = allocate_max_levels();
            assert_eq!(levels1, levels2);
        }

        {
            let levels1 = allocate_max_levels();
            let _page = BuddyAllocator::get_instance().allocate_level(0);
            let levels2 = allocate_max_levels();
            assert_ne!(levels1, levels2);
        }

    }

    fn max_possible_page() -> Option<(BuddyBox, usize)> {
        let allocator = BuddyAllocator::get_instance();
        for level in (0..32).rev() {
            let res = allocator.allocate_level(level);
            if let Some(page) = res {
                return Some((page, level));
            }
        }
        None
    }

    pub fn all() {
        size_to_level_test();
        allocate_test();
        allocate_big_twice_test();
    }
}
<commit_msg>comments added<commit_after>
mod single;

use ::prelude::*;
use ::core::ops::Deref;
use ::mem::paging::PAGE_SIZE;
use ::utility::log2_ceil;
use ::mem::memory_map::{ MMAP_MAX_LEN as MAX_FRAMES_CNT, MemoryMap };
use self::single::Single;

#[derive(PartialEq, Eq)]
pub struct BuddyBox {
    single_num: usize,
    pointer: NonZero<*mut u8>,
}

/// Similar to std::boxed::Box.
/// Provides ownership for an allocation, and drop its content when it go out of scope.
impl Deref for BuddyBox {
    type Target = *mut u8;
    fn deref(&self) -> &Self::Target {
        &(*self.pointer)
    }
}

impl Drop for BuddyBox {
    fn drop(&mut self) {
        unsafe { BuddyAllocator::get_instance().deallocate(self) };
    }
}

pub struct BuddyAllocator {
    singles: [Option<Shared<Single>>; MAX_FRAMES_CNT],
}

static mut INSTANCE: BuddyAllocator = BuddyAllocator { singles: [None; MAX_FRAMES_CNT] };
static mut INITIALIZED: bool = false;

impl BuddyAllocator {

    /// unsafe because it depends on mmap correctness
    pub unsafe fn init_default(mmap: &MemoryMap) {
        if !INITIALIZED {
            let mut cnt = 0;
            for entry in mmap.iter() {
                if let Some(single_ref) = Single::new(entry) {
                    INSTANCE.singles[cnt] = Some(Shared::new(single_ref));
                    cnt += 1;
                }
            }
            INITIALIZED = true;
        }
    }

    pub fn get_instance() -> &'static mut Self {
        unsafe { &mut INSTANCE }
    }

    pub fn allocate_level(&mut self, level: usize) -> Option<BuddyBox> {
        let mut num = 0;
        while let Some(ptr) = self.singles[num] {
            if let Some(address) = unsafe{ (**ptr).allocate(level) } {
                return Some(BuddyBox { single_num: num, pointer: address });
            }
            num += 1;
        }

        None
    }

    pub fn allocate(&mut self, size: usize) -> Option<BuddyBox> {
        self.allocate_level(Self::size_to_level(size))
    }

    pub fn size_to_level(size: usize) -> usize {
        debug_assert!(size > 0);
        log2_ceil((size + PAGE_SIZE - 1) / PAGE_SIZE)
    }

    unsafe fn deallocate(&mut self, bbox: &BuddyBox) {
        (**(self.singles[bbox.single_num].unwrap())).deallocate(bbox.pointer);
    }
}

#[cfg(os_test)]
pub mod buddy_tests {
    use super::*;

    fn size_to_level_test() {
        assert_eq!(0, BuddyAllocator::size_to_level(1));
        assert_eq!(0, BuddyAllocator::size_to_level(4000));
        assert_eq!(0, BuddyAllocator::size_to_level(4096));
        assert_eq!(1, BuddyAllocator::size_to_level(4097));
        assert_eq!(1, BuddyAllocator::size_to_level(4096 * 2));
        assert_eq!(2, BuddyAllocator::size_to_level(4096 * 2 + 1));
        assert_eq!(5, BuddyAllocator::size_to_level(4096 * (1 << 5)));
        assert_eq!(6, BuddyAllocator::size_to_level(4096 * (1 << 6) - 1));
    }

    fn allocate_test() {
        let allocator = BuddyAllocator::get_instance();
        let page1 = allocator.allocate(123).unwrap();
        let page2 = allocator.allocate(123).unwrap();
        let page3 = allocator.allocate(4096 * 2).unwrap();
        let page4 = allocator.allocate(4096 * 10).unwrap();

        assert!(*page1 as usize % 4096 == 0);
        assert!(*page2 as usize % 4096 == 0);
        assert!(*page3 as usize % 4096 == 0);
        assert!(*page4 as usize % 4096 == 0);

        assert!(page1 != page2);
        assert!(page1 != page3);
        assert!(page1 != page4);
        assert!(page2 != page3);
        assert!(page2 != page4);
        assert!(page3 != page4);
    }

    fn allocate_big_twice_test() {
        const N: usize = 32;

        let allocate_max = || generate![max_possible_page(); N];

        // assert allocate_max correctness
        {
            let _foo = allocate_max();
            let bar = allocate_max();  // must be empty
            for x in bar.iter() {
                assert!(x.is_none());
            }
        }

        let _page = BuddyAllocator::get_instance().allocate_level(0);
        let allocate_max_levels = || {
            let pages = allocate_max();
            let mut levels = [0; N];
            for (i, item) in pages.iter().enumerate() {
                levels[i] = match item.as_ref() {
                    Some(pair) => pair.1 as isize,
                    None => -1,
                };
            }
            levels
        };

        // Allocate all, clear, allocate all again.
        // Results of allocations must be identical.
        {
            let levels1 = allocate_max_levels();
            let levels2 = allocate_max_levels();
            assert_eq!(levels1, levels2);
        }

        // Allocate all, clear, allocate some memory, allocate all again.
        // Results of allocations must be different.
        {
            let levels1 = allocate_max_levels();
            let _page = BuddyAllocator::get_instance().allocate_level(0);
            let levels2 = allocate_max_levels();
            assert_ne!(levels1, levels2);
        }

    }

    fn max_possible_page() -> Option<(BuddyBox, usize)> {
        let allocator = BuddyAllocator::get_instance();
        for level in (0..32).rev() {
            let res = allocator.allocate_level(level);
            if let Some(page) = res {
                return Some((page, level));
            }
        }
        None
    }

    pub fn all() {
        size_to_level_test();
        allocate_test();
        allocate_big_twice_test();
    }
}
<|endoftext|>


<commit_before>use super::LinuxType;
use util::get_from_str;

pub fn check_rhel(id: &str, version_id: Option<&str>) -> Option<LinuxType> {
    if id == "rhel" {
        return check_rhel_version(version_id);
    }

    None
}

fn check_rhel_version(version_id: Option<&str>) -> Option<LinuxType> {
    let mut version_numbers = try_option!(version_id).split('.');

    match version_numbers.next().and_then(get_from_str::<u8>) {
        Some(i) if i >= 7 => Some(LinuxType::Rhel7),
        Some(i) if i >= 6 => Some(LinuxType::Rhel6),
        _ => None,
    }
}
<commit_msg>fix distro detection for centos<commit_after>use super::LinuxType;
use util::get_from_str;

pub fn check_rhel(id: &str, version_id: Option<&str>) -> Option<LinuxType> {
    if id == "rhel" || id == "centos" {
        return check_rhel_version(version_id);
    }

    None
}

fn check_rhel_version(version_id: Option<&str>) -> Option<LinuxType> {
    let mut version_numbers = try_option!(version_id).split('.');

    match version_numbers.next().and_then(get_from_str::<u8>) {
        Some(i) if i >= 7 => Some(LinuxType::Rhel7),
        Some(i) if i >= 6 => Some(LinuxType::Rhel6),
        _ => None,
    }
}
<|endoftext|>


<commit_before>
use std::io::Write;
use std::io::Error as IoError;
use super::{Parsable, Streamable, ParseError};

// RFC 5234, B.1  Core Rules
//const CR: u8 = 0x0D;     //   CR             =  %x0D      ; carriage return
//const LF: u8 = 0x0A;     //   LF             =  %x0A      ; linefeed
const SP: u8 = 0x20;     //   SP             =  %x20
const HTAB: u8 = 0x09;   //   HTAB           =  %x09      ; horizontal tab
//const DQUOTE: u8 = 0x22; //   DQUOTE         =  %x22      ; " (Double Quote)

// RFC 5234, B.1  Core Rules
// VCHAR           =  %x21-7E   ; visible (printing) characters)
#[inline]
pub fn is_vchar(c: u8) -> bool { c>=0x21 && c<=0x7E }
def_cclass!(VChar, is_vchar);

// RFC 5234, B.1  Core Rules  WSP            =  SP / HTAB ; white space
#[inline]
pub fn is_wsp(c: u8) -> bool { c==SP || c==HTAB }
def_cclass!(WSP, is_wsp);

// RFC 5234, B.1  Core Rules  CHAR           =  %x01-7F ; any 7-bit US-ASCII character,
//                                                      ;  excluding NUL
#[inline]
pub fn is_ascii(c: u8) -> bool { c>=1 && c<=127 }
def_cclass!(ASCII, is_ascii);

// RFC 5234, B.1  Core Rules  DIGIT          =  %x30-39   ; 0-9
#[inline]
pub fn is_digit(c: u8) -> bool { c>=0x30 && c<=0x39 }
def_cclass!(Digit, is_digit);

// RFC 5234, B.1  Core Rules  ALPHA          = %x41-5A / %x61-7A   ; A-Z / a-z
#[inline]
pub fn is_alpha(c: u8) -> bool { (c>=0x41 && c<=0x5A) || (c>=0x61 && c<=0x7A) }
def_cclass!(Alpha, is_alpha);

// 3.2.1
// quoted-pair     =   ("\" (VCHAR / WSP)) / obs-qp
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct QuotedPair(pub u8);
impl Parsable for QuotedPair {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut pos: usize = 0;
        if pos >= input.len() { return Err(ParseError::Eof); }
        if pos + 1 >= input.len() { return Err(ParseError::NotFound); }
        if input[pos]!=b'\\' { return Err(ParseError::NotFound); }
        if is_vchar(input[pos + 1]) || is_wsp(input[pos + 1]) {
            pos += 2;
            let qp = QuotedPair(input[pos - 1]);
            return Ok((qp, &input[pos..]));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for QuotedPair {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        Ok(try!(w.write(b"\\"))
           + try!(w.write(&[self.0])))
    }
}

// 3.2.2
// FWS             =   ([*WSP CRLF] 1*WSP) /  obs-FWS
//                                        ; Folding white space
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FWS;
impl Parsable for FWS {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if rem.len() == 0 { return Err(ParseError::Eof); }
        while rem.len() > 0 {
            if is_wsp(rem[0]) {
                rem = &rem[1..];
            }
            else if rem.len() > 2 && &rem[0..2]==b"\r\n" && is_wsp(rem[2]) {
                rem = &rem[3..];
            }
            else {
                break;
            }
        }
        if rem.len() == input.len() { Err(ParseError::NotFound) }
        else { Ok((FWS, rem)) }
    }
}
impl Streamable for FWS {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        Ok(try!(w.write(b" "))) // FIXME - fold?
    }
}

// 3.2.2
// ctext           =   %d33-39 /          ; Printable US-ASCII
//                     %d42-91 /          ;  characters not including
//                     %d93-126 /         ;  "(", ")", or "\"
//                     obs-ctext
#[inline]
pub fn is_ctext(c: u8) -> bool { (c>=33 && c<=39) || (c>=42 && c<=91) || (c>=93 && c<=126) }
def_cclass!(CText, is_ctext);

// 3.2.2
// ccontent        =   ctext / quoted-pair / comment
#[derive(Debug, Clone, PartialEq)]
pub enum CContent {
    CText(CText),
    QuotedPair(QuotedPair),
    Comment(Comment),
}
impl Parsable for CContent {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        if let Ok((na, rem)) = CText::parse(input) {
            Ok((CContent::CText(na), rem))
        }
        else if let Ok((asp, rem)) = QuotedPair::parse(input) {
            Ok((CContent::QuotedPair(asp), rem))
        }
        else if let Ok((c, rem)) = Comment::parse(input) {
            Ok((CContent::Comment(c), rem))
        }
        else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for CContent {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            CContent::CText(ref x) => x.stream(w),
            CContent::QuotedPair(ref x) => x.stream(w),
            CContent::Comment(ref x) => x.stream(w),
        }
    }
}

// 3.2.2
// comment         =   "(" *([FWS] ccontent) [FWS] ")"
#[derive(Debug, Clone, PartialEq)]
pub struct Comment {
    pub ccontent: Vec<(bool, CContent)>, // bool representing if whitespace preceeds it
    pub trailing_ws: bool,
}
impl Parsable for Comment {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem: &[u8] = input;
        if rem.len() == 0 { return Err(ParseError::Eof); }
        req!(rem, b"(", input);
        let mut ccontent: Vec<(bool, CContent)> = Vec::new();
        let mut ws: bool = false;
        while rem.len() > 0 {
            let t = parse!(FWS, rem);
            ws = t.is_ok();
            if let Ok(cc) = parse!(CContent, rem) {
                ccontent.push((ws, cc));
                continue;
            }
            break;
        }
        req!(rem, b")", input);
        return Ok((Comment {
            ccontent: ccontent,
            trailing_ws: ws,
        }, rem));
    }
}
impl Streamable for Comment {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(w.write(b"("));
        for &(ws, ref cc) in &self.ccontent {
            if ws { count += try!(w.write(b" ")) }
            count += try!(cc.stream(w));
        }
        if self.trailing_ws { count += try!(w.write(b" ")) }
        count += try!(w.write(b")"));
        Ok(count)
    }
}

// 3.2.2
// CFWS            =   (1*([FWS] comment) [FWS]) / FWS
#[derive(Debug, Clone, PartialEq)]
pub struct CFWS {
    pub comments: Vec<(bool, Comment)>, // bool representing if whitespace preceeds it
    pub trailing_ws: bool,
}
impl Parsable for CFWS {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        if input.len() == 0 { return Err(ParseError::Eof); }
        let mut comments: Vec<(bool, Comment)> = Vec::new();
        let mut rem = input;
        let mut ws: bool = false;
        while rem.len() > 0 {
            let w = parse!(FWS, rem);
            ws = w.is_ok();
            if let Ok(comment) = parse!(Comment, rem) {
                comments.push((ws, comment));
                continue;
            }
            break;
        }
        if comments.len() > 0 || ws {
            Ok((CFWS {
                comments: comments,
                trailing_ws: ws,
            }, rem))
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for CFWS {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        for &(ws, ref comment) in &self.comments {
            if ws { count += try!(w.write(b" ")) }
            count += try!(comment.stream(w));
        }
        if self.trailing_ws { count += try!(w.write(b" ")) }
        Ok(count)
    }
}
<commit_msg>rfc5322: 3.2.3 atext<commit_after>
use std::io::Write;
use std::io::Error as IoError;
use super::{Parsable, Streamable, ParseError};

// RFC 5234, B.1  Core Rules
//const CR: u8 = 0x0D;     //   CR             =  %x0D      ; carriage return
//const LF: u8 = 0x0A;     //   LF             =  %x0A      ; linefeed
const SP: u8 = 0x20;     //   SP             =  %x20
const HTAB: u8 = 0x09;   //   HTAB           =  %x09      ; horizontal tab
//const DQUOTE: u8 = 0x22; //   DQUOTE         =  %x22      ; " (Double Quote)

// RFC 5234, B.1  Core Rules
// VCHAR           =  %x21-7E   ; visible (printing) characters)
#[inline]
pub fn is_vchar(c: u8) -> bool { c>=0x21 && c<=0x7E }
def_cclass!(VChar, is_vchar);

// RFC 5234, B.1  Core Rules  WSP            =  SP / HTAB ; white space
#[inline]
pub fn is_wsp(c: u8) -> bool { c==SP || c==HTAB }
def_cclass!(WSP, is_wsp);

// RFC 5234, B.1  Core Rules  CHAR           =  %x01-7F ; any 7-bit US-ASCII character,
//                                                      ;  excluding NUL
#[inline]
pub fn is_ascii(c: u8) -> bool { c>=1 && c<=127 }
def_cclass!(ASCII, is_ascii);

// RFC 5234, B.1  Core Rules  DIGIT          =  %x30-39   ; 0-9
#[inline]
pub fn is_digit(c: u8) -> bool { c>=0x30 && c<=0x39 }
def_cclass!(Digit, is_digit);

// RFC 5234, B.1  Core Rules  ALPHA          = %x41-5A / %x61-7A   ; A-Z / a-z
#[inline]
pub fn is_alpha(c: u8) -> bool { (c>=0x41 && c<=0x5A) || (c>=0x61 && c<=0x7A) }
def_cclass!(Alpha, is_alpha);

// 3.2.1
// quoted-pair     =   ("\" (VCHAR / WSP)) / obs-qp
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct QuotedPair(pub u8);
impl Parsable for QuotedPair {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut pos: usize = 0;
        if pos >= input.len() { return Err(ParseError::Eof); }
        if pos + 1 >= input.len() { return Err(ParseError::NotFound); }
        if input[pos]!=b'\\' { return Err(ParseError::NotFound); }
        if is_vchar(input[pos + 1]) || is_wsp(input[pos + 1]) {
            pos += 2;
            let qp = QuotedPair(input[pos - 1]);
            return Ok((qp, &input[pos..]));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for QuotedPair {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        Ok(try!(w.write(b"\\"))
           + try!(w.write(&[self.0])))
    }
}

// 3.2.2
// FWS             =   ([*WSP CRLF] 1*WSP) /  obs-FWS
//                                        ; Folding white space
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FWS;
impl Parsable for FWS {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if rem.len() == 0 { return Err(ParseError::Eof); }
        while rem.len() > 0 {
            if is_wsp(rem[0]) {
                rem = &rem[1..];
            }
            else if rem.len() > 2 && &rem[0..2]==b"\r\n" && is_wsp(rem[2]) {
                rem = &rem[3..];
            }
            else {
                break;
            }
        }
        if rem.len() == input.len() { Err(ParseError::NotFound) }
        else { Ok((FWS, rem)) }
    }
}
impl Streamable for FWS {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        Ok(try!(w.write(b" "))) // FIXME - fold?
    }
}

// 3.2.2
// ctext           =   %d33-39 /          ; Printable US-ASCII
//                     %d42-91 /          ;  characters not including
//                     %d93-126 /         ;  "(", ")", or "\"
//                     obs-ctext
#[inline]
pub fn is_ctext(c: u8) -> bool { (c>=33 && c<=39) || (c>=42 && c<=91) || (c>=93 && c<=126) }
def_cclass!(CText, is_ctext);

// 3.2.2
// ccontent        =   ctext / quoted-pair / comment
#[derive(Debug, Clone, PartialEq)]
pub enum CContent {
    CText(CText),
    QuotedPair(QuotedPair),
    Comment(Comment),
}
impl Parsable for CContent {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        if let Ok((na, rem)) = CText::parse(input) {
            Ok((CContent::CText(na), rem))
        }
        else if let Ok((asp, rem)) = QuotedPair::parse(input) {
            Ok((CContent::QuotedPair(asp), rem))
        }
        else if let Ok((c, rem)) = Comment::parse(input) {
            Ok((CContent::Comment(c), rem))
        }
        else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for CContent {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            CContent::CText(ref x) => x.stream(w),
            CContent::QuotedPair(ref x) => x.stream(w),
            CContent::Comment(ref x) => x.stream(w),
        }
    }
}

// 3.2.2
// comment         =   "(" *([FWS] ccontent) [FWS] ")"
#[derive(Debug, Clone, PartialEq)]
pub struct Comment {
    pub ccontent: Vec<(bool, CContent)>, // bool representing if whitespace preceeds it
    pub trailing_ws: bool,
}
impl Parsable for Comment {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem: &[u8] = input;
        if rem.len() == 0 { return Err(ParseError::Eof); }
        req!(rem, b"(", input);
        let mut ccontent: Vec<(bool, CContent)> = Vec::new();
        let mut ws: bool = false;
        while rem.len() > 0 {
            let t = parse!(FWS, rem);
            ws = t.is_ok();
            if let Ok(cc) = parse!(CContent, rem) {
                ccontent.push((ws, cc));
                continue;
            }
            break;
        }
        req!(rem, b")", input);
        return Ok((Comment {
            ccontent: ccontent,
            trailing_ws: ws,
        }, rem));
    }
}
impl Streamable for Comment {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(w.write(b"("));
        for &(ws, ref cc) in &self.ccontent {
            if ws { count += try!(w.write(b" ")) }
            count += try!(cc.stream(w));
        }
        if self.trailing_ws { count += try!(w.write(b" ")) }
        count += try!(w.write(b")"));
        Ok(count)
    }
}

// 3.2.2
// CFWS            =   (1*([FWS] comment) [FWS]) / FWS
#[derive(Debug, Clone, PartialEq)]
pub struct CFWS {
    pub comments: Vec<(bool, Comment)>, // bool representing if whitespace preceeds it
    pub trailing_ws: bool,
}
impl Parsable for CFWS {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        if input.len() == 0 { return Err(ParseError::Eof); }
        let mut comments: Vec<(bool, Comment)> = Vec::new();
        let mut rem = input;
        let mut ws: bool = false;
        while rem.len() > 0 {
            let w = parse!(FWS, rem);
            ws = w.is_ok();
            if let Ok(comment) = parse!(Comment, rem) {
                comments.push((ws, comment));
                continue;
            }
            break;
        }
        if comments.len() > 0 || ws {
            Ok((CFWS {
                comments: comments,
                trailing_ws: ws,
            }, rem))
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for CFWS {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        for &(ws, ref comment) in &self.comments {
            if ws { count += try!(w.write(b" ")) }
            count += try!(comment.stream(w));
        }
        if self.trailing_ws { count += try!(w.write(b" ")) }
        Ok(count)
    }
}

// 3.2.3
// atext           =   ALPHA / DIGIT /    ; Printable US-ASCII
//                     "!" / "#" /        ;  characters not including
//                     "$" / "%" /        ;  specials.  Used for atoms.
//                     "&" / "'" /
//                     "*" / "+" /
//                     "-" / "/" /
//                     "=" / "?" /
//                     "^" / "_" /
//                     "`" / "{" /
//                     "|" / "}" /
//                     "~"
#[inline]
pub fn is_atext(c: u8) -> bool {
    is_alpha(c) || is_digit(c)
        || c==b'!' || c==b'#'  || c==b'$' || c==b'%'
        || c==b'&' || c==b'\'' || c==b'*' || c==b'+'
        || c==b'-' || c==b'/'  || c==b'=' || c==b'?'
        || c==b'^' || c==b'_'  || c==b'`' || c==b'{'
        || c==b'|' || c==b'}'  || c==b'~'
}
def_cclass!(AText, is_atext);
<|endoftext|>


<commit_before>// Copyright 2013-2014 Jeffery Olson
//
// Licensed under the 3-Clause BSD License, see LICENSE.txt
// at the top-level of this repository.
// This file may not be copied, modified, or distributed
// except according to those terms.
#![crate_id="http://olsonjeffery.github.io#schemers:0.0.1"]
#![desc = "Simple Scheme Interpreter"]
#![license = "3-Clause BSD"]

use std::from_str::FromStr;

// this is starting out as a straight port of Peter Norvig's
// lis.py (the first iteration) to Rust

#[cfg(not(test))]
fn main() {
    use std::os::args;
    let args: Vec<~str> = args().move_iter().skip(1).collect();
    if args.len() == 0 {
        fail!("no program provided");
    }
    let program: ~str = args.move_iter()
        .fold(~"", |memo, arg| memo + arg + " ").trim().to_owned();
    let tokens = &mut tokenize(program);
    let parsed_items = parse(tokens);
    println!("{}", parsed_items.print());
}

// parser impl
fn parse_str(input: ~str) -> Expr {
    parse(&mut tokenize(input))
}

fn pad_input(input: ~str) -> ~str {
    input.replace("(", " ( ").replace(")", " ) ")
}

fn tokenize(input: ~str) -> Vec<~str> {
    pad_input(input).split_str(" ")
        .filter(|i| *i != "").map(|i| i.to_owned()).collect()
}

#[deriving(Eq, Show)]
pub enum Expr {
    Atom(AtomType),
    List(Vec<~Expr>)
}
impl Expr {
    pub fn new_atom(input: ~str) -> Expr {
        let first_char = input.char_at(0);
        match first_char {
            '0' | '1' | '2' | '3' | '4' | '5' |
            '6' | '7' | '8' | '9' => {
                match input.contains(".") {
                    true => {
                        match FromStr::from_str(input) {
                            Some(val) => Atom(Float(val)),
                            None => fail!("Cannot parse number-like input of: {}", input)
                        }
                    },
                    false => {
                        match FromStr::from_str(input) {
                            Some(val) => Atom(Integer(val)),
                            None => fail!("Cannot parse number-like input of: {}", input)
                        }
                    }
                }
            },
            _ => Atom(Symbol(input))
        }
    }

    pub fn is_atom(&self) -> bool {
        match *self {
            Atom(_) => true,
            _ => false
        }
    }

    pub fn print(&self) -> ~str {
        match self {
            &List(ref items) => {
                let out = items.iter().map(|i| i.print())
                    .fold(~"(", |m, v| m + v + " ");
                out.trim() + ")"
            },
            &Atom(ref v) => v.print()
        }
    }
    pub fn un_cons(self) -> (Expr, Option<Expr>) {
        match self {
            Atom(v) => (Atom(v), None),
            List(mut items) => {
                if items.len() == 0 {
                    fail!("cannot build car/cdr of empty list");
                }
                let car = items.shift()
                    .expect("at least one item in the least; shouldn't happen");
                (*car, Some(List(items)))
            }
        }
    }
}

#[deriving(Eq, Show)]
pub enum AtomType {
    Symbol(~str),
    Integer(i64),
    Float(f64)
}

impl AtomType {
    pub fn print(&self) -> ~str {
        match self {
            &Symbol(ref v) => v.to_owned(),
            &Integer(ref v) => v.to_str(),
            &Float(ref v) => v.to_str()
        }
    }
}

fn parse(tokens: &mut Vec<~str>) -> Expr {
    let current_token =
        tokens.shift().expect("calling parse() w/ empty token list; shouldn't happen");
    match current_token {
        ref x if *x == ~"(" => {
            let mut list = Vec::new();
            while *tokens.get(0) != ~")" {
                list.push(box parse(tokens));
            }
            tokens.shift();
            List(list)
        },
        ref x if *x == ~")" => fail!("hit ) token; shouldn't happen"),
        x => Expr::new_atom(x)
    }
}

#[cfg(test)]
mod parser_test {
    use super::{pad_input, tokenize, parse, Atom, List, Symbol, Integer, Float};

    #[test]
    fn pad_input_should_insert_spaces_before_and_after_parens() {
        assert_eq!(pad_input(~"(x 1 2 3)"), ~" ( x 1 2 3 ) ");
    }

    #[test]
    fn tokenize_string_into_tokens() {
        let tokens = tokenize(~"(foo 12 3)");
        assert_eq!(tokens.len(), 5);
        assert_eq!(tokens.get(0).to_owned(), ~"(");
        assert_eq!(tokens.get(1).to_owned(), ~"foo");
        assert_eq!(tokens.get(2).to_owned(), ~"12");
        assert_eq!(tokens.get(3).to_owned(), ~"3");
        assert_eq!(tokens.get(4).to_owned(), ~")");
    }

    #[test]
    fn should_parse_tokens_consisting_of_a_str_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"bar");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Symbol(val)) => {
                assert_eq!(val, ~"bar")
            },
            _ => assert!(false)
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_an_int_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"42");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Integer(val)) => {
                assert_eq!(val, 42)
            },
            _ => assert!(false)
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_a_float_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"1.1");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Float(val)) => {
                assert_eq!(val, 1.1)
            },
            _ => assert!(false)
        }
    }

    #[test]
    fn should_parse_tokens_consisting_of_a_list_of_atoms_and_parse_it() {
        let tokens = &mut tokenize(~"(bar 12 45)");
        let parsed_item = parse(tokens);
        assert_eq!(parsed_item.is_atom(), false);
        match parsed_item {
            List(items) => {
                let items_len = items.len();
                assert!(3 == items_len);
                assert_eq!(items.get(0).is_atom(), true);
                assert_eq!(*items.get(0), ~Atom(Symbol(~"bar")));
                assert_eq!(items.get(1).is_atom(), true);
                assert_eq!(*items.get(1), ~Atom(Integer(12)));
                assert_eq!(items.get(2).is_atom(), true);
                assert_eq!(*items.get(2), ~Atom(Integer(45)));
            },
            _ => fail!("got back an atom, it seems")
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_a_list_of_atoms_and_a_nested_list() {
        let tokens = &mut tokenize(~"((2 3) bar 12 (hee (hah)))");
        let parsed_item = parse(tokens);
        assert_eq!(parsed_item.is_atom(), false);
        match parsed_item {
            List(items) => {
                let items_len = items.len();
                assert!(4 == items_len);
                match *items.get(0) {
                    ~List(ref items) => {
                        assert_eq!(items.len(), 2);
                        assert_eq!(*items.get(0), ~Atom(Integer(2)));
                        assert_eq!(*items.get(1), ~Atom(Integer(3)));
                    },
                    _ => fail!("shoulda got a list")
                }
                assert_eq!(*items.get(1), ~Atom(Symbol(~"bar")));
                assert_eq!(*items.get(2), ~Atom(Integer(12)));
                match *items.get(3) {
                    ~List(ref items) => {
                        assert_eq!(items.len(), 2);
                        assert_eq!(*items.get(0), ~Atom(Symbol(~"hee")));
                        match *items.get(1) {
                            ~List(ref items) => {
                                assert_eq!(items.len(), 1);
                                assert_eq!(*items.get(0), ~Atom(Symbol(~"hah")));
                            },
                            _ => fail!("shoulda got a list")
                        }
                    },
                    _ => fail!("shoulda got a list")
                }
            },
            _ => fail!("got back an atom, it seems")
        }
    }
    #[test]
    fn parsed_tokens_of_a_list_of_atoms_and_a_nested_list_should_to_str_correctly() {
        let input = ~"((2 3) bar 12 (hee (hah)))";
        let tokens = &mut tokenize(input.to_owned());
        let parsed_item = parse(tokens);
        let output = parsed_item.print();
        assert_eq!(input, output);
    }
}

#[cfg(test)]
mod eval_test {
    mod un_cons {
        use super::super::{parse_str, Atom, List, Symbol};
        #[test]
        fn an_atom_expr_returns_the_atom_in_the_car_with_none_in_the_cdr() {
            let expr = parse_str(~"x");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"x")));
            assert_eq!(cdr, None);
        }

        #[test]
        fn a_list_with_one_atom_element_returns_an_atom_car_and_an_empty_list_in_the_cdr() {
            let expr = parse_str(~"(x)");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"x")));
            match cdr.expect("should be a value here") {
                List(items) => assert_eq!(items.len(), 0),
                _ => fail!("expected a list")
            }
        }

        #[test]
        fn a_list_with_multiple_elements_returns_the_first_car_and_the_rust_in_the_cdr() {
            let expr = parse_str(~"(y 2 3)");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"y")));
            match cdr.expect("should be a value here") {
                List(items) => {
                    assert_eq!(items.len(), 2)
                },
                _ => fail!("expected a list")
            }
        }

        #[test]
        #[should_fail]
        fn calling_extract_on_an_empty_list_should_fail() {
            let expr = parse_str(~"()");
            expr.un_cons();
        }
    }
}
<commit_msg>use parse_str in main() and add Clone deriving to Expr + AtomType<commit_after>// Copyright 2013-2014 Jeffery Olson
//
// Licensed under the 3-Clause BSD License, see LICENSE.txt
// at the top-level of this repository.
// This file may not be copied, modified, or distributed
// except according to those terms.
#![crate_id="http://olsonjeffery.github.io#schemers:0.0.1"]
#![desc = "Simple Scheme Interpreter"]
#![license = "3-Clause BSD"]

use std::from_str::FromStr;

// this is starting out as a straight port of Peter Norvig's
// lis.py (the first iteration) to Rust

#[cfg(not(test))]
fn main() {
    use std::os::args;
    let args: Vec<~str> = args().move_iter().skip(1).collect();
    if args.len() == 0 {
        fail!("no program provided");
    }
    let program: ~str = args.move_iter()
        .fold(~"", |memo, arg| memo + arg + " ").trim().to_owned();
    let parsed_items = parse_str(program);
    println!("{}", parsed_items.print());
}

// parser impl
fn parse_str(input: ~str) -> Expr {
    parse(&mut tokenize(input))
}

fn pad_input(input: ~str) -> ~str {
    input.replace("(", " ( ").replace(")", " ) ")
}

fn tokenize(input: ~str) -> Vec<~str> {
    pad_input(input).split_str(" ")
        .filter(|i| *i != "").map(|i| i.to_owned()).collect()
}

#[deriving(Eq, Show, Clone)]
pub enum Expr {
    Atom(AtomType),
    List(Vec<~Expr>)
}
impl Expr {
    pub fn new_atom(input: ~str) -> Expr {
        let first_char = input.char_at(0);
        match first_char {
            '0' | '1' | '2' | '3' | '4' | '5' |
            '6' | '7' | '8' | '9' => {
                match input.contains(".") {
                    true => {
                        match FromStr::from_str(input) {
                            Some(val) => Atom(Float(val)),
                            None => fail!("Cannot parse number-like input of: {}", input)
                        }
                    },
                    false => {
                        match FromStr::from_str(input) {
                            Some(val) => Atom(Integer(val)),
                            None => fail!("Cannot parse number-like input of: {}", input)
                        }
                    }
                }
            },
            _ => Atom(Symbol(input))
        }
    }

    pub fn is_atom(&self) -> bool {
        match *self {
            Atom(_) => true,
            _ => false
        }
    }

    pub fn print(&self) -> ~str {
        match self {
            &List(ref items) => {
                let out = items.iter().map(|i| i.print())
                    .fold(~"(", |m, v| m + v + " ");
                out.trim() + ")"
            },
            &Atom(ref v) => v.print()
        }
    }
    pub fn un_cons(self) -> (Expr, Option<Expr>) {
        match self {
            Atom(v) => (Atom(v), None),
            List(mut items) => {
                if items.len() == 0 {
                    fail!("cannot build car/cdr of empty list");
                }
                let car = items.shift()
                    .expect("at least one item in the least; shouldn't happen");
                (*car, Some(List(items)))
            }
        }
    }
}

#[deriving(Eq, Show, Clone)]
pub enum AtomType {
    Symbol(~str),
    Integer(i64),
    Float(f64)
}

impl AtomType {
    pub fn print(&self) -> ~str {
        match self {
            &Symbol(ref v) => v.to_owned(),
            &Integer(ref v) => v.to_str(),
            &Float(ref v) => v.to_str()
        }
    }
}

fn parse(tokens: &mut Vec<~str>) -> Expr {
    let current_token =
        tokens.shift().expect("calling parse() w/ empty token list; shouldn't happen");
    match current_token {
        ref x if *x == ~"(" => {
            let mut list = Vec::new();
            while *tokens.get(0) != ~")" {
                list.push(box parse(tokens));
            }
            tokens.shift();
            List(list)
        },
        ref x if *x == ~")" => fail!("hit ) token; shouldn't happen"),
        x => Expr::new_atom(x)
    }
}

#[cfg(test)]
mod parser_test {
    use super::{pad_input, tokenize, parse, Atom, List, Symbol, Integer, Float};

    #[test]
    fn pad_input_should_insert_spaces_before_and_after_parens() {
        assert_eq!(pad_input(~"(x 1 2 3)"), ~" ( x 1 2 3 ) ");
    }

    #[test]
    fn tokenize_string_into_tokens() {
        let tokens = tokenize(~"(foo 12 3)");
        assert_eq!(tokens.len(), 5);
        assert_eq!(tokens.get(0).to_owned(), ~"(");
        assert_eq!(tokens.get(1).to_owned(), ~"foo");
        assert_eq!(tokens.get(2).to_owned(), ~"12");
        assert_eq!(tokens.get(3).to_owned(), ~"3");
        assert_eq!(tokens.get(4).to_owned(), ~")");
    }

    #[test]
    fn should_parse_tokens_consisting_of_a_str_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"bar");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Symbol(val)) => {
                assert_eq!(val, ~"bar")
            },
            _ => assert!(false)
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_an_int_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"42");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Integer(val)) => {
                assert_eq!(val, 42)
            },
            _ => assert!(false)
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_a_float_atom_and_convert_it_to_a_parse_item() {
        let tokens = &mut tokenize(~"1.1");
        let parsed_item = parse(tokens);
        println!("{:?}", parsed_item);
        match parsed_item {
            Atom(Float(val)) => {
                assert_eq!(val, 1.1)
            },
            _ => assert!(false)
        }
    }

    #[test]
    fn should_parse_tokens_consisting_of_a_list_of_atoms_and_parse_it() {
        let tokens = &mut tokenize(~"(bar 12 45)");
        let parsed_item = parse(tokens);
        assert_eq!(parsed_item.is_atom(), false);
        match parsed_item {
            List(items) => {
                let items_len = items.len();
                assert!(3 == items_len);
                assert_eq!(items.get(0).is_atom(), true);
                assert_eq!(*items.get(0), ~Atom(Symbol(~"bar")));
                assert_eq!(items.get(1).is_atom(), true);
                assert_eq!(*items.get(1), ~Atom(Integer(12)));
                assert_eq!(items.get(2).is_atom(), true);
                assert_eq!(*items.get(2), ~Atom(Integer(45)));
            },
            _ => fail!("got back an atom, it seems")
        }
    }
    #[test]
    fn should_parse_tokens_consisting_of_a_list_of_atoms_and_a_nested_list() {
        let tokens = &mut tokenize(~"((2 3) bar 12 (hee (hah)))");
        let parsed_item = parse(tokens);
        assert_eq!(parsed_item.is_atom(), false);
        match parsed_item {
            List(items) => {
                let items_len = items.len();
                assert!(4 == items_len);
                match *items.get(0) {
                    ~List(ref items) => {
                        assert_eq!(items.len(), 2);
                        assert_eq!(*items.get(0), ~Atom(Integer(2)));
                        assert_eq!(*items.get(1), ~Atom(Integer(3)));
                    },
                    _ => fail!("shoulda got a list")
                }
                assert_eq!(*items.get(1), ~Atom(Symbol(~"bar")));
                assert_eq!(*items.get(2), ~Atom(Integer(12)));
                match *items.get(3) {
                    ~List(ref items) => {
                        assert_eq!(items.len(), 2);
                        assert_eq!(*items.get(0), ~Atom(Symbol(~"hee")));
                        match *items.get(1) {
                            ~List(ref items) => {
                                assert_eq!(items.len(), 1);
                                assert_eq!(*items.get(0), ~Atom(Symbol(~"hah")));
                            },
                            _ => fail!("shoulda got a list")
                        }
                    },
                    _ => fail!("shoulda got a list")
                }
            },
            _ => fail!("got back an atom, it seems")
        }
    }
    #[test]
    fn parsed_tokens_of_a_list_of_atoms_and_a_nested_list_should_to_str_correctly() {
        let input = ~"((2 3) bar 12 (hee (hah)))";
        let tokens = &mut tokenize(input.to_owned());
        let parsed_item = parse(tokens);
        let output = parsed_item.print();
        assert_eq!(input, output);
    }
}

#[cfg(test)]
mod eval_test {
    mod un_cons {
        use super::super::{parse_str, Atom, List, Symbol};
        #[test]
        fn an_atom_expr_returns_the_atom_in_the_car_with_none_in_the_cdr() {
            let expr = parse_str(~"x");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"x")));
            assert_eq!(cdr, None);
        }

        #[test]
        fn a_list_with_one_atom_element_returns_an_atom_car_and_an_empty_list_in_the_cdr() {
            let expr = parse_str(~"(x)");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"x")));
            match cdr.expect("should be a value here") {
                List(items) => assert_eq!(items.len(), 0),
                _ => fail!("expected a list")
            }
        }

        #[test]
        fn a_list_with_multiple_elements_returns_the_first_car_and_the_rust_in_the_cdr() {
            let expr = parse_str(~"(y 2 3)");
            let (car, cdr) = expr.un_cons();
            assert_eq!(car, Atom(Symbol(~"y")));
            match cdr.expect("should be a value here") {
                List(items) => {
                    assert_eq!(items.len(), 2)
                },
                _ => fail!("expected a list")
            }
        }

        #[test]
        #[should_fail]
        fn calling_extract_on_an_empty_list_should_fail() {
            let expr = parse_str(~"()");
            expr.un_cons();
        }
    }
}
<|endoftext|>


<commit_before>use std::any::Any;
use std::collections::{HashMap, VecDeque};

use lexer::{Item, ItemType, Lexer};
use node::*;

pub type Func<'a> = &'a Fn(Option<Box<Any>>) -> Option<Box<Any>>;

pub struct Tree<'a> {
    name: String,
    parse_name: String,
    root: Option<ListNode>,
    text: String,
    funcs: HashMap<String, Func<'a>>,
    lex: Option<Lexer>,
    token: VecDeque<Item>,
    peek_count: usize,
    vars: Vec<String>,
    tree_ids: HashMap<String, TreeId>,
    tree_set: HashMap<TreeId, Tree<'a>>,
}

impl<'a, 'b> Tree<'a> {
    fn new(name: String, funcs: HashMap<String, Func<'a>>) -> Tree<'a> {
        Tree {
            name,
            parse_name: String::default(),
            root: None,
            text: String::default(),
            funcs,
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }
    fn clone_new(t: &Tree) -> Tree<'b> {
        Tree {
            name: t.name.clone(),
            parse_name: t.parse_name.clone(),
            root: t.root.clone(),
            text: t.text.clone(),
            funcs: HashMap::new(),
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }
}

pub fn parse<'a>(name: String,
                 text: String,
                 funcs: HashMap<String, Func<'a>>)
                 -> HashMap<String, Tree<'a>> {
    let tree_ids = HashMap::new();
    tree_ids
}

impl<'a> Tree<'a> {
    fn next_from_lex(&mut self) -> Option<Item> {
        match &mut self.lex {
            &mut Some(ref mut l) => l.next(),
            &mut None => None,
        }
    }

    fn backup(&mut self, t: Item) {
        self.token.push_back(t);
        self.peek_count = 1;
    }

    fn backup2(&mut self, t0: Item, t1: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.peek_count = 2;
    }

    fn backup3(&mut self, t0: Item, t1: Item, t2: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.token.push_back(t2);
        self.peek_count = 3;
    }

    fn next_non_space(&mut self) -> Option<Item> {
        self.skip_while(|c| c.typ == ItemType::ItemSpace).next()
    }

    fn peek_non_space(&mut self) -> Option<&Item> {
        if let Some(t) = self.next_non_space() {
            self.backup(t);
            return self.token.front();
        }
        None
    }

    fn error_context(&mut self, n: Nodes) -> (String, String) {
        let pos = n.pos();
        let tree_id = n.tree();
        let tree = if tree_id == 0 && self.tree_set.contains_key(&tree_id) {
            self.tree_set.get(&tree_id).unwrap()
        } else {
            self
        };
        let text = &tree.text[0..pos];
        let byte_num = match text.rfind('\n') {
            Some(i) => pos - (i + 1),
            None => pos,
        };
        let line_num = text.chars().filter(|c| *c == '\n').count();
        let context = n.to_string();
        (format!("{}:{}:{}", tree.parse_name, line_num, byte_num), context)
    }
}

impl<'a> Iterator for Tree<'a> {
    type Item = Item;
    fn next(&mut self) -> Option<Item> {
        if self.peek_count > 0 {
            self.peek_count -= 1;
            self.token.pop_front()
        } else {
            self.next_from_lex()
        }
    }
}

#[cfg(test)]
mod tests_mocked {
    use super::*;
    use lexer::ItemType;

    /*
       ItemText
       ItemLeftDelim
       ItemSpace
       ItemIf
       ItemSpace
       ItemIdentifier
       ItemSpace
       ItemString
       ItemSpace
       ItemString
       ItemSpace
       ItemRightDelim
       ItemEOF
    */
    fn make_tree<'a>() -> Tree<'a> {
        let s = r#"something {{ if eq "foo" "bar" }}"#;
        let lex = Lexer::new("foo", s.to_owned());
        Tree {
            name: "foo".to_owned(),
            parse_name: "bar".to_owned(),
            root: None,
            text: "nope".to_owned(),
            funcs: HashMap::new(),
            lex: Some(lex),
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }

    #[test]
    fn test_iter() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        assert_eq!(t.collect::<Vec<_>>().len(), 12);
    }

    #[test]
    fn test_backup() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        let i = t.next().unwrap();
        let s = i.to_string();
        t.backup(i);
        assert_eq!(t.next().and_then(|n| Some(n.to_string())), Some(s));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }
    #[test]
    fn test_backup3() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        let t0 = t.next().unwrap();
        let t1 = t.next().unwrap();
        let t2 = t.next().unwrap();
        assert_eq!(t0.typ, ItemType::ItemLeftDelim);
        assert_eq!(t1.typ, ItemType::ItemSpace);
        assert_eq!(t2.typ, ItemType::ItemIf);
        t.backup3(t0, t1, t2);
        let t0 = t.next().unwrap();
        let t1 = t.next().unwrap();
        let t2 = t.next().unwrap();
        assert_eq!(t0.typ, ItemType::ItemLeftDelim);
        assert_eq!(t1.typ, ItemType::ItemSpace);
        assert_eq!(t2.typ, ItemType::ItemIf);
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }

    #[test]
    fn test_next_non_space() {
        let mut t = make_tree();
        t.next();
        let i = t.next().unwrap();
        let typ = i.typ;
        assert_eq!(typ, ItemType::ItemLeftDelim);
        assert_eq!(t.next_non_space().and_then(|n| Some(n.typ)),
                   Some(ItemType::ItemIf));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }

    #[test]
    fn test_peek_non_space() {
        let mut t = make_tree();
        t.next();
        let i = t.next().unwrap();
        let typ = i.typ;
        assert_eq!(typ, ItemType::ItemLeftDelim);
        assert_eq!(t.peek_non_space().and_then(|n| Some(&n.typ)),
                   Some(&ItemType::ItemIf));
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemIf));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }
}
<commit_msg>more parse<commit_after>use std::any::Any;
use std::collections::{HashMap, VecDeque};

use lexer::{Item, ItemType, Lexer};
use node::*;

pub type Func<'a> = &'a Fn(Option<Box<Any>>) -> Option<Box<Any>>;

pub struct Tree<'a> {
    name: String,
    parse_name: String,
    root: Option<ListNode>,
    text: String,
    funcs: HashMap<String, Func<'a>>,
    lex: Option<Lexer>,
    token: VecDeque<Item>,
    peek_count: usize,
    vars: Vec<String>,
    tree_ids: HashMap<String, TreeId>,
    tree_set: HashMap<TreeId, Tree<'a>>,
}

impl<'a, 'b> Tree<'a> {
    fn new(name: String, funcs: HashMap<String, Func<'a>>) -> Tree<'a> {
        Tree {
            name,
            parse_name: String::default(),
            root: None,
            text: String::default(),
            funcs,
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }
    fn clone_new(t: &Tree) -> Tree<'b> {
        Tree {
            name: t.name.clone(),
            parse_name: t.parse_name.clone(),
            root: t.root.clone(),
            text: t.text.clone(),
            funcs: HashMap::new(),
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }
}

pub fn parse<'a>(name: String,
                 text: String,
                 funcs: HashMap<String, Func<'a>>)
                 -> HashMap<String, Tree<'a>> {
    let tree_ids = HashMap::new();
    tree_ids
}

impl<'a> Tree<'a> {
    fn next_from_lex(&mut self) -> Option<Item> {
        match &mut self.lex {
            &mut Some(ref mut l) => l.next(),
            &mut None => None,
        }
    }

    fn backup(&mut self, t: Item) {
        self.token.push_back(t);
        self.peek_count = 1;
    }

    fn backup2(&mut self, t0: Item, t1: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.peek_count = 2;
    }

    fn backup3(&mut self, t0: Item, t1: Item, t2: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.token.push_back(t2);
        self.peek_count = 3;
    }

    fn next_non_space(&mut self) -> Option<Item> {
        self.skip_while(|c| c.typ == ItemType::ItemSpace).next()
    }

    fn peek_non_space(&mut self) -> Option<&Item> {
        if let Some(t) = self.next_non_space() {
            self.backup(t);
            return self.token.front();
        }
        None
    }

    fn error_context(&mut self, n: Nodes) -> (String, String) {
        let pos = n.pos();
        let tree_id = n.tree();
        let tree = if tree_id == 0 && self.tree_set.contains_key(&tree_id) {
            self.tree_set.get(&tree_id).unwrap()
        } else {
            self
        };
        let text = &tree.text[0..pos];
        let byte_num = match text.rfind('\n') {
            Some(i) => pos - (i + 1),
            None => pos,
        };
        let line_num = text.chars().filter(|c| *c == '\n').count();
        let context = n.to_string();
        (format!("{}:{}:{}", tree.parse_name, line_num, byte_num), context)
    }

    fn start_parse(&mut self,
                   funcs: HashMap<String, Func<'a>>,
                   lex: Lexer,
                   tree_ids: HashMap<String, TreeId>,
                   tree_set: HashMap<TreeId, Tree<'a>>) {
        self.root = None;
        self.lex = Some(lex);
        self.vars = vec!["$".to_owned()];
        self.funcs = funcs;
        self.tree_ids = tree_ids;
        self.tree_set = tree_set;
    }

    fn stop_parse(&mut self) {
        self.lex = None;
        self.vars = vec![];
        self.funcs = HashMap::new();
        self.tree_ids = HashMap::new();
        self.tree_set = HashMap::new();
    }

    fn parse(&mut self,
             text: String,
             tree_ids: HashMap<String, TreeId>,
             tree_set: HashMap<TreeId, Tree<'a>>,
             funcs: HashMap<String, Func<'a>>)
             -> Result<(), String> {
        self.parse_name = self.name.clone();
        let lex_name = self.name.clone();
        self.start_parse(funcs, Lexer::new(&lex_name, text.clone()), tree_ids, tree_set);
        self.text = text;
        //self.do_parse();
        self.stop_parse();
        Ok(())
    }
}

impl<'a> Iterator for Tree<'a> {
    type Item = Item;
    fn next(&mut self) -> Option<Item> {
        if self.peek_count > 0 {
            self.peek_count -= 1;
            self.token.pop_front()
        } else {
            self.next_from_lex()
        }
    }
}

#[cfg(test)]
mod tests_mocked {
    use super::*;
    use lexer::ItemType;

    /*
       ItemText
       ItemLeftDelim
       ItemSpace
       ItemIf
       ItemSpace
       ItemIdentifier
       ItemSpace
       ItemString
       ItemSpace
       ItemString
       ItemSpace
       ItemRightDelim
       ItemEOF
    */
    fn make_tree<'a>() -> Tree<'a> {
        let s = r#"something {{ if eq "foo" "bar" }}"#;
        let lex = Lexer::new("foo", s.to_owned());
        Tree {
            name: "foo".to_owned(),
            parse_name: "bar".to_owned(),
            root: None,
            text: "nope".to_owned(),
            funcs: HashMap::new(),
            lex: Some(lex),
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
        }
    }

    #[test]
    fn test_iter() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        assert_eq!(t.collect::<Vec<_>>().len(), 12);
    }

    #[test]
    fn test_backup() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        let i = t.next().unwrap();
        let s = i.to_string();
        t.backup(i);
        assert_eq!(t.next().and_then(|n| Some(n.to_string())), Some(s));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }
    #[test]
    fn test_backup3() {
        let mut t = make_tree();
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemText));
        let t0 = t.next().unwrap();
        let t1 = t.next().unwrap();
        let t2 = t.next().unwrap();
        assert_eq!(t0.typ, ItemType::ItemLeftDelim);
        assert_eq!(t1.typ, ItemType::ItemSpace);
        assert_eq!(t2.typ, ItemType::ItemIf);
        t.backup3(t0, t1, t2);
        let t0 = t.next().unwrap();
        let t1 = t.next().unwrap();
        let t2 = t.next().unwrap();
        assert_eq!(t0.typ, ItemType::ItemLeftDelim);
        assert_eq!(t1.typ, ItemType::ItemSpace);
        assert_eq!(t2.typ, ItemType::ItemIf);
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }

    #[test]
    fn test_next_non_space() {
        let mut t = make_tree();
        t.next();
        let i = t.next().unwrap();
        let typ = i.typ;
        assert_eq!(typ, ItemType::ItemLeftDelim);
        assert_eq!(t.next_non_space().and_then(|n| Some(n.typ)),
                   Some(ItemType::ItemIf));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }

    #[test]
    fn test_peek_non_space() {
        let mut t = make_tree();
        t.next();
        let i = t.next().unwrap();
        let typ = i.typ;
        assert_eq!(typ, ItemType::ItemLeftDelim);
        assert_eq!(t.peek_non_space().and_then(|n| Some(&n.typ)),
                   Some(&ItemType::ItemIf));
        assert_eq!(t.next().and_then(|n| Some(n.typ)), Some(ItemType::ItemIf));
        assert_eq!(t.last().and_then(|n| Some(n.typ)), Some(ItemType::ItemEOF));
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}

type FileID = u32;

#[derive(Debug, Clone)]
pub enum NodeValue {
    Owned(String),
    Children(Vec<Node>),
}

#[derive(Debug, Clone)]
pub struct Node {
    pub value: NodeValue,
    pub file_id: FileID,
    pub lineno: i32,
}

impl Node {
    pub fn new(value: NodeValue, file_id: FileID, lineno: i32) -> Self {
        Node {
            value,
            file_id,
            lineno,
        }
    }

    pub fn new_children(value: Vec<Node>, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Children(value),
            file_id,
            lineno,
        }
    }

    pub fn new_string<S: Into<String>>(value: S, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Owned(value.into()),
            file_id,
            lineno,
        }
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self.value {
            NodeValue::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            NodeValue::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
    file_id: FileID,
    lineno: i32,
}

impl StateRocket {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateRocket {
            root: vec![Node::new_string("concat", file_id, lineno)],
            buffer: vec![],
            file_id: file_id,
            lineno,
        }
    }

    fn ensure_string(&mut self) -> &mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match *token {
            Token::Text(_, s) => {
                self.buffer.push(s.to_owned());
            }
            Token::Quote(_) => {
                self.ensure_string().push('"');
            }
            Token::StartBlock(lineno) => {
                if !self.buffer.is_empty() {
                    self.root
                        .push(Node::new_string(self.buffer.concat(), self.file_id, lineno));
                    self.buffer.clear();
                }
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::RightParen => {
                self.ensure_string().push(')');
            }
            Token::Rocket(_) => {
                self.ensure_string().push_str("=>");
            }
            Token::Dedent => {
                // We need to pop both the rocket and the expression that started the rocket
                return StackRequest::Pop(2);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        if !self.buffer.is_empty() {
            let string = self.buffer.concat();
            self.root.push(Node::new_string(string, self.file_id, -1));
        }

        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "rocket"
    }
}

struct StateExpression {
    root: Vec<Node>,
    file_id: FileID,
    lineno: i32,

    quote: Vec<String>,
    in_quote: bool,
}

impl StateExpression {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateExpression {
            root: vec![],
            file_id,
            lineno,
            quote: vec![],
            in_quote: false,
        }
    }
}

impl TokenHandler for StateExpression {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        if self.in_quote {
            match *token {
                Token::Text(_, s) => self.quote.push(s.to_owned()),
                Token::Quote(lineno) => {
                    self.root
                        .push(Node::new_string(self.quote.concat(), self.file_id, lineno));

                    self.in_quote = false;
                    self.quote.clear();
                }
                Token::StartBlock(_) => self.quote.push("(:".to_owned()),
                Token::RightParen => self.quote.push(")".to_owned()),
                Token::Rocket(_) => self.quote.push("=>".to_owned()),
                Token::Dedent => (),
            }
            return StackRequest::None;
        }

        match *token {
            Token::Text(lineno, s) => {
                // When in an expression, whitespace only serves to separate tokens.
                if !PAT_IS_WHITESPACE.is_match(s) {
                    self.root
                        .push(Node::new_string(s.to_owned(), self.file_id, lineno));
                }
            }
            Token::Quote(_) => {
                self.in_quote = true;
            }
            Token::StartBlock(lineno) => {
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::Rocket(lineno) => {
                return StackRequest::Push(Box::new(StateRocket::new(self.file_id, lineno)));
            }
            Token::RightParen | Token::Dedent => {
                return StackRequest::Pop(1);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "expression"
    }
}

struct ParseContextStack {
    stack: Vec<Box<TokenHandler>>,
}

impl ParseContextStack {
    fn new(file_id: FileID, lineno: i32) -> Self {
        ParseContextStack {
            stack: vec![
                Box::new(StateRocket {
                    root: vec![Node::new_string("concat", file_id, lineno)],
                    buffer: vec![],
                    file_id: file_id,
                    lineno: lineno,
                }),
            ],
        }
    }

    fn handle(&mut self, token: &Token) {
        match self.stack
            .last_mut()
            .expect("Empty parse stack")
            .handle_token(token)
        {
            StackRequest::Push(handler) => {
                self.stack.push(handler);
            }
            StackRequest::Pop(n) => for _ in 0..n {
                let mut handler = self.stack.pop().expect("Cannot pop last handler");
                (**self.stack.last_mut().expect("Empty parse stack")).push(handler.finish());
            },
            StackRequest::None => (),
        }
    }
}

pub struct Parser {
    file_ids: Vec<PathBuf>,
}

impl Parser {
    pub fn new() -> Self {
        Parser { file_ids: vec![] }
    }

    pub fn get_node_source_path(&self, node: &Node) -> Option<&Path> {
        match self.file_ids.get(node.file_id as usize) {
            Some(p) => Some(p),
            None => None,
        }
    }

    pub fn parse(&mut self, path: &Path) -> Result<Node, String> {
        debug!("Parsing {}", path.to_string_lossy());

        let id = self.file_ids.len() as FileID;
        self.file_ids.push(path.to_owned());

        let mut file = match File::open(path) {
            Ok(f) => f,
            Err(_) => {
                return Err(format!("Failed to open {}", path.to_string_lossy()));
            }
        };
        let mut data = String::new();
        file.read_to_string(&mut data)
            .expect("Failed to read input file");

        let mut stack = ParseContextStack::new(id, 0);
        for token in lex(&data) {
            stack.handle(&token);
        }

        let root = stack.stack.pop().expect("Empty state stack").finish();
        match stack.stack.pop() {
            Some(_) => Err(format!(
                "Unterminated block started on line {}",
                root.lineno
            )),
            None => Ok(root),
        }
    }
}
<commit_msg>parse: Merge adjacent non-whitespace text tokens in expressions<commit_after>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}

type FileID = u32;

#[derive(Debug, Clone)]
pub enum NodeValue {
    Owned(String),
    Children(Vec<Node>),
}

#[derive(Debug, Clone)]
pub struct Node {
    pub value: NodeValue,
    pub file_id: FileID,
    pub lineno: i32,
}

impl Node {
    pub fn new(value: NodeValue, file_id: FileID, lineno: i32) -> Self {
        Node {
            value,
            file_id,
            lineno,
        }
    }

    pub fn new_children(value: Vec<Node>, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Children(value),
            file_id,
            lineno,
        }
    }

    pub fn new_string<S: Into<String>>(value: S, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Owned(value.into()),
            file_id,
            lineno,
        }
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self.value {
            NodeValue::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            NodeValue::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
    file_id: FileID,
    lineno: i32,
}

impl StateRocket {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateRocket {
            root: vec![Node::new_string("concat", file_id, lineno)],
            buffer: vec![],
            file_id: file_id,
            lineno,
        }
    }

    fn ensure_string(&mut self) -> &mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match *token {
            Token::Text(_, s) => {
                self.buffer.push(s.to_owned());
            }
            Token::Quote(_) => {
                self.ensure_string().push('"');
            }
            Token::StartBlock(lineno) => {
                if !self.buffer.is_empty() {
                    self.root
                        .push(Node::new_string(self.buffer.concat(), self.file_id, lineno));
                    self.buffer.clear();
                }
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::RightParen => {
                self.ensure_string().push(')');
            }
            Token::Rocket(_) => {
                self.ensure_string().push_str("=>");
            }
            Token::Dedent => {
                // We need to pop both the rocket and the expression that started the rocket
                return StackRequest::Pop(2);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        if !self.buffer.is_empty() {
            let string = self.buffer.concat();
            self.root.push(Node::new_string(string, self.file_id, -1));
        }

        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "rocket"
    }
}

struct StateExpression {
    root: Vec<Node>,
    file_id: FileID,
    lineno: i32,

    quote: Vec<String>,
    in_quote: bool,
    new_node: bool,
}

impl StateExpression {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateExpression {
            root: vec![],
            file_id,
            lineno,
            quote: vec![],
            in_quote: false,
            new_node: true,
        }
    }
}

impl TokenHandler for StateExpression {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        if self.in_quote {
            match *token {
                Token::Text(_, s) => self.quote.push(s.to_owned()),
                Token::Quote(lineno) => {
                    self.root
                        .push(Node::new_string(self.quote.concat(), self.file_id, lineno));

                    self.in_quote = false;
                    self.quote.clear();
                }
                Token::StartBlock(_) => self.quote.push("(:".to_owned()),
                Token::RightParen => self.quote.push(")".to_owned()),
                Token::Rocket(_) => self.quote.push("=>".to_owned()),
                Token::Dedent => (),
            }
            return StackRequest::None;
        }

        match *token {
            Token::Text(lineno, s) => {
                // When in an expression, whitespace only serves to separate tokens.
                if PAT_IS_WHITESPACE.is_match(s) {
                    self.new_node = true;
                } else {
                    let mut new_node = self.new_node;

                    if !new_node {
                        if let Some(last) = self.root.last_mut() {
                            match last.value {
                                NodeValue::Owned(ref mut val) => val.push_str(s),
                                NodeValue::Children(_) => new_node = true,
                            }
                        } else {
                            new_node = true;
                        }
                    }

                    if new_node {
                        self.root
                            .push(Node::new_string(s.to_owned(), self.file_id, lineno));
                    }
                    self.new_node = false;
                }
            }
            Token::Quote(_) => {
                self.in_quote = true;
            }
            Token::StartBlock(lineno) => {
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::Rocket(lineno) => {
                return StackRequest::Push(Box::new(StateRocket::new(self.file_id, lineno)));
            }
            Token::RightParen | Token::Dedent => {
                return StackRequest::Pop(1);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "expression"
    }
}

struct ParseContextStack {
    stack: Vec<Box<TokenHandler>>,
}

impl ParseContextStack {
    fn new(file_id: FileID, lineno: i32) -> Self {
        ParseContextStack {
            stack: vec![
                Box::new(StateRocket {
                    root: vec![Node::new_string("concat", file_id, lineno)],
                    buffer: vec![],
                    file_id: file_id,
                    lineno: lineno,
                }),
            ],
        }
    }

    fn handle(&mut self, token: &Token) {
        match self.stack
            .last_mut()
            .expect("Empty parse stack")
            .handle_token(token)
        {
            StackRequest::Push(handler) => {
                self.stack.push(handler);
            }
            StackRequest::Pop(n) => for _ in 0..n {
                let mut handler = self.stack.pop().expect("Cannot pop last handler");
                (**self.stack.last_mut().expect("Empty parse stack")).push(handler.finish());
            },
            StackRequest::None => (),
        }
    }
}

pub struct Parser {
    file_ids: Vec<PathBuf>,
}

impl Parser {
    pub fn new() -> Self {
        Parser { file_ids: vec![] }
    }

    pub fn get_node_source_path(&self, node: &Node) -> Option<&Path> {
        match self.file_ids.get(node.file_id as usize) {
            Some(p) => Some(p),
            None => None,
        }
    }

    pub fn parse(&mut self, path: &Path) -> Result<Node, String> {
        debug!("Parsing {}", path.to_string_lossy());

        let id = self.file_ids.len() as FileID;
        self.file_ids.push(path.to_owned());

        let mut file = match File::open(path) {
            Ok(f) => f,
            Err(_) => {
                return Err(format!("Failed to open {}", path.to_string_lossy()));
            }
        };
        let mut data = String::new();
        file.read_to_string(&mut data)
            .expect("Failed to read input file");

        let mut stack = ParseContextStack::new(id, 0);
        for token in lex(&data) {
            stack.handle(&token);
        }

        let root = stack.stack.pop().expect("Empty state stack").finish();
        match stack.stack.pop() {
            Some(_) => Err(format!(
                "Unterminated block started on line {}",
                root.lineno
            )),
            None => Ok(root),
        }
    }
}
<|endoftext|>