<commit_before>//! Functions for parsing a molecule or a chemical reaction.
//! Mostly based on the HTML parser from the robinson browser engine:
//! https://github.com/mbrubeck/robinson
//!
//! The parser works with the following grammar:
//! R = (R)eaction
//! S = one (S)ide of a reaction
//! M = (M)olecule
//! P = Combination of a (P)eriodic element and maybe a coefficient
//! E = Periodic (E)lement
//! C = (C)oefficient
//!
//! R --> S -> S
//! S --> M + S
//!    |  M
//! M --> PM
//!    |  P(M)
//!    |  P
//! P --> EC
//!    |  E

use std::str::CharRange;
use token::Token;
use token::TokenKind::*;
use error::{CTResult, CTError};

pub struct Parser {
    pos: usize,
    input: String,
}

impl Parser {
    pub fn new(input: &str) -> Parser {
        Parser { pos: 0, input: String::from_str(input) }
    }

    pub fn parse_molecule(&mut self) -> CTResult<Vec<Token>> {
        let mut out = Vec::new();
        let mut per = try!(self.parse_periodic());
        out.extend(per.drain());

        if !self.eof() {
            if self.peek_char() == '(' {
                out.push(Token { tok: ParenOpen, pos: self.pos, len: 1 });
                self.consume_char();
                let mut molecule = try!(self.parse_molecule());
                out.extend(molecule.drain());

                if self.eof() || self.consume_char() != ')' {
                    return Err(CTError {
                        desc: "Missing closing parentheses".to_string(),
                        pos: self.pos - 1,
                        len: 1,
                    });
                } else {
                    out.push(Token { tok: ParenClose, pos: self.pos - 1, len: 1 });
                }
            } else if self.peek_char().is_alphabetic() {
                let mut molecule = try!(self.parse_molecule());
                out.extend(molecule.drain());
            }
        }

        Ok(out)
    }

    fn parse_periodic(&mut self) -> CTResult<Vec<Token>> {
        let mut out = Vec::new();
        let elem = try!(self.parse_element());
        out.push(elem);

        if !self.eof() && self.peek_char().is_numeric() {
            let coef = try!(self.parse_coefficient());
            out.push(coef);
        }

        Ok(out)
    }

    fn parse_element(&mut self) -> CTResult<Token> {
        if self.eof() {
            return Err(CTError {
                desc: "Found no periodic element".to_string(),
                pos: self.pos,
                len: 1,
            });
        }
        let first = self.consume_char();
        if first.is_uppercase() {
            let mut name = String::new();
            let start_pos = self.pos;
            name.push(first);
            name.push_str(self.consume_while(|ch| ch.is_lowercase()).as_slice());
            let len = name.len();
            Ok(Token{ tok: Elem(name), pos: start_pos, len: len })
        } else {
            println!("{:?}", first);
            Err(CTError {
                desc: "Missing uppercase letter at the beginning of the element".to_string(),
                pos: self.pos - 1,
                len: 1,
            })
        }
    }

    fn parse_coefficient(&mut self) -> CTResult<Token> {
        let start_pos = self.pos;
        let num_str = self.consume_while(|ch| ch.is_numeric());
        if let Some(num) = num_str.parse::<u32>() {
            Ok(Token { tok: Coefficient(num), pos: start_pos, len: num_str.len() })
        } else {
            Err(CTError {
                desc: "Could not parse coefficient".to_string(),
                pos: start_pos,
                len: num_str.len(),
            })
        }
    }

    fn peek_char(&self) -> char {
        self.input.char_at(self.pos)
    }

    fn consume_char(&mut self) -> char {
        let CharRange { ch, next } = self.input.char_range_at(self.pos);
        self.pos = next;
        ch
    }

    fn consume_while<F>(&mut self, pred: F) -> String where F: Fn(char) -> bool {
        let mut out = String::new();
        while !self.eof() && pred(self.peek_char()) {
            out.push(self.consume_char());
        }
        out
    }

    fn consume_whitespace(&mut self) {
        self.consume_while(|ch| ch.is_whitespace());
    }

    fn eof(&mut self) -> bool {
        self.pos >= self.input.len()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use error::CTResult;
    use token::Token;
    use token::TokenKind::*;

    fn check_raw_result(raw_result: CTResult<Vec<Token>>, expected: Vec<Token>) {
        if let Ok(result) = raw_result {
            assert_eq!(result, expected);
        } else {
            panic!(raw_result);
        }
    }

    #[test]
    fn elems() {
        let mut parser = Parser::new("CHeH");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: Elem("C".to_string()), pos: 0, len: 1 },
                            Token { tok: Elem("He".to_string()), pos: 1, len: 2 },
                            Token { tok: Elem("H".to_string()), pos: 3, len: 1 });
        check_raw_result(raw_result, expected);
    }

    #[test]
    fn coefs() {
        let mut parser = Parser::new("C23");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: Elem("C".to_string()), pos: 0, len: 1 },
                            Token { tok: Coefficient(23), pos: 1, len: 2 });
        check_raw_result(raw_result, expected);
    }

    #[test]
    fn parens() {
        let mut parser = Parser::new("(C)2");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: ParenOpen, pos: 0, len: 1 },
                            Token { tok: Elem("C".to_string()), pos: 1, len: 1},
                            Token { tok: ParenClose, pos: 2, len: 1},
                            Token { tok: Coefficient(2), pos: 3, len: 1 });
        check_raw_result(raw_result, expected);
    }
}<commit_msg>Rewrite parser to pass tests<commit_after>//! Functions for parsing a molecule or a chemical reaction.
//! Mostly based on the HTML parser from the robinson browser engine:
//! https://github.com/mbrubeck/robinson
//!
//! The parser works with the following grammar:
//! R = (R)eaction
//! S = one (S)ide of a reaction
//! M = (M)olecule
//! P = Combination of a (P)eriodic element and maybe a coefficient
//! E = Periodic (E)lement
//! C = (C)oefficient
//!
//! R --> S -> S
//! S --> M + S
//!    |  M
//! M --> PM
//!    |  P
//! P --> EC
//!    |  E
//! E --> <text>
//!    |  (M)
//! C --> <number>

use std::str::CharRange;
use token::Token;
use token::TokenKind::*;
use error::{CTResult, CTError};

pub struct Parser {
    pos: usize,
    input: String,
}

impl Parser {
    pub fn new(input: &str) -> Parser {
        Parser { pos: 0, input: String::from_str(input) }
    }

    pub fn parse_molecule(&mut self) -> CTResult<Vec<Token>> {
        let mut out = Vec::new();
        let mut per = try!(self.parse_periodic());
        out.extend(per.drain());

        // TODO: Make this cleaner
        if !self.eof() && (self.peek_char().is_alphabetic() || self.peek_char() == '(') {
            let mut molecule = try!(self.parse_molecule());
            out.extend(molecule.drain());
        }

        Ok(out)
    }

    fn parse_periodic(&mut self) -> CTResult<Vec<Token>> {
        let mut out = Vec::new();
        let mut elem = try!(self.parse_element());
        out.extend(elem.drain());

        if !self.eof() && self.peek_char().is_numeric() {
            let coef = try!(self.parse_coefficient());
            out.push(coef);
        }

        Ok(out)
    }

    fn parse_element(&mut self) -> CTResult<Vec<Token>> {
        if self.eof() {
            return Err(CTError {
                desc: "Found no periodic element".to_string(),
                pos: self.pos,
                len: 1,
            });
        }
        let start_pos = self.pos;
        let first = self.consume_char();
        if first == '(' {
            let mut out = Vec::new();
            out.push(Token { tok: ParenOpen, pos: start_pos, len: 1 });
            let mut molecule = try!(self.parse_molecule());
            out.extend(molecule.drain());

            if self.eof() || self.consume_char() != ')' {
                Err(CTError {
                    desc: "Missing closing parentheses".to_string(),
                    pos: self.pos - 1,
                    len: 1,
                })
            } else {
                out.push(Token { tok: ParenClose, pos: self.pos - 1, len: 1 });
                Ok(out)
            }
        } else if first.is_uppercase() {
            let mut name = String::new();
            name.push(first);
            name.push_str(self.consume_while(|ch| ch.is_lowercase()).as_slice());
            let len = name.len();
            Ok(vec!(Token{ tok: Elem(name), pos: start_pos, len: len }))
        } else {
            println!("{:?}", first);
            Err(CTError {
                desc: "Missing uppercase letter at the beginning of the element".to_string(),
                pos: self.pos - 1,
                len: 1,
            })
        }
    }

    fn parse_coefficient(&mut self) -> CTResult<Token> {
        let start_pos = self.pos;
        let num_str = self.consume_while(|ch| ch.is_numeric());
        if let Some(num) = num_str.parse::<u32>() {
            Ok(Token { tok: Coefficient(num), pos: start_pos, len: num_str.len() })
        } else {
            Err(CTError {
                desc: "Could not parse coefficient".to_string(),
                pos: start_pos,
                len: num_str.len(),
            })
        }
    }

    fn peek_char(&self) -> char {
        self.input.char_at(self.pos)
    }

    fn consume_char(&mut self) -> char {
        let CharRange { ch, next } = self.input.char_range_at(self.pos);
        self.pos = next;
        ch
    }

    fn consume_while<F>(&mut self, pred: F) -> String where F: Fn(char) -> bool {
        let mut out = String::new();
        while !self.eof() && pred(self.peek_char()) {
            out.push(self.consume_char());
        }
        out
    }

    fn consume_whitespace(&mut self) {
        self.consume_while(|ch| ch.is_whitespace());
    }

    fn eof(&mut self) -> bool {
        self.pos >= self.input.len()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use error::CTResult;
    use token::Token;
    use token::TokenKind::*;

    fn check_raw_result(raw_result: CTResult<Vec<Token>>, expected: Vec<Token>) {
        if let Ok(result) = raw_result {
            assert_eq!(result, expected);
        } else {
            panic!(raw_result);
        }
    }

    #[test]
    fn elems() {
        let mut parser = Parser::new("CHeH");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: Elem("C".to_string()), pos: 0, len: 1 },
                            Token { tok: Elem("He".to_string()), pos: 1, len: 2 },
                            Token { tok: Elem("H".to_string()), pos: 3, len: 1 });
        check_raw_result(raw_result, expected);
    }

    #[test]
    fn coefs() {
        let mut parser = Parser::new("C23");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: Elem("C".to_string()), pos: 0, len: 1 },
                            Token { tok: Coefficient(23), pos: 1, len: 2 });
        check_raw_result(raw_result, expected);
    }

    #[test]
    fn parens() {
        let mut parser = Parser::new("(C)2");
        let raw_result = parser.parse_molecule();
        let expected = vec!(Token { tok: ParenOpen, pos: 0, len: 1 },
                            Token { tok: Elem("C".to_string()), pos: 1, len: 1},
                            Token { tok: ParenClose, pos: 2, len: 1},
                            Token { tok: Coefficient(2), pos: 3, len: 1 });
        check_raw_result(raw_result, expected);
    }
}<|endoftext|>


<commit_before>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Divide,
    Word(String),
    Invalid(char),
    Number(i32),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let string;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Divide => "\\",
            Word(ref w) => &*w,
            Number(i) => {
                string = format!("{}", i);
                &*string
            }
            Invalid(w) => {
                string = w.to_string();
                &*string
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let atom = try!(atom(tokens, model));
    match tokens.peek() {
        Some(&Plus) | Some(&Minus) => rhs(atom, tokens, model),
        None => Ok(atom),
        _ => Err(format!("Invalid token '{:?}'", tokens.peek().unwrap()))
    }
    // let token = try!(tokens.next().ok_or("An expression may not be empty"));
    // match token {
    //     LParen => {
    //         let vec = try!(expression(tokens, model));
    //         let rparen = try!(tokens.next().ok_or("Unbalanced parentheses"));
    //         if rparen != RParen {
    //             return Err(format!("Expected ')', found '{}'", rparen))
    //         }
    //         rhs(vec, tokens, model)
    //     },
    //     Word(word) => {
    //         let vec = try!(get_word(model, &*word));
    //         rhs(Ref(vec), tokens, model)
    //     },
    //     _ => Err(format!("An expression may not start with '{}'", token)),
    // }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        Divide | Number(_) => Err("Not sure how we got here!".to_string()),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    {
        let token = try!(tokens.peek().ok_or("Invalid end of expression"));
        match *token {
            Plus | Minus | Invalid(_) | RParen => {
                return Err(format!("Invalid token {}", token));
            }
            _ => {},
        }
    }

    let token = tokens.next().unwrap();
    let atom = match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => expr,
                Some(t) => return Err(format!("Expected ')', found '{}'", t)),
                None => return Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ref(vec)
        }
        _ => unreachable!(),
    };

    if tokens.peek() != Some(&Divide) {
        return Ok(atom);
    }

    tokens.next();

    let num = try!(tokens.next().ok_or(
        "An expression may not end with a division operator"));

    match num {
        Number(num) => Ok(Val(atom.take() / num)),
        _ => Err(format!("'{}' is not a valid numerator: must be an integer", num)),
    }

}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            Some('0'...'9') => return Some(self.number()),
            // Some('0'..'9') | Some('.') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

    fn number(&mut self) -> Token {
        let mut token = String::new();
        while let Some('0'...'9') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Number(token.parse().unwrap())
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }
}<commit_msg>Test for division expressions<commit_after>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Divide,
    Word(String),
    Invalid(char),
    Number(i32),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let string;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Divide => "/",
            Word(ref w) => &*w,
            Number(i) => {
                string = format!("{}", i);
                &*string
            }
            Invalid(w) => {
                string = w.to_string();
                &*string
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let atom = try!(atom(tokens, model));
    match tokens.peek() {
        Some(&Plus) | Some(&Minus) => rhs(atom, tokens, model),
        None => Ok(atom),
        _ => Err(format!("Invalid token '{:?}'", tokens.peek().unwrap()))
    }
    // let token = try!(tokens.next().ok_or("An expression may not be empty"));
    // match token {
    //     LParen => {
    //         let vec = try!(expression(tokens, model));
    //         let rparen = try!(tokens.next().ok_or("Unbalanced parentheses"));
    //         if rparen != RParen {
    //             return Err(format!("Expected ')', found '{}'", rparen))
    //         }
    //         rhs(vec, tokens, model)
    //     },
    //     Word(word) => {
    //         let vec = try!(get_word(model, &*word));
    //         rhs(Ref(vec), tokens, model)
    //     },
    //     _ => Err(format!("An expression may not start with '{}'", token)),
    // }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        Divide | Number(_) => Err("Not sure how we got here!".to_string()),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    {
        let token = try!(tokens.peek().ok_or("Invalid end of expression"));
        match *token {
            Plus | Minus | Invalid(_) | RParen => {
                return Err(format!("Invalid token {}", token));
            }
            _ => {},
        }
    }

    let token = tokens.next().unwrap();
    let atom = match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => expr,
                Some(t) => return Err(format!("Expected ')', found '{}'", t)),
                None => return Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ref(vec)
        }
        _ => unreachable!(),
    };

    if tokens.peek() != Some(&Divide) {
        return Ok(atom);
    }

    tokens.next();

    let num = try!(tokens.next().ok_or(
        "An expression may not end with a division operator"));

    match num {
        Number(num) => Ok(Val(atom.take() / num)),
        _ => Err(format!("'{}' is not a valid numerator: must be an integer", num)),
    }

}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            Some('0'...'9') => return Some(self.number()),
            // Some('0'..'9') | Some('.') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            '/' => Divide,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

    fn number(&mut self) -> Token {
        let mut token = String::new();
        while let Some('0'...'9') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Number(token.parse().unwrap())
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }

    #[test]
    fn test_div_expression() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        assert_eq!(a, parse("(a + a) / 2", &model).unwrap());
    }

}<|endoftext|>


<commit_before>/// For some reason, Pest cannot find the grammar file if listed in parser/mod.rs, so I listed it here
use pest::Parser;
use pest::iterators::Pair;
use pest::inputs::StrInput;

use ast::Expression;

use std::f64;

const _GRAMMAR: &'static str = include_str!("grammar/grammar.pest");

#[derive(Parser)]
#[grammar = "grammar/grammar.pest"]
pub struct ExpressionParser;

fn parse_expression(input: Pair<Rule, StrInput>) -> Expression {
    match input.as_rule() {
        Rule::function_literal => {
            let mut pairs = input.into_inner().flatten();

            println!("{:?}", pairs);

            Expression::FuncLiteral(
//                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
//               Box::new(parse_expression(pairs.nth(1).unwrap().into_span())),
//                Box::new(parse_expression(pairs.nth(2).unwrap().into_span()))
                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
                Box::new(parse_expression(pairs.nth(1).unwrap())),
                Box::new(parse_expression(pairs.nth(2).unwrap()))
            )
        }

        Rule::function_call_literal => {
            let mut pairs = input.into_inner();

            println!("{:?}", pairs);

            Expression::FuncCall(
                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
                Some(Box::new(parse_expression(pairs.nth(2).unwrap())))
            )
        }

        Rule::boolean_literal =>
            Expression::Boolean(
                input.into_span().as_str().parse::<bool>().unwrap()
            ),

        Rule::string_literal =>
            Expression::String(format!("{}", input.into_span().as_str())
            ),

        Rule::identifier_literal =>
            Expression::Identifier(input.into_span().as_str().to_string()),

        // TODO: For now, just conver the value into a normal f64. No formatting!
        Rule::decimal_digits_literal =>
            Expression::Number(
                input.into_span().as_str().parse::<f64>().unwrap()
            ),

        Rule::binary_digits_literal =>
            // The parse function does not support binary digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 2).unwrap())
            ),

        Rule::oct_digits_literal =>
            // The parse function does not support octal digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 8).unwrap())
            ),

        Rule::hex_digits_literal =>
            // The parse function does not support hexadecimal digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 16).unwrap())
            ),

        Rule::list_literal =>
            Expression::List(input
                            .into_inner()
                            .into_iter()
                            .map(|r| Box::new(parse_expression(r)))
                            .collect()),

        // Temporary
        _ => unreachable!()
    }

}

pub fn parse(input: String) -> Result<Expression, String> {
    // TODO: Remove unwrap
    match ExpressionParser::parse_str(Rule::main, &input) {
        Ok(mut pair) =>
            Ok(parse_expression(pair.nth(0).unwrap().into_inner().nth(0).unwrap())),

        Err(e) => Err(format!("{}", e))
    }
}<commit_msg>Debug parser<commit_after>/// For some reason, Pest cannot find the grammar file if listed in parser/mod.rs, so I listed it here
use pest::Parser;
use pest::iterators::Pair;
use pest::inputs::StrInput;

use ast::Expression;

use std::f64;

const _GRAMMAR: &'static str = include_str!("grammar/grammar.pest");

#[derive(Parser)]
#[grammar = "grammar/grammar.pest"]
pub struct ExpressionParser;

fn parse_expression(input: Pair<Rule, StrInput>) -> Expression {
    match input.as_rule() {
        Rule::function_literal => {
            let mut pairs = input.into_inner().flatten();

            panic!("Nth(0)", pairs.nth(0));
            panic!("Nth(1)", pairs.nth(1));
            panic!("Nth(2)", pairs.nth(2));

            Expression::FuncLiteral(
//                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
//               Box::new(parse_expression(pairs.nth(1).unwrap().into_span())),
//                Box::new(parse_expression(pairs.nth(2).unwrap().into_span()))
                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
                Box::new(parse_expression(pairs.nth(1).unwrap())),
                Box::new(parse_expression(pairs.nth(2).unwrap()))
            )
        }

        Rule::function_call_literal => {
            let mut pairs = input.into_inner();

            println!("{:?}", pairs);

            Expression::FuncCall(
                format!("{}", pairs.nth(0).unwrap().into_span().as_str()),
                Some(Box::new(parse_expression(pairs.nth(2).unwrap())))
            )
        }

        Rule::boolean_literal =>
            Expression::Boolean(
                input.into_span().as_str().parse::<bool>().unwrap()
            ),

        Rule::string_literal =>
            Expression::String(format!("{}", input.into_span().as_str())
            ),

        Rule::identifier_literal =>
            Expression::Identifier(input.into_span().as_str().to_string()),

        // TODO: For now, just conver the value into a normal f64. No formatting!
        Rule::decimal_digits_literal =>
            Expression::Number(
                input.into_span().as_str().parse::<f64>().unwrap()
            ),

        Rule::binary_digits_literal =>
            // The parse function does not support binary digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 2).unwrap())
            ),

        Rule::oct_digits_literal =>
            // The parse function does not support octal digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 8).unwrap())
            ),

        Rule::hex_digits_literal =>
            // The parse function does not support hexadecimal digits, so do it the hard way
            Expression::Number(
                f64::from(i32::from_str_radix(&input.into_span().as_str()[2..], 16).unwrap())
            ),

        Rule::list_literal =>
            Expression::List(input
                            .into_inner()
                            .into_iter()
                            .map(|r| Box::new(parse_expression(r)))
                            .collect()),

        // Temporary
        _ => unreachable!()
    }

}

pub fn parse(input: String) -> Result<Expression, String> {
    // TODO: Remove unwrap
    match ExpressionParser::parse_str(Rule::main, &input) {
        Ok(mut pair) =>
            Ok(parse_expression(pair.nth(0).unwrap().into_inner().nth(0).unwrap())),

        Err(e) => Err(format!("{}", e))
    }
}<|endoftext|>


<commit_before>use super::Token;

use std::iter::{Iterator, Peekable};
use std::fmt;

#[derive(PartialEq)]
enum LexToken {
	Begin,
	End,
	Colon,
	Questionmark,
	Exclamation,
	Plus,
	Slash,
	Quote,
	Character(char)
}

impl LexToken {
	fn push_to_buf(&self, buf: &mut String) {
		match *self {
			LexToken::Begin => {
				buf.push('[');
				buf.push('[');
			},
			LexToken::End => {
				buf.push(']');
				buf.push(']');
			},
			LexToken::Colon => buf.push(':'),
			LexToken::Questionmark => buf.push('?'),
			LexToken::Exclamation => buf.push('!'),
			LexToken::Plus => buf.push('+'),
			LexToken::Slash => buf.push('/'),
			LexToken::Quote => buf.push('"'),
			LexToken::Character(c) => buf.push(c)
		}
	}
}

impl fmt::Display for LexToken {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		match *self {
			LexToken::Begin => {
				'['.fmt(f).and_then(|_| '['.fmt(f))
			},
			LexToken::End => ']'.fmt(f).and_then(|_| ']'.fmt(f)),
			LexToken::Colon => ':'.fmt(f),
			LexToken::Questionmark => '?'.fmt(f),
			LexToken::Exclamation => '!'.fmt(f),
			LexToken::Plus => '+'.fmt(f),
			LexToken::Slash => '/'.fmt(f),
			LexToken::Quote => '"'.fmt(f),
			LexToken::Character(c) => c.fmt(f)
		}
	}
}

struct Parser<T: Iterator<Item=V>, V> {
    tokens: Peekable<V, T>
}

impl<T: Iterator<Item=V>, V: PartialEq> Parser<T, V> {
	#[inline]
	fn eat(&mut self, expected: V) -> bool {
		let eaten = match self.peek() {
			Some(t) if *t == expected => {
				true
			},
			_ => false
		};

		if eaten {
			self.next();
		}

		eaten
	}

	fn eat_while<F: Fn(&V) -> bool>(&mut self, is_edible: F) {
		loop {
			let eaten = match self.peek() {
				Some(t) => is_edible(t),
				_ => false
			};

			if eaten {
				self.next();
			} else {
				break;
			}
		}
	}

	#[inline]
	fn peek<'a>(&'a mut self) -> Option<&'a V> {
		self.tokens.peek()
	}
}

impl<T: Iterator<Item=V>, V> Iterator for Parser<T, V> {
	type Item = V;

	#[inline]
	fn next(&mut self) -> Option<V> {
		self.tokens.by_ref().next()
	}
}

pub fn parse<T: Iterator<Item=Result<char, String>>>(chars: T) -> Result<Vec<Token>, String> {
	let tokens = try!(lex(chars));
	parse_block(&mut Parser{
		tokens: tokens.into_iter().by_ref().peekable()
	})
}

fn lex<T: Iterator<Item=Result<char, String>>>(chars: T) -> Result<Vec<LexToken>, String> {
	let mut chars = chars.peekable();
	let mut tokens = Vec::new();

	loop {
		match chars.next() {
			Some(Ok(c)) => match c {
				'[' => match chars.peek() {
					Some(&Ok('[')) => {
						tokens.push(LexToken::Begin);
						chars.next();
					},
					_ => tokens.push(LexToken::Character('['))
				},
				']' => match chars.peek() {
					Some(&Ok(']')) => {
						tokens.push(LexToken::End);
						chars.next();
					},
					_ => tokens.push(LexToken::Character(']'))
				},
				':' => tokens.push(LexToken::Colon),
				'?' => tokens.push(LexToken::Questionmark),
				'!' => tokens.push(LexToken::Exclamation),
				'+' => tokens.push(LexToken::Plus),
				'/' => tokens.push(LexToken::Slash),
				'"' => tokens.push(LexToken::Quote),
				'\\' => match chars.next() {
					Some(Ok(c)) => {
						tokens.push(LexToken::Character(c));
					},
					Some(Err(e)) => return Err(e),
					None => break
				},
				c => tokens.push(LexToken::Character(c))
			},
			Some(Err(e)) => return Err(e),
			None => break
		}
	}

	Ok(tokens)
}

fn parse_block<T: Iterator<Item=LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Vec<Token>, String> {
	let mut result = Vec::new();
	let mut string = String::new();

	loop {
		match tokens.next() {
			Some(LexToken::Begin) => match tokens.next() {
				Some(LexToken::Colon) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_placeholder(tokens)));
				},
				Some(LexToken::Questionmark) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_conditional(tokens)));
				},
				Some(LexToken::Plus) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_generator(tokens)));
				},
				Some(LexToken::Slash) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					parse_block_end(tokens);
					break
				},
				Some(t) => {
					return Err(format!("parse error: unknown token type: '{}'", t))
				},
				None => LexToken::Begin.push_to_buf(&mut string)
			},
			Some(t) => t.push_to_buf(&mut string),
			None => break
		}
	}

	if string.len() > 0 {
		result.push(Token::String(string));
	}

	Ok(result)
}

fn parse_placeholder<T: Iterator<Item=LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, String> {
	let mut label = String::new();

	for t in tokens.by_ref().take_while(|t| *t != LexToken::End) {
		t.push_to_buf(&mut label);
	}

	Ok(Token::Placeholder(label))
}

fn parse_conditional<T: Iterator<Item=LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, String> {
	let negative = tokens.eat(LexToken::Exclamation);
	let content_cond = tokens.eat(LexToken::Colon);
	let mut label = String::new();

	for t in tokens.by_ref().take_while(|t| *t != LexToken::End) {
		t.push_to_buf(&mut label);
	}

	let content = try!(parse_block(tokens));

	if content_cond {
		Ok(Token::ContentConditional(label, !negative, content))
	} else {
		Ok(Token::Conditional(label, !negative, content))
	}
}

fn parse_generator<T: Iterator<Item=LexToken>>(tokens: &mut Parser<T, LexToken>) -> Result<Token, String> {
	let mut label = String::new();
	let mut args = Vec::new();

	if tokens.eat(LexToken::Quote) {
		for t in tokens.by_ref().take_while(|t| *t != LexToken::Quote) {
			t.push_to_buf(&mut label);
		}
	} else {
		loop {
			match tokens.next() {
				Some(LexToken::End) => return Ok(Token::Generated(label, Vec::new())),
				Some(LexToken::Character(c)) if c.is_whitespace() => break,
				Some(t) => t.push_to_buf(&mut label),
				None => break
			}
		}
	}

	'arg_list: loop {
		let mut new_arg = String::new();

		tokens.eat_while(|t| match *t {LexToken::Character(c) if c.is_whitespace() => true, _ => false});

		if tokens.eat(LexToken::Quote) {
			for t in tokens.by_ref().take_while(|t| *t != LexToken::Quote) {
				t.push_to_buf(&mut new_arg);
			}
		} else {
			'arg: loop {
				match tokens.next() {
					Some(LexToken::End) => if new_arg.len() > 0 {
						args.push(new_arg);
						break 'arg_list;
					},
					Some(LexToken::Character(c)) if c.is_whitespace() => {
						tokens.eat_while(|t| match *t {LexToken::Character(c) if c.is_whitespace() => true, _ => false});
						break 'arg
					},
					Some(t) => t.push_to_buf(&mut new_arg),
					None => break 'arg_list
				}
			}
		}

		args.push(new_arg);
	}

	Ok(Token::Generated(label, args))
}

fn parse_block_end<T: Iterator<Item=LexToken>>(tokens: &mut Parser<T, LexToken>) {
	tokens.by_ref().all(|t| t != LexToken::End);
}

<commit_msg>Use some more associated types in parser<commit_after>use super::Token;

use std::iter::{Iterator, Peekable};
use std::fmt;

#[derive(PartialEq)]
enum LexToken {
	Begin,
	End,
	Colon,
	Questionmark,
	Exclamation,
	Plus,
	Slash,
	Quote,
	Character(char)
}

impl LexToken {
	fn push_to_buf(&self, buf: &mut String) {
		match *self {
			LexToken::Begin => {
				buf.push('[');
				buf.push('[');
			},
			LexToken::End => {
				buf.push(']');
				buf.push(']');
			},
			LexToken::Colon => buf.push(':'),
			LexToken::Questionmark => buf.push('?'),
			LexToken::Exclamation => buf.push('!'),
			LexToken::Plus => buf.push('+'),
			LexToken::Slash => buf.push('/'),
			LexToken::Quote => buf.push('"'),
			LexToken::Character(c) => buf.push(c)
		}
	}
}

impl fmt::Display for LexToken {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		match *self {
			LexToken::Begin => {
				'['.fmt(f).and_then(|_| '['.fmt(f))
			},
			LexToken::End => ']'.fmt(f).and_then(|_| ']'.fmt(f)),
			LexToken::Colon => ':'.fmt(f),
			LexToken::Questionmark => '?'.fmt(f),
			LexToken::Exclamation => '!'.fmt(f),
			LexToken::Plus => '+'.fmt(f),
			LexToken::Slash => '/'.fmt(f),
			LexToken::Quote => '"'.fmt(f),
			LexToken::Character(c) => c.fmt(f)
		}
	}
}

struct Parser<I: Iterator> {
    tokens: Peekable<I>
}

impl<I: Iterator<Item=V>, V: PartialEq> Parser<I> {
	#[inline]
	fn eat(&mut self, expected: V) -> bool {
		let eaten = match self.peek() {
			Some(t) if *t == expected => {
				true
			},
			_ => false
		};

		if eaten {
			self.next();
		}

		eaten
	}

	fn eat_while<F: Fn(&V) -> bool>(&mut self, is_edible: F) {
		loop {
			let eaten = match self.peek() {
				Some(t) => is_edible(t),
				_ => false
			};

			if eaten {
				self.next();
			} else {
				break;
			}
		}
	}

	#[inline]
	fn peek<'a>(&'a mut self) -> Option<&'a V> {
		self.tokens.peek()
	}
}

impl<I: Iterator<Item=V>, V> Iterator for Parser<I> {
	type Item = V;

	#[inline]
	fn next(&mut self) -> Option<V> {
		self.tokens.by_ref().next()
	}
}

pub fn parse<T: Iterator<Item=Result<char, String>>>(chars: T) -> Result<Vec<Token>, String> {
	let tokens = try!(lex(chars));
	parse_block(&mut Parser{
		tokens: tokens.into_iter().by_ref().peekable()
	})
}

fn lex<T: Iterator<Item=Result<char, String>>>(chars: T) -> Result<Vec<LexToken>, String> {
	let mut chars = chars.peekable();
	let mut tokens = Vec::new();

	loop {
		match chars.next() {
			Some(Ok(c)) => match c {
				'[' => match chars.peek() {
					Some(&Ok('[')) => {
						tokens.push(LexToken::Begin);
						chars.next();
					},
					_ => tokens.push(LexToken::Character('['))
				},
				']' => match chars.peek() {
					Some(&Ok(']')) => {
						tokens.push(LexToken::End);
						chars.next();
					},
					_ => tokens.push(LexToken::Character(']'))
				},
				':' => tokens.push(LexToken::Colon),
				'?' => tokens.push(LexToken::Questionmark),
				'!' => tokens.push(LexToken::Exclamation),
				'+' => tokens.push(LexToken::Plus),
				'/' => tokens.push(LexToken::Slash),
				'"' => tokens.push(LexToken::Quote),
				'\\' => match chars.next() {
					Some(Ok(c)) => {
						tokens.push(LexToken::Character(c));
					},
					Some(Err(e)) => return Err(e),
					None => break
				},
				c => tokens.push(LexToken::Character(c))
			},
			Some(Err(e)) => return Err(e),
			None => break
		}
	}

	Ok(tokens)
}

fn parse_block<I: Iterator<Item=LexToken>>(tokens: &mut Parser<I>) -> Result<Vec<Token>, String> {
	let mut result = Vec::new();
	let mut string = String::new();

	loop {
		match tokens.next() {
			Some(LexToken::Begin) => match tokens.next() {
				Some(LexToken::Colon) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_placeholder(tokens)));
				},
				Some(LexToken::Questionmark) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_conditional(tokens)));
				},
				Some(LexToken::Plus) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					result.push(try!(parse_generator(tokens)));
				},
				Some(LexToken::Slash) => {
					if string.len() > 0 {
						result.push(Token::String(string));
						string = String::new();
					}

					parse_block_end(tokens);
					break
				},
				Some(t) => {
					return Err(format!("parse error: unknown token type: '{}'", t))
				},
				None => LexToken::Begin.push_to_buf(&mut string)
			},
			Some(t) => t.push_to_buf(&mut string),
			None => break
		}
	}

	if string.len() > 0 {
		result.push(Token::String(string));
	}

	Ok(result)
}

fn parse_placeholder<I: Iterator<Item=LexToken>>(tokens: &mut Parser<I>) -> Result<Token, String> {
	let mut label = String::new();

	for t in tokens.by_ref().take_while(|t| *t != LexToken::End) {
		t.push_to_buf(&mut label);
	}

	Ok(Token::Placeholder(label))
}

fn parse_conditional<I: Iterator<Item=LexToken>>(tokens: &mut Parser<I>) -> Result<Token, String> {
	let negative = tokens.eat(LexToken::Exclamation);
	let content_cond = tokens.eat(LexToken::Colon);
	let mut label = String::new();

	for t in tokens.by_ref().take_while(|t| *t != LexToken::End) {
		t.push_to_buf(&mut label);
	}

	let content = try!(parse_block(tokens));

	if content_cond {
		Ok(Token::ContentConditional(label, !negative, content))
	} else {
		Ok(Token::Conditional(label, !negative, content))
	}
}

fn parse_generator<I: Iterator<Item=LexToken>>(tokens: &mut Parser<I>) -> Result<Token, String> {
	let mut label = String::new();
	let mut args = Vec::new();

	if tokens.eat(LexToken::Quote) {
		for t in tokens.by_ref().take_while(|t| *t != LexToken::Quote) {
			t.push_to_buf(&mut label);
		}
	} else {
		loop {
			match tokens.next() {
				Some(LexToken::End) => return Ok(Token::Generated(label, Vec::new())),
				Some(LexToken::Character(c)) if c.is_whitespace() => break,
				Some(t) => t.push_to_buf(&mut label),
				None => break
			}
		}
	}

	'arg_list: loop {
		let mut new_arg = String::new();

		tokens.eat_while(|t| match *t {LexToken::Character(c) if c.is_whitespace() => true, _ => false});

		if tokens.eat(LexToken::Quote) {
			for t in tokens.by_ref().take_while(|t| *t != LexToken::Quote) {
				t.push_to_buf(&mut new_arg);
			}
		} else {
			'arg: loop {
				match tokens.next() {
					Some(LexToken::End) => if new_arg.len() > 0 {
						args.push(new_arg);
						break 'arg_list;
					},
					Some(LexToken::Character(c)) if c.is_whitespace() => {
						tokens.eat_while(|t| match *t {LexToken::Character(c) if c.is_whitespace() => true, _ => false});
						break 'arg
					},
					Some(t) => t.push_to_buf(&mut new_arg),
					None => break 'arg_list
				}
			}
		}

		args.push(new_arg);
	}

	Ok(Token::Generated(label, args))
}

fn parse_block_end<I: Iterator<Item=LexToken>>(tokens: &mut Parser<I>) {
	tokens.by_ref().all(|t| t != LexToken::End);
}

<|endoftext|>


<commit_before>use std::error::Error;
use std::fs::File;
use std::iter::Peekable;
use std::str::Chars;

use super::sql::*;

//pub enum Keyword {
//    Select,
//    From,
//    Where
//}

pub enum Token {
    Identifier(String),
    Keyword(String),
    Operator(String),
    Whitespace
}

#[derive(Debug,Clone)]
pub enum ParserError {
    TokenizerError(String)
}

struct Tokenizer {
    query: String,
}

impl Tokenizer {

    pub fn tokenize(&mut self) -> Result<Vec<Token>, ParserError> {

        let mut peekable = self.query.chars().peekable();

        let mut tokens : Vec<Token> = vec![];

        while let Some(token) = self.next_token(&mut peekable)? {
            tokens.push(token);
        }

        Ok(tokens)
    }

    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<Token>, ParserError> {
        match chars.peek() {
            Some(&ch) => match ch {
                ' ' | '\t' | '\n' => {
                    chars.next();
                    Ok(Some(Token::Whitespace))
                },
                _ => Err(ParserError::TokenizerError(String::from("unhandled case in tokenizer")))
            },
            None => Ok(None)
        }
    }

}

<commit_msg>add unit test<commit_after>use std::error::Error;
use std::fs::File;
use std::iter::Peekable;
use std::str::Chars;

use super::sql::*;

//pub enum Keyword {
//    Select,
//    From,
//    Where
//}

#[derive(Debug,Clone)]
pub enum Token {
    Identifier(String),
    Keyword(String),
    Operator(String),
    Whitespace
}

#[derive(Debug,Clone)]
pub enum ParserError {
    TokenizerError(String)
}

struct Tokenizer {
    query: String,
}

impl Tokenizer {

    pub fn tokenize(&mut self) -> Result<Vec<Token>, ParserError> {

        let mut peekable = self.query.chars().peekable();

        let mut tokens : Vec<Token> = vec![];

        while let Some(token) = self.next_token(&mut peekable)? {
            tokens.push(token);
        }

        Ok(tokens)
    }

    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<Token>, ParserError> {
        match chars.peek() {
            Some(&ch) => match ch {
                ' ' | '\t' | '\n' => {
                    chars.next();
                    Ok(Some(Token::Whitespace))
                },
                _ => Err(ParserError::TokenizerError(String::from("unhandled case in tokenizer")))
            },
            None => Ok(None)
        }
    }

}

#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn tokenize_simple_select()  {
        let sql = String::from("SELECT 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        println!("tokens = {:?}", tokens);
        assert_eq!(2, tokens.len());
    }
}

<|endoftext|>


<commit_before>use std::io::{Cursor, Read};

use super::frame::*;
use super::from_bytes;

pub fn parse_frame(vec: Vec<u8>) -> Frame {
    let mut cursor = Cursor::new(vec);
    let mut version_bytes = [0; VERSION_LEN];
    let mut flag_bytes = [0; FLAG_LEN];
    let mut opcode_bytes = [0; OPCODE_LEN];
    let mut stream_bytes = [0; STREAM_LEN];
    let mut length_bytes =[0; LENGTH_LEN];

    // NOTE: order of reads matters
    // TODO: add error logging here
    if let Err(err) = cursor.read(&mut version_bytes) {
        error!("Parse Cassandra version error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut flag_bytes) {
        error!("Parse Cassandra flag error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut stream_bytes) {
        error!("Parse Cassandra stream error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut opcode_bytes) {
        error!("Parse Cassandra opcode error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut length_bytes) {
        error!("Parse Cassandra length error: {}", err);
        panic!(err);
    }

    let version = Version::from(version_bytes.to_vec());
    let flag = Flag::from(flag_bytes.to_vec());
    let stream = from_bytes(stream_bytes.to_vec());
    let opcode = Opcode::from(opcode_bytes.to_vec());
    let length = from_bytes(length_bytes.to_vec()) as usize;

    let mut body_bytes = Vec::with_capacity(length);
    if let Err(err) = cursor.read(&mut body_bytes) {
        error!("Parse Cassandra body error: {}", err);
        panic!(err);
    }

    return Frame {
        version: version,
        flag: flag,
        opcode: opcode,
        stream: stream,
        body: body_bytes
    }
}
<commit_msg>Replace unwrap by logger<commit_after>use std::io::{Cursor, Read};

use super::frame::*;
use super::from_bytes;

pub fn parse_frame(vec: Vec<u8>) -> Frame {
    let mut cursor = Cursor::new(vec);
    let mut version_bytes = [0; VERSION_LEN];
    let mut flag_bytes = [0; FLAG_LEN];
    let mut opcode_bytes = [0; OPCODE_LEN];
    let mut stream_bytes = [0; STREAM_LEN];
    let mut length_bytes =[0; LENGTH_LEN];

    // NOTE: order of reads matters
    if let Err(err) = cursor.read(&mut version_bytes) {
        error!("Parse Cassandra version error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut flag_bytes) {
        error!("Parse Cassandra flag error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut stream_bytes) {
        error!("Parse Cassandra stream error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut opcode_bytes) {
        error!("Parse Cassandra opcode error: {}", err);
        panic!(err);
    }
    if let Err(err) = cursor.read(&mut length_bytes) {
        error!("Parse Cassandra length error: {}", err);
        panic!(err);
    }

    let version = Version::from(version_bytes.to_vec());
    let flag = Flag::from(flag_bytes.to_vec());
    let stream = from_bytes(stream_bytes.to_vec());
    let opcode = Opcode::from(opcode_bytes.to_vec());
    let length = from_bytes(length_bytes.to_vec()) as usize;

    let mut body_bytes = Vec::with_capacity(length);
    if let Err(err) = cursor.read(&mut body_bytes) {
        error!("Parse Cassandra body error: {}", err);
        panic!(err);
    }

    return Frame {
        version: version,
        flag: flag,
        opcode: opcode,
        stream: stream,
        body: body_bytes
    }
}
<|endoftext|>


<commit_before>//! Manage score and scores

use super::cards;

/// Returns the number of score `card` is worth, with the current trump suit.
pub fn score(card: cards::Card, trump: cards::Suit) -> i32 {
    let r = card.rank();
    if card.suit() == trump {
        trump_score(r)
    } else {
        usual_score(r)
    }
}

/// Returns the strength of `card`, with the current trump suit.
pub fn strength(card: cards::Card, trump: cards::Suit) -> i32 {
    let r = card.rank();
    if card.suit() == trump {
        8 + trump_strength(r)
    } else {
        usual_strength(r)
    }
}

/// Returns the score for the given rank when it is the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn trump_score(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::RankJ => 20,
        cards::Rank::Rank9 => 14,
        _ => usual_score(rank),
    }
}

/// Returns the score for the given rank when it is not the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn usual_score(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 0,
        cards::Rank::Rank9 => 0,
        cards::Rank::RankJ => 2,
        cards::Rank::RankQ => 3,
        cards::Rank::RankK => 4,
        cards::Rank::RankX => 10,
        cards::Rank::RankA => 11,
    }
}

/// Returns the strength for the given rank when it is the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn trump_strength(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 1,
        cards::Rank::RankQ => 2,
        cards::Rank::RankK => 3,
        cards::Rank::RankX => 4,
        cards::Rank::RankA => 5,
        cards::Rank::Rank9 => 6,
        cards::Rank::RankJ => 7,
    }
}

/// Returns the strength for the given rank when it is not the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn usual_strength(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 1,
        cards::Rank::Rank9 => 2,
        cards::Rank::RankJ => 3,
        cards::Rank::RankQ => 4,
        cards::Rank::RankK => 5,
        cards::Rank::RankX => 6,
        cards::Rank::RankA => 7,
    }
}
<commit_msg>Fix typo in doc<commit_after>//! Manage points and scores

use super::cards;

/// Returns the number of score `card` is worth, with the current trump suit.
pub fn score(card: cards::Card, trump: cards::Suit) -> i32 {
    let r = card.rank();
    if card.suit() == trump {
        trump_score(r)
    } else {
        usual_score(r)
    }
}

/// Returns the strength of `card`, with the current trump suit.
pub fn strength(card: cards::Card, trump: cards::Suit) -> i32 {
    let r = card.rank();
    if card.suit() == trump {
        8 + trump_strength(r)
    } else {
        usual_strength(r)
    }
}

/// Returns the score for the given rank when it is the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn trump_score(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::RankJ => 20,
        cards::Rank::Rank9 => 14,
        _ => usual_score(rank),
    }
}

/// Returns the score for the given rank when it is not the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn usual_score(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 0,
        cards::Rank::Rank9 => 0,
        cards::Rank::RankJ => 2,
        cards::Rank::RankQ => 3,
        cards::Rank::RankK => 4,
        cards::Rank::RankX => 10,
        cards::Rank::RankA => 11,
    }
}

/// Returns the strength for the given rank when it is the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn trump_strength(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 1,
        cards::Rank::RankQ => 2,
        cards::Rank::RankK => 3,
        cards::Rank::RankX => 4,
        cards::Rank::RankA => 5,
        cards::Rank::Rank9 => 6,
        cards::Rank::RankJ => 7,
    }
}

/// Returns the strength for the given rank when it is not the trump.
///
/// # Panics
/// If `rank` is invalid.
pub fn usual_strength(rank: cards::Rank) -> i32 {
    match rank {
        cards::Rank::Rank7 => 0,
        cards::Rank::Rank8 => 1,
        cards::Rank::Rank9 => 2,
        cards::Rank::RankJ => 3,
        cards::Rank::RankQ => 4,
        cards::Rank::RankK => 5,
        cards::Rank::RankX => 6,
        cards::Rank::RankA => 7,
    }
}
<|endoftext|>


<commit_before>extern crate rand;

pub use self::rand::Rng;

use super::structure::{Program,Condition,Command,Expression,Sensor};

#[macro_export]
macro_rules! pick {
    ($( $weight: expr, $expression: expr),+) => {{
        let total = 0 $(+ $weight)+;
        let mut bound = 0;

        let mut rng = rand::thread_rng();
        let random_number = rng.gen_range(0, total);
        let result = $( if bound <= random_number && random_number < { bound += $weight; bound } {
            $expression
        } else )+ {
           panic!();
        };
        result
    }}
}

impl rand::Rand for Program {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            1, Program::If(Condition::rand(rng), Box::new(Program::rand(rng)), Box::new(Program::rand(rng))),
            1, Program::Command(Command::rand(rng))
        ]
	  }
}

impl rand::Rand for Condition {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            8, Condition::True,
			      8, Condition::False,
			      2, Condition::Not(Box::new(Condition::rand(rng))),
			      1, Condition::Or(Box::new(Condition::rand(rng)), Box::new(Condition::rand(rng))),
			      1, Condition::And(Box::new(Condition::rand(rng)), Box::new(Condition::rand(rng))),
			      1, Condition::Less(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::LessEqual(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::Equal(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::GreaterEqual(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::Greater(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng)))
		    ]
	  }
}

impl rand::Rand for Command {
	fn rand<R: rand::Rng>(_: &mut R) -> Self {
		  pick![1, Command::Skip, 1, Command::Left, 1, Command::Right, 1, Command::Up]
  }
}

impl rand::Rand for Expression {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            5, Expression::Constant(rng.next_f32()),
            5, Expression::Sensor(Sensor::rand(rng)),
            1, Expression::Plus(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Minus(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Multiply(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Divide(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng)))
        ]
	  }
}

impl rand::Rand for Sensor {
	  fn rand<R: rand::Rng>(_: &mut R) -> Self {
        pick![1, Sensor::X, 1, Sensor::Y, 1, Sensor::Vx, 1, Sensor::Vy, 1, Sensor::O, 1, Sensor::W]
	}
}
<commit_msg>Module documentation for random<commit_after>//! The `random` module creates random `ast::structure::Program`.
//!
//! It implements `rand::Rand` for all important structures. It does this by
//! making use of a macro `pick` that can choose an expression according to
//! weight.
//!
//! # Examples
//!
//! ```
//! #[macro_use]
//! extern crate ast;
//! extern crate rand;
//! extern crate rustc_serialize;
//!
//! pub use rand::Rng;
//!
//! use ast::structure::Program;
//! use ast::source::Source;
//!
//! fn main() {
//!     let generated: Program = rand::random();
//!     println!("{}", generated.source());
//! }
//! ```

extern crate rand;

pub use self::rand::Rng;

use super::structure::{Program,Condition,Command,Expression,Sensor};

#[macro_export]
macro_rules! pick {
    ($( $weight: expr, $expression: expr),+) => {{
        let total = 0 $(+ $weight)+;
        let mut bound = 0;

        let mut rng = rand::thread_rng();
        let random_number = rng.gen_range(0, total);
        let result = $( if bound <= random_number && random_number < { bound += $weight; bound } {
            $expression
        } else )+ {
           panic!();
        };
        result
    }}
}

impl rand::Rand for Program {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            1, Program::If(Condition::rand(rng), Box::new(Program::rand(rng)), Box::new(Program::rand(rng))),
            1, Program::Command(Command::rand(rng))
        ]
	  }
}

impl rand::Rand for Condition {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            8, Condition::True,
			      8, Condition::False,
			      2, Condition::Not(Box::new(Condition::rand(rng))),
			      1, Condition::Or(Box::new(Condition::rand(rng)), Box::new(Condition::rand(rng))),
			      1, Condition::And(Box::new(Condition::rand(rng)), Box::new(Condition::rand(rng))),
			      1, Condition::Less(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::LessEqual(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::Equal(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::GreaterEqual(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
			      1, Condition::Greater(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng)))
		    ]
	  }
}

impl rand::Rand for Command {
	fn rand<R: rand::Rng>(_: &mut R) -> Self {
		  pick![1, Command::Skip, 1, Command::Left, 1, Command::Right, 1, Command::Up]
  }
}

impl rand::Rand for Expression {
	  fn rand<R: rand::Rng>(rng: &mut R) -> Self {
        pick![
            5, Expression::Constant(rng.next_f32()),
            5, Expression::Sensor(Sensor::rand(rng)),
            1, Expression::Plus(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Minus(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Multiply(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng))),
            1, Expression::Divide(Box::new(Expression::rand(rng)), Box::new(Expression::rand(rng)))
        ]
	  }
}

impl rand::Rand for Sensor {
	  fn rand<R: rand::Rng>(_: &mut R) -> Self {
        pick![1, Sensor::X, 1, Sensor::Y, 1, Sensor::Vx, 1, Sensor::Vy, 1, Sensor::O, 1, Sensor::W]
	}
}
<|endoftext|>


<commit_before>//! Reader types to parse a configuration.

use std::io::Read;
use std::path::Path;
use std::fs::File;

use types::Config;
use parser::parse;
use error::ConfigError;
use error::{from_io_err, from_parse_err};

/// Reads a configuration from a generic stream.
/// Errors can be caused by:
///
/// * An I/O error on `stream`, in which case no parsing was done
/// * A syntax error
///
/// If a syntax error is reported, it means that the stream successfully delivered every piece of
/// data, since parsing doesn't start until the whole input is read to memory.
/// # Examples
/// For educational / demonstration purposes, we can wrap a string inside a `Cursor` to simulate
/// a stream of data:
///
/// ```
/// use std::io::Cursor;
/// use config::reader::from_stream;
///
/// let sample_conf = "windows=NO;\nlinux = YES;\n";
/// let mut cursor = Cursor::new(sample_conf.as_bytes());
/// let parsed = from_stream(&mut cursor);
/// assert!(parsed.is_ok());
/// ```
///
/// In this example, we do the same, but with a broken conf:
///
/// ```ignore
/// use std::io::Cursor;
/// use config::reader::from_stream;
/// use config::error::ConfigErrorKind;
///
/// let sample_conf = "windows=\n";
/// let mut cursor = Cursor::new(sample_conf.as_bytes());
/// let parsed = from_stream(&mut cursor);
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
/// ```
///
/// The other situation where an error is returned is when the underlying stream
/// yields an I/O error. We can simulate this behavior by implementing a reader that
/// always returns an error:
///
/// ```
/// use std::io::{Read, Cursor};
/// use std::io::Error as IoError;
/// use std::io::Result as IoResult;
/// use std::io::ErrorKind;
///
/// use config::reader::from_stream;
/// use config::error::ConfigErrorKind;
///
/// struct BadCursor;
///
/// impl Read for BadCursor {
///     fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
///         Err(IoError::new(ErrorKind::Other, "An I/O error has occurred."))
///     }
/// }
///
/// let parsed = from_stream(&mut BadCursor);
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::IoError);
///
/// ```
///
pub fn from_stream<T: Read>(stream: &mut T) -> Result<Config, ConfigError> {
    let mut buf = String::new();

    match stream.read_to_string(&mut buf) {
        Ok(_) => from_str(&buf[..]),
        Err(e) => Err(from_io_err(e))
    }
}

/// Reads a configuration from a UTF-8 file.
/// Errors can be caused by:
///
/// * An error when trying to locate / open the file. This is treated as an I/O error.
/// * A syntax error
///
/// Errors upon opening the file can happen due to the file not existing, or bad permissions, etc.
///
/// # Examples
/// This reads and parses a configuration stored in `examples/sample.conf`:
///
/// ```
/// use std::path::Path;
///
/// use config::reader::from_file;
///
/// let parsed = from_file(Path::new("tests/sample.conf"));
/// assert!(parsed.is_ok());
/// ```
///
pub fn from_file(path: &Path) -> Result<Config, ConfigError> {
    let mut file = match File::open(path) {
        Ok(f) => f,
        Err(e) => return Err(from_io_err(e))
    };
    from_stream(&mut file)
}

/// Reads a configuration from a string slice.
/// The only possible error that can occur is a syntax error.
/// # Examples
///
/// ```
/// use config::reader::from_str;
/// use config::error::ConfigErrorKind;
///
/// let parsed = from_str("windows=NO;\nlinux=true;\n");
/// assert!(parsed.is_ok());
/// ```
///
/// This will return a syntax error (missing a semi-colon)
///
/// ```
/// use config::reader::from_str;
/// use config::error::ConfigErrorKind;
///
/// let parsed = from_str("windows=NO\n");
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
/// ```
///
pub fn from_str(input: &str) -> Result<Config, ConfigError> {
    parse(input).map_err(|e| from_parse_err(e))
}

#[cfg(test)]
mod test {

    use super::{from_str, from_stream, from_file};

    use types::Config;
    use types::{SettingsList, Setting, Value, ScalarValue};
    use error::{ConfigErrorKind};

    use std::io::{Read, Cursor};
    use std::io::Error as IoError;
    use std::io::ErrorKind;
    use std::io::Result as IoResult;
    use std::path::Path;

    struct BadStrCursor<'a> {
        cursor: Cursor<&'a [u8]>,
        calls: u16,
        max_calls_before_err: u16
    }

    impl<'a> BadStrCursor<'a> {
        fn new(data: &'a [u8], max_calls: u16) -> BadStrCursor<'a> {
            BadStrCursor { cursor: Cursor::new(data), calls: 0, max_calls_before_err: max_calls }
        }
    }

    impl<'a> Read for BadStrCursor<'a> {
        fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
            self.calls += 1;
            if self.calls >= self.max_calls_before_err {
                Err(IoError::new(ErrorKind::Other, "An I/O error has occurred."))
            } else {
                self.cursor.read(buf)
            }
        }
    }

    #[test]
    fn conf_from_str() {
        let parsed = from_str("windows=NO;\nlinux = true;\nUNIX\t=\nFaLsE;\n");
        assert!(parsed.is_ok());

        let mut expected = SettingsList::new();
        expected.insert("windows".to_string(),
                        Setting::new("windows".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));
        expected.insert("linux".to_string(),
                        Setting::new("linux".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(true))));
        expected.insert("UNIX".to_string(),
                        Setting::new("UNIX".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));

        assert_eq!(parsed.unwrap(), Config::new(expected));
    }

    #[test]
    fn conf_from_str_parse_err() {
        let parsed = from_str("windows=NO\nlinux=true;\n");
        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
    }

    #[test]
    fn conf_from_stream() {
        let sample_conf = "windows=NO;\nlinux = true;\nUNIX\t=\nFaLsE;\n";
        let mut cursor = Cursor::new(sample_conf.as_bytes());
        let parsed = from_stream(&mut cursor);

        assert!(parsed.is_ok());
        let mut expected = SettingsList::new();
        expected.insert("windows".to_string(),
                        Setting::new("windows".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));
        expected.insert("linux".to_string(),
                        Setting::new("linux".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(true))));
        expected.insert("UNIX".to_string(),
                        Setting::new("UNIX".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));

        assert_eq!(parsed.unwrap(), Config::new(expected));
    }

    #[test]
    fn conf_from_stream_parse_err() {
        let sample_conf = "windows=NO\nlinux = true;\n";
        let mut cursor = Cursor::new(sample_conf.as_bytes());
        let parsed = from_stream(&mut cursor);

        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
    }

    #[test]
    fn conf_from_stream_io_err() {
        let sample_conf = "windows=NO;\nlinux = true;\n";
        let mut bad_cursor = BadStrCursor::new(sample_conf.as_bytes(), 1);
        let parsed = from_stream(&mut bad_cursor);

        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::IoError);
    }

    #[test]
    fn conf_from_file() {
        let parsed = from_file(Path::new("tests/sample.conf"));
        assert!(parsed.is_ok());
    }
}
<commit_msg>Fixed compilation error in documentation tests.<commit_after>//! Reader types to parse a configuration.

use std::io::Read;
use std::path::Path;
use std::fs::File;

use types::Config;
use parser::parse;
use error::ConfigError;
use error::{from_io_err, from_parse_err};

/// Reads a configuration from a generic stream.
/// Errors can be caused by:
///
/// * An I/O error on `stream`, in which case no parsing was done
/// * A syntax error
///
/// If a syntax error is reported, it means that the stream successfully delivered every piece of
/// data, since parsing doesn't start until the whole input is read to memory.
/// # Examples
/// For educational / demonstration purposes, we can wrap a string inside a `Cursor` to simulate
/// a stream of data:
///
/// ```
/// use std::io::Cursor;
/// use config::reader::from_stream;
///
/// let sample_conf = "windows=NO;\nlinux = YES;\n";
/// let mut cursor = Cursor::new(sample_conf.as_bytes());
/// let parsed = from_stream(&mut cursor);
/// assert!(parsed.is_ok());
/// ```
///
/// In this example, we do the same, but with a broken conf:
///
/// ```
/// use std::io::Cursor;
/// use config::reader::from_stream;
/// use config::error::ConfigErrorKind;
///
/// let sample_conf = "windows=\n";
/// let mut cursor = Cursor::new(sample_conf.as_bytes());
/// let parsed = from_stream(&mut cursor);
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
/// ```
///
/// The other situation where an error is returned is when the underlying stream
/// yields an I/O error. We can simulate this behavior by implementing a reader that
/// always returns an error:
///
/// ```
/// use std::io::{Read, Cursor};
/// use std::io::Error as IoError;
/// use std::io::Result as IoResult;
/// use std::io::ErrorKind;
///
/// use config::reader::from_stream;
/// use config::error::ConfigErrorKind;
///
/// struct BadCursor;
///
/// impl Read for BadCursor {
///     fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
///         Err(IoError::new(ErrorKind::Other, "An I/O error has occurred."))
///     }
/// }
///
/// let parsed = from_stream(&mut BadCursor);
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::IoError);
///
/// ```
///
pub fn from_stream<T: Read>(stream: &mut T) -> Result<Config, ConfigError> {
    let mut buf = String::new();

    match stream.read_to_string(&mut buf) {
        Ok(_) => from_str(&buf[..]),
        Err(e) => Err(from_io_err(e))
    }
}

/// Reads a configuration from a UTF-8 file.
/// Errors can be caused by:
///
/// * An error when trying to locate / open the file. This is treated as an I/O error.
/// * A syntax error
///
/// Errors upon opening the file can happen due to the file not existing, or bad permissions, etc.
///
/// # Examples
/// This reads and parses a configuration stored in `examples/sample.conf`:
///
/// ```
/// use std::path::Path;
///
/// use config::reader::from_file;
///
/// let parsed = from_file(Path::new("tests/sample.conf"));
/// assert!(parsed.is_ok());
/// ```
///
pub fn from_file(path: &Path) -> Result<Config, ConfigError> {
    let mut file = match File::open(path) {
        Ok(f) => f,
        Err(e) => return Err(from_io_err(e))
    };
    from_stream(&mut file)
}

/// Reads a configuration from a string slice.
/// The only possible error that can occur is a syntax error.
/// # Examples
///
/// ```
/// use config::reader::from_str;
/// use config::error::ConfigErrorKind;
///
/// let parsed = from_str("windows=NO;\nlinux=true;\n");
/// assert!(parsed.is_ok());
/// ```
///
/// This will return a syntax error (missing a semi-colon)
///
/// ```
/// use config::reader::from_str;
/// use config::error::ConfigErrorKind;
///
/// let parsed = from_str("windows=NO\n");
/// assert!(parsed.is_err());
/// assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
/// ```
///
pub fn from_str(input: &str) -> Result<Config, ConfigError> {
    parse(input).map_err(|e| from_parse_err(e))
}

#[cfg(test)]
mod test {

    use super::{from_str, from_stream, from_file};

    use types::Config;
    use types::{SettingsList, Setting, Value, ScalarValue};
    use error::{ConfigErrorKind};

    use std::io::{Read, Cursor};
    use std::io::Error as IoError;
    use std::io::ErrorKind;
    use std::io::Result as IoResult;
    use std::path::Path;

    struct BadStrCursor<'a> {
        cursor: Cursor<&'a [u8]>,
        calls: u16,
        max_calls_before_err: u16
    }

    impl<'a> BadStrCursor<'a> {
        fn new(data: &'a [u8], max_calls: u16) -> BadStrCursor<'a> {
            BadStrCursor { cursor: Cursor::new(data), calls: 0, max_calls_before_err: max_calls }
        }
    }

    impl<'a> Read for BadStrCursor<'a> {
        fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
            self.calls += 1;
            if self.calls >= self.max_calls_before_err {
                Err(IoError::new(ErrorKind::Other, "An I/O error has occurred."))
            } else {
                self.cursor.read(buf)
            }
        }
    }

    #[test]
    fn conf_from_str() {
        let parsed = from_str("windows=NO;\nlinux = true;\nUNIX\t=\nFaLsE;\n");
        assert!(parsed.is_ok());

        let mut expected = SettingsList::new();
        expected.insert("windows".to_string(),
                        Setting::new("windows".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));
        expected.insert("linux".to_string(),
                        Setting::new("linux".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(true))));
        expected.insert("UNIX".to_string(),
                        Setting::new("UNIX".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));

        assert_eq!(parsed.unwrap(), Config::new(expected));
    }

    #[test]
    fn conf_from_str_parse_err() {
        let parsed = from_str("windows=NO\nlinux=true;\n");
        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
    }

    #[test]
    fn conf_from_stream() {
        let sample_conf = "windows=NO;\nlinux = true;\nUNIX\t=\nFaLsE;\n";
        let mut cursor = Cursor::new(sample_conf.as_bytes());
        let parsed = from_stream(&mut cursor);

        assert!(parsed.is_ok());
        let mut expected = SettingsList::new();
        expected.insert("windows".to_string(),
                        Setting::new("windows".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));
        expected.insert("linux".to_string(),
                        Setting::new("linux".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(true))));
        expected.insert("UNIX".to_string(),
                        Setting::new("UNIX".to_string(),
                                     Value::Svalue(ScalarValue::Boolean(false))));

        assert_eq!(parsed.unwrap(), Config::new(expected));
    }

    #[test]
    fn conf_from_stream_parse_err() {
        let sample_conf = "windows=NO\nlinux = true;\n";
        let mut cursor = Cursor::new(sample_conf.as_bytes());
        let parsed = from_stream(&mut cursor);

        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::ParseError);
    }

    #[test]
    fn conf_from_stream_io_err() {
        let sample_conf = "windows=NO;\nlinux = true;\n";
        let mut bad_cursor = BadStrCursor::new(sample_conf.as_bytes(), 1);
        let parsed = from_stream(&mut bad_cursor);

        assert!(parsed.is_err());
        assert_eq!(parsed.unwrap_err().kind, ConfigErrorKind::IoError);
    }

    #[test]
    fn conf_from_file() {
        let parsed = from_file(Path::new("tests/sample.conf"));
        assert!(parsed.is_ok());
    }
}
<|endoftext|>


<commit_before>use rustc_serialize::json;
use rustc_serialize::json::Json;
use rustc_serialize::Encodable;
use rmp_serialize::encode;
use time::Tm;
use std::io;
use retry;
use forwardable::msgpack::Message;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Record<T: Encodable> {
    pub tag: String,
    pub time: Tm,
    pub record: T,
}

pub enum FluentError {
    EncodeError(json::EncoderError),
    MsgpackEncodeError(encode::Error),
    IOError(io::Error),
    RetryError(retry::RetryError),
}

impl From<io::Error> for FluentError {
    fn from(err: io::Error) -> FluentError {
        FluentError::IOError(err)
    }
}

impl From<encode::Error> for FluentError {
    fn from(err: encode::Error) -> FluentError {
        FluentError::MsgpackEncodeError(err)
    }
}

impl From<retry::RetryError> for FluentError {
    fn from(err: retry::RetryError) -> FluentError {
        FluentError::RetryError(err)
    }
}

impl From<json::EncoderError> for FluentError {
    fn from(err: json::EncoderError) -> FluentError {
        FluentError::EncodeError(err)
    }
}

impl<T: Encodable> Record<T> {
    pub fn new(tag: String, time: Tm, record: T) -> Record<T> {
        Record {
            tag: tag,
            time: time,
            record: record,
        }
    }

    pub fn make_forwardable_json(self) -> Result<String, FluentError> {
        let tag = try!(json::encode(&self.tag));
        let record = try!(json::encode(&self.record));
        let option = Json::Null;
        let message = format!("[{},{},{},{}]",
                              tag,
                              self.time.to_timespec().sec,
                              record,
                              option);
        Ok(message)
    }

    pub fn to_message(self) -> Message<T> {
        Message::new(self)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use time;
    use std::collections::HashMap;
    use rustc_serialize::json;
    use rustc_serialize::json::Json;

    #[test]
    fn test_json_format() {
        let tag = "fruently".to_string();
        let time = time::now();
        let mut obj: HashMap<String, String> = HashMap::new();
        obj.insert("name".to_string(), "fruently".to_string());
        let record = Record::new(tag.clone(), time, obj.clone());
        let forwardable_json = record.make_forwardable_json().ok().unwrap();
        let json_tag = json::encode(&tag.clone()).ok().unwrap();
        let json_obj = json::encode(&obj.clone()).ok().unwrap();
        let expected = format!("[{},{},{},{}]",
                               json_tag,
                               time.to_timespec().sec,
                               json_obj,
                               Json::Null);
        assert_eq!(expected, forwardable_json);
    }
}
<commit_msg>record: EncodeError -> JsonEncodeError<commit_after>use rustc_serialize::json;
use rustc_serialize::json::Json;
use rustc_serialize::Encodable;
use rmp_serialize::encode;
use time::Tm;
use std::io;
use retry;
use forwardable::msgpack::Message;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Record<T: Encodable> {
    pub tag: String,
    pub time: Tm,
    pub record: T,
}

pub enum FluentError {
    JsonEncodeError(json::EncoderError),
    MsgpackEncodeError(encode::Error),
    IOError(io::Error),
    RetryError(retry::RetryError),
}

impl From<io::Error> for FluentError {
    fn from(err: io::Error) -> FluentError {
        FluentError::IOError(err)
    }
}

impl From<encode::Error> for FluentError {
    fn from(err: encode::Error) -> FluentError {
        FluentError::MsgpackEncodeError(err)
    }
}

impl From<retry::RetryError> for FluentError {
    fn from(err: retry::RetryError) -> FluentError {
        FluentError::RetryError(err)
    }
}

impl From<json::EncoderError> for FluentError {
    fn from(err: json::EncoderError) -> FluentError {
        FluentError::JsonEncodeError(err)
    }
}

impl<T: Encodable> Record<T> {
    pub fn new(tag: String, time: Tm, record: T) -> Record<T> {
        Record {
            tag: tag,
            time: time,
            record: record,
        }
    }

    pub fn make_forwardable_json(self) -> Result<String, FluentError> {
        let tag = try!(json::encode(&self.tag));
        let record = try!(json::encode(&self.record));
        let option = Json::Null;
        let message = format!("[{},{},{},{}]",
                              tag,
                              self.time.to_timespec().sec,
                              record,
                              option);
        Ok(message)
    }

    pub fn to_message(self) -> Message<T> {
        Message::new(self)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use time;
    use std::collections::HashMap;
    use rustc_serialize::json;
    use rustc_serialize::json::Json;

    #[test]
    fn test_json_format() {
        let tag = "fruently".to_string();
        let time = time::now();
        let mut obj: HashMap<String, String> = HashMap::new();
        obj.insert("name".to_string(), "fruently".to_string());
        let record = Record::new(tag.clone(), time, obj.clone());
        let forwardable_json = record.make_forwardable_json().ok().unwrap();
        let json_tag = json::encode(&tag.clone()).ok().unwrap();
        let json_obj = json::encode(&obj.clone()).ok().unwrap();
        let expected = format!("[{},{},{},{}]",
                               json_tag,
                               time.to_timespec().sec,
                               json_obj,
                               Json::Null);
        assert_eq!(expected, forwardable_json);
    }
}
<|endoftext|>