<commit_before>use token::Token;
use tokenizer::Tokenizer;

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

#[derive(PartialEq, Debug)]
pub enum ParseError {
    SyntaxError,
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        Err(ParseError::SyntaxError)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Started to implement parse<commit_after>use token::Token;
use tokenizer::Tokenizer;

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

#[derive(PartialEq, Debug)]
pub enum ParseError {
    SyntaxError,
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        let mut lines: Program = Vec::new();

        loop {
            let line = try!(self.line());
            if line.is_none() {
                break;
            }

            let line = line.unwrap();
            lines.push(line);
        }

        Ok(lines)
    }

    fn line(&mut self) -> Result<Option<Statement>, ParseError> {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>


<commit_before>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Eof,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    pub fn next_token(&mut self) -> Token {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return Token::new(self.position, self.position, TokenKind::Eof),
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Token::new(self.token_start, self.position, kind)
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => continue };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),
                                            ErrorLevel::Error,
                                            format!("invalid character escape: {:?}", c));
                            c
                        }
                    });
                }

                '"' => break,
                c => string.push(c),
            }
        }

        string
    }

    // fn report_error<S: Into<String>>(&self, message: S) {
    //     errors().report(Span::new(self.token_start, self.position), message.into());
    // }

    // fn peek(&self) -> Option<char> {
    //     self.current.and_then(|c| self.char_at(self.position + c.len_utf8()))
    // }

    fn char_at(&self, pos: usize) -> Option<char> {
        self.source[pos..].chars().next()
    }

    /// Move to the next char in the source code.
    fn advance(&mut self) {
        if let Some(c) = self.current {
            self.position += c.len_utf8();
            self.current = self.char_at(self.position);
        } else {
            panic!("lexer attempted to advance past the end of the source code");
        }
    }
}

impl<'src> Iterator for Lexer<'src> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        if self.position == self.source.len() {
            None
        } else {
            Some(self.next_token())
        }
    }
}

// pub fn tokenize(source: &str) -> Vec<Token> {
//     let mut lexer = Lexer::new(source);
//     vec![]
// }

////////////////////////////////////////////////////////////////////////////////
// Parser
////////////////////////////////////////////////////////////////////////////////

// #[derive(Clone, Debug)]
// struct Parser<'src> {
//     lexer: Lexer<'src>,
//     errors: Vec<Error>,
// }

    // fn parse_module(&mut self) -> ParseResult<ast::Module> {
    //     let mut fns = Vec::new();
    //     while self.token != Token::Eof {
    //         fns.push(try!(self.parse_fn_def()));
    //     }
    //     Ok(ast::Module { fns: fns })
    // }

    // fn parse_fn_def(&mut self) -> ParseResult<ast::FnDef> {
    //     try!(self.expect(&Token::KeywordFn));

    //     let name = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     // TODO(tsion): Parse arguments.

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::BraceL));

    //     let mut body = Vec::new();

    //     while self.token != Token::BraceR {
    //         body.push(try!(self.parse_expr()));
    //     }

    //     // Skip the closing brace.
    //     self.advance();

    //     Ok(ast::FnDef {
    //         name: name,
    //         return_ty: ast::Type::Unit,
    //         args: Vec::new(),
    //         body: body,
    //     })
    // }

    // fn parse_expr(&mut self) -> ParseResult<ast::Expr> {
    //     let func = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     let str = match self.advance_and_get() {
    //         Token::StrLit(str) => str,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::Semicolon));

    //     Ok(ast::Expr::FnCall {
    //         func: func,
    //         args: vec![ast::Expr::StrLit(str)],
    //     })
    // }

////////////////////////////////////////////////////////////////////////////////
// Character classes
////////////////////////////////////////////////////////////////////////////////

/// Returns `true` if the given character is whitespace.
fn is_whitespace(c: char) -> bool {
    match c {
        ' ' | '\t' | '\r' | '\n' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is a digit.
fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid at the start of an identifier.
fn is_ident_start(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '_' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid after the start of an identifier.
fn is_ident_continue(c: char) -> bool {
    is_ident_start(c) || is_digit(c)
}

////////////////////////////////////////////////////////////////////////////////
// Errors
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error {
    pub message: String,
    pub level: ErrorLevel,
    pub span: Span,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ErrorLevel {
    Error,
    Warning,
    Note,
}

pub struct ErrorReporter {
    pub errors: RefCell<Vec<Error>>,
}

impl ErrorReporter {
    fn new() -> Self {
        ErrorReporter { errors: RefCell::new(Vec::new()) }
    }

    fn report<S: Into<String>>(&self, span: Span, level: ErrorLevel, message: S) {
        self.errors.borrow_mut().push(Error {
            message: message.into(),
            level: level,
            span: span,
        });
    }

    fn is_empty(&self) -> bool {
        self.errors.borrow().is_empty()
    }

    fn reset(&self) {
        self.errors.borrow_mut().clear();
    }
}

pub fn errors() -> Rc<ErrorReporter> {
    thread_local!(static KEY: Rc<ErrorReporter> = Rc::new(ErrorReporter::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
mod test {
    use super::{errors, ErrorLevel, Lexer, Span, Token, TokenKind};
    use super::ErrorLevel::*;
    use super::TokenKind::*;

    #[test]
    fn lex_empty_string() {
        lexer_test(r#""""#, vec![StrLit(String::from(""))], &[]);
    }

    #[test]
    fn lex_unterminated_empty_string() {
        lexer_test(r#"""#, vec![StrLit(String::from(""))], &[
            (1, 1, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_empty_string() {
        lexer_test(r#""\"#, vec![StrLit(String::from(""))], &[
            (2, 2, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_string() {
        lexer_test(r#""foo\"#, vec![StrLit(String::from("foo"))], &[
            (5, 5, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string() {
        lexer_test(r#""foobar""#, vec![StrLit(String::from("foobar"))], &[]);
    }

    #[test]
    fn lex_unterminated_string() {
        lexer_test(r#""foobar"#, vec![StrLit(String::from("foobar"))], &[
            (7, 7, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_just_newline() {
        lexer_test(r#""\n""#, vec![StrLit(String::from("\n"))], &[]);
    }

    #[test]
    fn lex_unterminated_string_with_just_newline() {
        lexer_test(r#""\n"#, vec![StrLit(String::from("\n"))], &[
            (3, 3, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_newline() {
        lexer_test(r#""foo\nbar""#, vec![StrLit(String::from("foo\nbar"))], &[]);
    }

    #[test]
    fn lex_unterminated_string_with_newline() {
        lexer_test(r#""foo\nbar"#, vec![StrLit(String::from("foo\nbar"))], &[
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_invalid_escape_string() {
        lexer_test(r#""foo\cbar""#, vec![StrLit(String::from("foocbar"))], &[
            (4, 6, Error, "invalid character escape: 'c'"),
        ]);
    }

    #[test]
    fn lex_unterminated_invalid_escape_string() {
        lexer_test(r#""foo\cbar"#, vec![StrLit(String::from("foocbar"))], &[
            (4, 6, Error, "invalid character escape: 'c'"),
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    fn lexer_test(source: &str,
                  expected: Vec<TokenKind>,
                  expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let mut lexer = Lexer::new(source);

        for expected_token in expected {
            assert_eq!(lexer.next_token().kind, expected_token);
        }

        let extra_tokens: Vec<Token> = lexer.collect();
        assert_eq!(extra_tokens, &[]);

        assert_errors(expected_errors);
    }

    fn assert_errors(expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let reporter = errors();

        {
            let errors = reporter.errors.borrow();

            for (i, &(start, end, level, message)) in expected_errors.iter().enumerate() {
                assert_eq!(errors[i], super::Error {
                    message: String::from(message),
                    level: level,
                    span: Span::new(start, end),
                });
            }

            // Checking that there are no unexpected errors this way will pretty-print the
            // unexpected errors if they exist.
            assert_eq!(&errors[expected_errors.len()..], &[]);
        }

        reporter.reset();
    }
}
<commit_msg>Replace Token::Eof with None (using Option<Token>).<commit_after>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Iterator for Lexer<'src> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return None,
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Some(Token::new(self.token_start, self.position, kind))
    }
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => continue };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),
                                            ErrorLevel::Error,
                                            format!("invalid character escape: {:?}", c));
                            c
                        }
                    });
                }

                '"' => break,
                c => string.push(c),
            }
        }

        string
    }

    // fn report_error<S: Into<String>>(&self, message: S) {
    //     errors().report(Span::new(self.token_start, self.position), message.into());
    // }

    // fn peek(&self) -> Option<char> {
    //     self.current.and_then(|c| self.char_at(self.position + c.len_utf8()))
    // }

    fn char_at(&self, pos: usize) -> Option<char> {
        self.source[pos..].chars().next()
    }

    /// Move to the next char in the source code.
    fn advance(&mut self) {
        if let Some(c) = self.current {
            self.position += c.len_utf8();
            self.current = self.char_at(self.position);
        } else {
            panic!("lexer attempted to advance past the end of the source code");
        }
    }
}

// pub fn tokenize(source: &str) -> Vec<Token> {
//     let mut lexer = Lexer::new(source);
//     vec![]
// }

////////////////////////////////////////////////////////////////////////////////
// Parser
////////////////////////////////////////////////////////////////////////////////

// #[derive(Clone, Debug)]
// struct Parser<'src> {
//     lexer: Lexer<'src>,
//     errors: Vec<Error>,
// }

    // fn parse_module(&mut self) -> ParseResult<ast::Module> {
    //     let mut fns = Vec::new();
    //     while self.token != Token::Eof {
    //         fns.push(try!(self.parse_fn_def()));
    //     }
    //     Ok(ast::Module { fns: fns })
    // }

    // fn parse_fn_def(&mut self) -> ParseResult<ast::FnDef> {
    //     try!(self.expect(&Token::KeywordFn));

    //     let name = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     // TODO(tsion): Parse arguments.

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::BraceL));

    //     let mut body = Vec::new();

    //     while self.token != Token::BraceR {
    //         body.push(try!(self.parse_expr()));
    //     }

    //     // Skip the closing brace.
    //     self.advance();

    //     Ok(ast::FnDef {
    //         name: name,
    //         return_ty: ast::Type::Unit,
    //         args: Vec::new(),
    //         body: body,
    //     })
    // }

    // fn parse_expr(&mut self) -> ParseResult<ast::Expr> {
    //     let func = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     let str = match self.advance_and_get() {
    //         Token::StrLit(str) => str,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::Semicolon));

    //     Ok(ast::Expr::FnCall {
    //         func: func,
    //         args: vec![ast::Expr::StrLit(str)],
    //     })
    // }

////////////////////////////////////////////////////////////////////////////////
// Character classes
////////////////////////////////////////////////////////////////////////////////

/// Returns `true` if the given character is whitespace.
fn is_whitespace(c: char) -> bool {
    match c {
        ' ' | '\t' | '\r' | '\n' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is a digit.
fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid at the start of an identifier.
fn is_ident_start(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '_' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid after the start of an identifier.
fn is_ident_continue(c: char) -> bool {
    is_ident_start(c) || is_digit(c)
}

////////////////////////////////////////////////////////////////////////////////
// Errors
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error {
    pub message: String,
    pub level: ErrorLevel,
    pub span: Span,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ErrorLevel {
    Error,
    Warning,
    Note,
}

pub struct ErrorReporter {
    pub errors: RefCell<Vec<Error>>,
}

impl ErrorReporter {
    fn new() -> Self {
        ErrorReporter { errors: RefCell::new(Vec::new()) }
    }

    fn report<S: Into<String>>(&self, span: Span, level: ErrorLevel, message: S) {
        self.errors.borrow_mut().push(Error {
            message: message.into(),
            level: level,
            span: span,
        });
    }

    fn is_empty(&self) -> bool {
        self.errors.borrow().is_empty()
    }

    fn reset(&self) {
        self.errors.borrow_mut().clear();
    }
}

pub fn errors() -> Rc<ErrorReporter> {
    thread_local!(static KEY: Rc<ErrorReporter> = Rc::new(ErrorReporter::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
mod test {
    use super::{errors, ErrorLevel, Lexer, Span, Token, TokenKind};
    use super::ErrorLevel::*;
    use super::TokenKind::*;

    #[test]
    fn lex_empty_string() {
        lexer_test(r#""""#, vec![StrLit(String::from(""))], &[]);
    }

    #[test]
    fn lex_unterminated_empty_string() {
        lexer_test(r#"""#, vec![StrLit(String::from(""))], &[
            (1, 1, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_empty_string() {
        lexer_test(r#""\"#, vec![StrLit(String::from(""))], &[
            (2, 2, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_string() {
        lexer_test(r#""foo\"#, vec![StrLit(String::from("foo"))], &[
            (5, 5, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string() {
        lexer_test(r#""foobar""#, vec![StrLit(String::from("foobar"))], &[]);
    }

    #[test]
    fn lex_unterminated_string() {
        lexer_test(r#""foobar"#, vec![StrLit(String::from("foobar"))], &[
            (7, 7, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_just_newline() {
        lexer_test(r#""\n""#, vec![StrLit(String::from("\n"))], &[]);
    }

    #[test]
    fn lex_unterminated_string_with_just_newline() {
        lexer_test(r#""\n"#, vec![StrLit(String::from("\n"))], &[
            (3, 3, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_newline() {
        lexer_test(r#""foo\nbar""#, vec![StrLit(String::from("foo\nbar"))], &[]);
    }

    #[test]
    fn lex_unterminated_string_with_newline() {
        lexer_test(r#""foo\nbar"#, vec![StrLit(String::from("foo\nbar"))], &[
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_invalid_escape_string() {
        lexer_test(r#""foo\cbar""#, vec![StrLit(String::from("foocbar"))], &[
            (4, 6, Error, "invalid character escape: 'c'"),
        ]);
    }

    #[test]
    fn lex_unterminated_invalid_escape_string() {
        lexer_test(r#""foo\cbar"#, vec![StrLit(String::from("foocbar"))], &[
            (4, 6, Error, "invalid character escape: 'c'"),
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    fn lexer_test(source: &str,
                  expected: Vec<TokenKind>,
                  expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let mut lexer = Lexer::new(source);

        for expected_token in expected {
            assert_eq!(lexer.next().unwrap().kind, expected_token);
        }

        let extra_tokens: Vec<Token> = lexer.collect();
        assert_eq!(extra_tokens, &[]);

        assert_errors(expected_errors);
    }

    fn assert_errors(expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let reporter = errors();

        {
            let errors = reporter.errors.borrow();

            for (i, &(start, end, level, message)) in expected_errors.iter().enumerate() {
                assert_eq!(errors[i], super::Error {
                    message: String::from(message),
                    level: level,
                    span: Span::new(start, end),
                });
            }

            // Checking that there are no unexpected errors this way will pretty-print the
            // unexpected errors if they exist.
            assert_eq!(&errors[expected_errors.len()..], &[]);
        }

        reporter.reset();
    }
}
<|endoftext|>


<commit_before>use libc;

use ffi;
use error::{YamlError, YamlErrorContext, YamlMark};
use event::YamlEvent;
use document::{YamlDocument};
use codecs;

use std::mem;
use std::io::{IoError, EndOfFile};
use std::c_vec::CVec;

pub struct YamlEventStream<P> {
    parser: Box<P>,
}

impl<P:YamlParser> Iterator<Result<YamlEvent, YamlError>> for YamlEventStream<P> {
    fn next(&mut self) -> Option<Result<YamlEvent, YamlError>> {
        unsafe {
            match self.parser.parse_event() {
                Some(YamlEvent::YamlNoEvent) => None,
                Some(evt) => Some(Ok(evt)),
                None => Some(Err(self.parser.get_error()))
            }
        }
    }
}

pub struct YamlDocumentStream<P> {
    parser: Box<P>,
}

impl<P:YamlParser> Iterator<Result<Box<YamlDocument>, YamlError>> for YamlDocumentStream<P> {
    fn next(&mut self) -> Option<Result<Box<YamlDocument>, YamlError>> {
        unsafe {
            match YamlDocument::parser_load(&mut self.parser.base_parser_ref().parser_mem) {
                Some(doc) => if doc.is_empty() {
                    None
                } else {
                    Some(Ok(doc))
                },
                None => Some(Err(self.parser.get_error()))
            }
        }
    }
}

pub struct InternalEvent {
    event_mem: ffi::yaml_event_t
}

impl Drop for InternalEvent {
    fn drop(&mut self) {
        unsafe {
            self.event_mem.delete()
        }
    }
}

pub trait YamlParser {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser;
    unsafe fn get_error(&mut self) -> YamlError;

    unsafe fn parse_event(&mut self) -> Option<YamlEvent> {
        let mut event = InternalEvent {
            event_mem: mem::uninitialized()
        };

        if !self.base_parser_ref().parse(&mut event.event_mem) {
            None
        } else {
            Some(YamlEvent::load(&event.event_mem))
        }
    }

    fn parse(self: Box<Self>) -> YamlEventStream<Self> {
        YamlEventStream {
            parser: self,
        }
    }

    fn load(self: Box<Self>) -> YamlDocumentStream<Self> {
        YamlDocumentStream {
            parser: self,
        }
    }
}

extern fn handle_reader_cb(data: *mut YamlIoParser, buffer: *mut u8, size: libc::size_t, size_read: *mut libc::size_t) -> libc::c_int {
    unsafe {
        let mut buf = CVec::new(buffer, size as uint);
        let parser = &mut *data;
        match parser.reader.read(buf.as_mut_slice()) {
            Ok(size) => {
                *size_read = size as libc::size_t;
                return 1;
            },
            Err(err) => {
                match err.kind {
                    EndOfFile => {
                        *size_read = 0;
                        return 1;
                    },
                    _ => {
                        parser.io_error = Some(err);
                        return 0;
                    }
                }
            }
        }
    }
}

pub struct YamlBaseParser {
    parser_mem: ffi::yaml_parser_t,
}

impl YamlBaseParser {
    unsafe fn new() -> YamlBaseParser {
        YamlBaseParser {
            parser_mem: mem::uninitialized()
        }
    }

    unsafe fn initialize(&mut self) -> bool {
        ffi::yaml_parser_initialize(&mut self.parser_mem) != 0
    }

    unsafe fn set_input_string(&mut self, input: *const u8, size: uint) {
        ffi::yaml_parser_set_input_string(&mut self.parser_mem, input, size as libc::size_t);
    }

    unsafe fn parse(&mut self, event: &mut ffi::yaml_event_t) -> bool {
        ffi::yaml_parser_parse(&mut self.parser_mem, event) != 0
    }

    unsafe fn build_error(&self) -> YamlError {
        let context = YamlErrorContext {
            byte_offset: self.parser_mem.problem_offset as uint,
            problem_mark: YamlMark::conv(&self.parser_mem.problem_mark),
            context: codecs::decode_c_str(self.parser_mem.context as *const ffi::yaml_char_t),
            context_mark: YamlMark::conv(&self.parser_mem.context_mark),
        };

        YamlError {
            kind: self.parser_mem.error,
            problem: codecs::decode_c_str(self.parser_mem.problem as *const ffi::yaml_char_t),
            io_error: None,
            context: Some(context)
        }
    }
}

impl Drop for YamlBaseParser {
    fn drop(&mut self) {
        unsafe {
            ffi::yaml_parser_delete(&mut self.parser_mem);
        }
    }
}

pub struct YamlByteParser<'r> {
    base_parser: YamlBaseParser
}

impl<'r> YamlParser for YamlByteParser<'r> {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser {
        &mut self.base_parser
    }

    unsafe fn get_error(&mut self) -> YamlError {
        self.base_parser.build_error()
    }
}

impl<'r> YamlByteParser<'r> {
    pub fn init(bytes: &'r [u8], encoding: ffi::YamlEncoding) -> Box<YamlByteParser<'r>> {
        unsafe {
            let mut parser = box YamlByteParser {
                base_parser: YamlBaseParser::new()
            };

            if !parser.base_parser.initialize() {
                panic!("failed to initialize yaml_parser_t");
            }

            ffi::yaml_parser_set_encoding(&mut parser.base_parser.parser_mem, encoding);
            parser.base_parser.set_input_string(bytes.as_ptr(), bytes.len());

            parser
        }
    }
}

pub struct YamlIoParser<'r> {
    base_parser: YamlBaseParser,
    reader: &'r mut (Reader+'r),
    io_error: Option<IoError>,
}

impl<'r> YamlParser for YamlIoParser<'r> {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser {
        &mut self.base_parser
    }

    unsafe fn get_error(&mut self) -> YamlError {
        let mut error = self.base_parser.build_error();
        mem::swap(&mut (error.io_error), &mut (self.io_error));
        return error;
    }
}

impl<'r> YamlIoParser<'r> {
    pub fn init<'r>(reader: &'r mut Reader, encoding: ffi::YamlEncoding) -> Box<YamlIoParser<'r>> {
        unsafe {
            let mut parser = box YamlIoParser {
                base_parser: YamlBaseParser::new(),
                reader: reader,
                io_error: None
            };

            if !parser.base_parser.initialize() {
                panic!("failed to initialize yaml_parser_t");
            }

            ffi::yaml_parser_set_encoding(&mut parser.base_parser.parser_mem, encoding);

            ffi::yaml_parser_set_input(&mut parser.base_parser.parser_mem, handle_reader_cb, mem::transmute(&mut *parser));

            parser
        }
    }
}

#[cfg(test)]
mod test {
    use event::{YamlEvent, YamlSequenceParam, YamlScalarParam};
    use event::YamlEvent::*;
    use document::{YamlDocument, YamlNode};
    use parser;
    use parser::YamlParser;
    use error::YamlError;
    use ffi::YamlErrorType;
    use ffi::YamlEncoding::*;
    use ffi::YamlScalarStyle::*;
    use ffi::YamlSequenceStyle::*;
    use std::io;

    #[test]
    fn test_byte_parser() {
        let data = "[1, 2, 3]";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlSequenceStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "3".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlSequenceEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_io_parser() {
        let data = "[1, 2, 3]";
        let mut reader = io::BufReader::new(data.as_bytes());
        let parser = parser::YamlIoParser::init(&mut reader, YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlSequenceStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "3".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlSequenceEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_byte_parser_mapping() {
        let data = "{\"a\": 1, \"b\":2}";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlMappingStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "a".to_string(), plain_implicit: false, quoted_implicit: true, style: YamlDoubleQuotedScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "b".to_string(), plain_implicit: false, quoted_implicit: true, style: YamlDoubleQuotedScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlMappingEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_parser_error() {
        let data = "\"ab";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let mut stream = parser.parse();

        let stream_start = stream.next();
        assert_eq!(Some(Ok(YamlStreamStartEvent(YamlUtf8Encoding))), stream_start);

        let stream_err = stream.next();
        match stream_err {
            Some(Err(err)) => assert_eq!(YamlErrorType::YAML_SCANNER_ERROR, err.kind),
            evt => panic!("unexpected result: {}", evt),
        }
    }

    #[test]
    fn test_document() {
        let data = "[1, 2, 3]";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let docs_res:Result<Vec<Box<YamlDocument>>, YamlError> = parser.load().collect();

        match docs_res {
            Err(e) => panic!("unexpected result: {}", e),
            Ok(docs) => match docs.as_slice().head().and_then(|doc| doc.root()) {
                Some(YamlNode::YamlSequenceNode(seq)) => {
                    let values = seq.values().map(|node| {
                        match node {
                            YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                            _ => panic!("unexpected scalar")
                        }
                    }).collect();
                    assert_eq!(vec!["1".to_string(), "2".to_string(), "3".to_string()], values)
                },
                _ => panic!("unexpected result")
            }
        }
    }

    #[test]
    fn test_mapping_document() {
        let data = "{\"a\": 1, \"b\": 2}";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let docs_res:Result<Vec<Box<YamlDocument>>, YamlError> = parser.load().collect();

        match docs_res {
            Err(e) => panic!("unexpected result: {}", e),
            Ok(docs) => match docs.as_slice().head().and_then(|doc| doc.root()) {
                Some(YamlNode::YamlMappingNode(seq)) => {
                    let values = seq.pairs().map(|(key, value)| {
                        (
                            match key {
                                YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                                _ => panic!("unexpected scalar")
                            },
                            match value {
                                YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                                _ => panic!("unexpected scalar")
                            }
                        )
                    }).collect();
                    assert_eq!(vec![("a".to_string(), "1".to_string()), ("b".to_string(), "2".to_string())], values)
                },
                _ => panic!("unexpected result")
            }
        }
    }
}
<commit_msg>Type checker help...<commit_after>use libc;

use ffi;
use error::{YamlError, YamlErrorContext, YamlMark};
use event::YamlEvent;
use document::{YamlDocument};
use codecs;

use std::mem;
use std::io::{IoError, EndOfFile};
use std::c_vec::CVec;

pub struct YamlEventStream<P> {
    parser: Box<P>,
}

impl<P:YamlParser> Iterator<Result<YamlEvent, YamlError>> for YamlEventStream<P> {
    fn next(&mut self) -> Option<Result<YamlEvent, YamlError>> {
        unsafe {
            match self.parser.parse_event() {
                Some(YamlEvent::YamlNoEvent) => None,
                Some(evt) => Some(Ok(evt)),
                None => Some(Err(self.parser.get_error()))
            }
        }
    }
}

pub struct YamlDocumentStream<P> {
    parser: Box<P>,
}

impl<P:YamlParser> Iterator<Result<Box<YamlDocument>, YamlError>> for YamlDocumentStream<P> {
    fn next(&mut self) -> Option<Result<Box<YamlDocument>, YamlError>> {
        unsafe {
            match YamlDocument::parser_load(&mut self.parser.base_parser_ref().parser_mem) {
                Some(doc) => if doc.is_empty() {
                    None
                } else {
                    Some(Ok(doc))
                },
                None => Some(Err(self.parser.get_error()))
            }
        }
    }
}

pub struct InternalEvent {
    event_mem: ffi::yaml_event_t
}

impl Drop for InternalEvent {
    fn drop(&mut self) {
        unsafe {
            self.event_mem.delete()
        }
    }
}

pub trait YamlParser {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser;
    unsafe fn get_error(&mut self) -> YamlError;

    unsafe fn parse_event(&mut self) -> Option<YamlEvent> {
        let mut event = InternalEvent {
            event_mem: mem::uninitialized()
        };

        if !self.base_parser_ref().parse(&mut event.event_mem) {
            None
        } else {
            Some(YamlEvent::load(&event.event_mem))
        }
    }

    fn parse(self: Box<Self>) -> YamlEventStream<Self> {
        YamlEventStream {
            parser: self,
        }
    }

    fn load(self: Box<Self>) -> YamlDocumentStream<Self> {
        YamlDocumentStream {
            parser: self,
        }
    }
}

extern fn handle_reader_cb(data: *mut YamlIoParser, buffer: *mut u8, size: libc::size_t, size_read: *mut libc::size_t) -> libc::c_int {
    unsafe {
        let mut buf = CVec::new(buffer, size as uint);
        let parser = &mut *data;
        match parser.reader.read(buf.as_mut_slice()) {
            Ok(size) => {
                *size_read = size as libc::size_t;
                return 1;
            },
            Err(err) => {
                match err.kind {
                    EndOfFile => {
                        *size_read = 0;
                        return 1;
                    },
                    _ => {
                        parser.io_error = Some(err);
                        return 0;
                    }
                }
            }
        }
    }
}

pub struct YamlBaseParser {
    parser_mem: ffi::yaml_parser_t,
}

impl YamlBaseParser {
    unsafe fn new() -> YamlBaseParser {
        YamlBaseParser {
            parser_mem: mem::uninitialized()
        }
    }

    unsafe fn initialize(&mut self) -> bool {
        ffi::yaml_parser_initialize(&mut self.parser_mem) != 0
    }

    unsafe fn set_input_string(&mut self, input: *const u8, size: uint) {
        ffi::yaml_parser_set_input_string(&mut self.parser_mem, input, size as libc::size_t);
    }

    unsafe fn parse(&mut self, event: &mut ffi::yaml_event_t) -> bool {
        ffi::yaml_parser_parse(&mut self.parser_mem, event) != 0
    }

    unsafe fn build_error(&self) -> YamlError {
        let context = YamlErrorContext {
            byte_offset: self.parser_mem.problem_offset as uint,
            problem_mark: YamlMark::conv(&self.parser_mem.problem_mark),
            context: codecs::decode_c_str(self.parser_mem.context as *const ffi::yaml_char_t),
            context_mark: YamlMark::conv(&self.parser_mem.context_mark),
        };

        YamlError {
            kind: self.parser_mem.error,
            problem: codecs::decode_c_str(self.parser_mem.problem as *const ffi::yaml_char_t),
            io_error: None,
            context: Some(context)
        }
    }
}

impl Drop for YamlBaseParser {
    fn drop(&mut self) {
        unsafe {
            ffi::yaml_parser_delete(&mut self.parser_mem);
        }
    }
}

pub struct YamlByteParser<'r> {
    base_parser: YamlBaseParser
}

impl<'r> YamlParser for YamlByteParser<'r> {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser {
        &mut self.base_parser
    }

    unsafe fn get_error(&mut self) -> YamlError {
        self.base_parser.build_error()
    }
}

impl<'r> YamlByteParser<'r> {
    pub fn init(bytes: &'r [u8], encoding: ffi::YamlEncoding) -> Box<YamlByteParser<'r>> {
        unsafe {
            let mut parser = box YamlByteParser {
                base_parser: YamlBaseParser::new()
            };

            if !parser.base_parser.initialize() {
                panic!("failed to initialize yaml_parser_t");
            }

            ffi::yaml_parser_set_encoding(&mut parser.base_parser.parser_mem, encoding);
            parser.base_parser.set_input_string(bytes.as_ptr(), bytes.len());

            parser
        }
    }
}

pub struct YamlIoParser<'r> {
    base_parser: YamlBaseParser,
    reader: &'r mut (Reader+'r),
    io_error: Option<IoError>,
}

impl<'r> YamlParser for YamlIoParser<'r> {
    unsafe fn base_parser_ref<'r>(&'r mut self) -> &'r mut YamlBaseParser {
        &mut self.base_parser
    }

    unsafe fn get_error(&mut self) -> YamlError {
        let mut error = self.base_parser.build_error();
        mem::swap(&mut (error.io_error), &mut (self.io_error));
        return error;
    }
}

impl<'r> YamlIoParser<'r> {
    pub fn init<'r>(reader: &'r mut Reader, encoding: ffi::YamlEncoding) -> Box<YamlIoParser<'r>> {
        unsafe {
            let mut parser = box YamlIoParser {
                base_parser: YamlBaseParser::new(),
                reader: reader,
                io_error: None
            };

            if !parser.base_parser.initialize() {
                panic!("failed to initialize yaml_parser_t");
            }

            ffi::yaml_parser_set_encoding(&mut parser.base_parser.parser_mem, encoding);

            ffi::yaml_parser_set_input(&mut parser.base_parser.parser_mem, handle_reader_cb, mem::transmute(&mut *parser));

            parser
        }
    }
}

#[cfg(test)]
mod test {
    use event::{YamlEvent, YamlSequenceParam, YamlScalarParam};
    use event::YamlEvent::*;
    use document::{YamlDocument, YamlNode};
    use parser;
    use parser::YamlParser;
    use error::YamlError;
    use ffi::YamlErrorType;
    use ffi::YamlEncoding::*;
    use ffi::YamlScalarStyle::*;
    use ffi::YamlSequenceStyle::*;
    use std::io;

    #[test]
    fn test_byte_parser() {
        let data = "[1, 2, 3]";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlSequenceStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "3".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlSequenceEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_io_parser() {
        let data = "[1, 2, 3]";
        let mut reader = io::BufReader::new(data.as_bytes());
        let parser = parser::YamlIoParser::init(&mut reader, YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlSequenceStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "3".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlSequenceEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_byte_parser_mapping() {
        let data = "{\"a\": 1, \"b\":2}";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let expected = Ok(vec![
            YamlStreamStartEvent(YamlUtf8Encoding),
            YamlDocumentStartEvent(None, vec![], true),
            YamlMappingStartEvent(YamlSequenceParam{anchor: None, tag: None, implicit: true, style: YamlFlowSequenceStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "a".to_string(), plain_implicit: false, quoted_implicit: true, style: YamlDoubleQuotedScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "1".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "b".to_string(), plain_implicit: false, quoted_implicit: true, style: YamlDoubleQuotedScalarStyle}),
            YamlScalarEvent(YamlScalarParam{anchor: None, tag: None, value: "2".to_string(), plain_implicit: true, quoted_implicit: false, style: YamlPlainScalarStyle}),
            YamlMappingEndEvent,
            YamlDocumentEndEvent(true),
            YamlStreamEndEvent
        ]);

        let stream: Result<Vec<YamlEvent>, YamlError> = parser.parse().collect();

        assert_eq!(expected, stream);
    }

    #[test]
    fn test_parser_error() {
        let data = "\"ab";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let mut stream = parser.parse();

        let stream_start = stream.next();
        assert_eq!(Some(Ok(YamlStreamStartEvent(YamlUtf8Encoding))), stream_start);

        let stream_err = stream.next();
        match stream_err {
            Some(Err(err)) => assert_eq!(YamlErrorType::YAML_SCANNER_ERROR, err.kind),
            evt => panic!("unexpected result: {}", evt),
        }
    }

    #[test]
    fn test_document() {
        let data = "[1, 2, 3]";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let docs_res:Result<Vec<Box<YamlDocument>>, YamlError> = parser.load().collect();

        match docs_res {
            Err(e) => panic!("unexpected result: {}", e),
            Ok(docs) => match docs.as_slice().head().and_then(|doc| doc.root()) {
                Some(YamlNode::YamlSequenceNode(seq)) => {
                    let values:Vec<String> = seq.values().map(|node| {
                        match node {
                            YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                            _ => panic!("unexpected scalar")
                        }
                    }).collect();
                    assert_eq!(vec!["1".to_string(), "2".to_string(), "3".to_string()], values)
                },
                _ => panic!("unexpected result")
            }
        }
    }

    #[test]
    fn test_mapping_document() {
        let data = "{\"a\": 1, \"b\": 2}";
        let parser = parser::YamlByteParser::init(data.as_bytes(), YamlUtf8Encoding);
        let docs_res:Result<Vec<Box<YamlDocument>>, YamlError> = parser.load().collect();

        match docs_res {
            Err(e) => panic!("unexpected result: {}", e),
            Ok(docs) => match docs.as_slice().head().and_then(|doc| doc.root()) {
                Some(YamlNode::YamlMappingNode(seq)) => {
                    let values:Vec<(String, String)> = seq.pairs().map(|(key, value)| {
                        (
                            match key {
                                YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                                _ => panic!("unexpected scalar")
                            },
                            match value {
                                YamlNode::YamlScalarNode(scalar) => scalar.get_value(),
                                _ => panic!("unexpected scalar")
                            }
                        )
                    }).collect();
                    assert_eq!(vec![("a".to_string(), "1".to_string()), ("b".to_string(), "2".to_string())], values)
                },
                _ => panic!("unexpected result")
            }
        }
    }
}
<|endoftext|>


<commit_before>use codemap::Spanned;
use lexer::Token;
use std::iter::Peekable;
use std::slice::Iter;
use self::ParseErrorKind::*;

type ParseIter<'a> = Peekable<Iter<'a, Spanned<Token>>>;
type Parse = Result<Node, ParseErrorKind>;

#[derive(Debug, Clone, PartialEq)]
pub enum Node {
  Block(Vec<Node>),
  If{cond: Box<Node>},
  Stmt(Box<Node>),
  Break,
  Continue,
  Expr,
  Pass,
  Index {
    lhs: Box<Node>,
    rhs: Box<Node>,
  },

  Method {
    owner: Box<Node>,
    meth: Box<Node>,
    args: Vec<Node>,
  },

  Func {
    func: Box<Node>,
    args: Vec<Node>,
  },

  BinExpr {
    lhs: Box<Node>,
    op: Token,
    rhs: Box<Node>,
  },

  UnExpr {
    val: Box<Node>,
    op: Token,
  },

  // Literals
  Null,
  Bool(bool),
  Float(f64),
  Int(i64),
  Str(String),
  Name(String),
  Table,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Op {
  Right(u32),
  Left(u32),
  None,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ParseErrorKind {
  UnexpectedToken(Token),
  UnexpectedEOF,
  UnknownBinaryOperator,
  UnknownUnaryOperator,
}

// Return true if the next token in `it` is `kind`
fn peek_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&tok) = it.peek() {
    tok.node == kind
  }
  else {
    false
  }
}

// Return true if the next token in `it` is `kind` *and* consume the token
fn use_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&tok) = it.peek() {
    if tok.node == kind {
      it.next();
    }
    tok.node == kind
  }
  else {
    false
  }
}

// Panic if the next token in `it` is *not* `kind`
fn require_token(it: &mut ParseIter, kind: Token) -> Result<(), ParseErrorKind> {
  if let Some(&tok) = it.peek() {
    it.next();

    if tok.node == kind {
      return Ok(());
    }

    return Err(UnexpectedToken(tok.node.clone()));
  }

  return Err(UnexpectedEOF);
}

fn op_precedence(op: &Token) -> Op {
  match *op {
    Token::Add | Token::Sub => Op::Left(10),
    Token::Div | Token::Mul => Op::Left(20),
    Token::Car => Op::Right(30),
    _ => Op::None,
  }
}

fn parse_bin_expr(it: &mut ParseIter) -> Parse {
  let mut expr = parse_unexpr(it)?;

  // prevents this from breaking the LHS until we know we made it
  // otherwise, things like (2 + 3) * 4 get restructured into 2 + (3 * 4)
  let mut break_left = false;

  while let Some(&tok) = it.peek() {
    let prec = op_precedence(&tok.node);

    if let Op::None = prec {
      break;
    }

    it.next();

    let rhs = parse_unexpr(it)?;

    expr = match (break_left, expr.clone()) {
      (true, Node::BinExpr{lhs: cur_lhs, op: cur_op, rhs: cur_rhs}) => {
        let cur_prec = op_precedence(&cur_op);
        match (cur_prec, prec) {
          // these should never happen
          (_, Op::None) => {break}
          (Op::None, _) => {break}

          // left-to-right
          // there has to be a better way to handle this, no?
          (Op::Left(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Right(m)) if n > m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Left(n), Op::Right(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }

          // right-to-left
          _ => {
            Node::BinExpr {
              lhs: cur_lhs,
              op: cur_op,
              rhs: Box::new(Node::BinExpr {lhs: cur_rhs, op: tok.node.clone(), rhs: Box::new(rhs)}),
            }
          }
        }
      }
      _ => {
        Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
      }
    };

    break_left = true;
  };

  Ok(expr)
}

fn parse_unexpr(it: &mut ParseIter) -> Parse {
  parse_simple(it)
}

fn parse_simple(it: &mut ParseIter) -> Parse {
  parse_atom(it)
}

fn parse_atom(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Pal => {
        it.next();
        let out = parse_bin_expr(it)?;
        require_token(it, Token::Par)?;
        Ok(out)
      }
      _ => parse_quark(it),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_name_as_str(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Name(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_name(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_quark(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Null => {it.next(); Ok(Node::Null)},
      Token::Bool(x) => {it.next(); Ok(Node::Bool(x))},
      Token::Float(x) => {it.next(); Ok(Node::Float(x))},
      Token::Int(x) => {it.next(); Ok(Node::Int(x))},
      Token::Str(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      Token::Table => {it.next(); Ok(Node::Table)},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_stmt(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Break => {
        it.next();
        Ok(Node::Break)
      }

      Token::Continue => {
        it.next();
        Ok(Node::Continue)
      }

      Token::Pass => {
        it.next();
        Ok(Node::Pass)
      }

      _ => {
        parse_bin_expr(it).map(|expr| Node::Stmt(Box::new(expr)))
      }
    }
  }

  Err(UnexpectedEOF)
}

pub fn parse(tokens: Vec<Spanned<Token>>) -> Parse {
  let mut it: ParseIter = tokens.iter().peekable();
  let mut nodes: Vec<Node> = vec![];

  while !peek_token(&mut it, Token::EOF) {
    nodes.push(parse_stmt(&mut it)?);
    require_token(&mut it, Token::End)?;
  }

  Ok(Node::Block(nodes))
}
<commit_msg>parser: Add parse_simple()<commit_after>use codemap::Spanned;
use lexer::Token;
use std::iter::Peekable;
use std::slice::Iter;
use self::ParseErrorKind::*;

type ParseIter<'a> = Peekable<Iter<'a, Spanned<Token>>>;
type Parse = Result<Node, ParseErrorKind>;

#[derive(Debug, Clone, PartialEq)]
pub enum Node {
  Block(Vec<Node>),
  If{cond: Box<Node>},
  Stmt(Box<Node>),
  Break,
  Continue,
  Expr,
  Pass,
  Index {
    lhs: Box<Node>,
    rhs: Box<Node>,
  },

  Method {
    owner: Box<Node>,
    meth: Box<Node>,
    args: Vec<Node>,
  },

  Func {
    func: Box<Node>,
    args: Vec<Node>,
  },

  BinExpr {
    lhs: Box<Node>,
    op: Token,
    rhs: Box<Node>,
  },

  UnExpr {
    val: Box<Node>,
    op: Token,
  },

  // Literals
  Null,
  Bool(bool),
  Float(f64),
  Int(i64),
  Str(String),
  Name(String),
  Table,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Op {
  Right(u32),
  Left(u32),
  None,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ParseErrorKind {
  UnexpectedToken(Token),
  UnexpectedEOF,
  UnknownBinaryOperator,
  UnknownUnaryOperator,
}

// Return true if the next token in `it` is `kind`
fn peek_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&tok) = it.peek() {
    tok.node == kind
  }
  else {
    false
  }
}

// Return true if the next token in `it` is `kind` *and* consume the token
fn use_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&tok) = it.peek() {
    if tok.node == kind {
      it.next();
    }
    tok.node == kind
  }
  else {
    false
  }
}

// Panic if the next token in `it` is *not* `kind`
fn require_token(it: &mut ParseIter, kind: Token) -> Result<(), ParseErrorKind> {
  if let Some(&tok) = it.peek() {
    it.next();

    if tok.node == kind {
      return Ok(());
    }

    return Err(UnexpectedToken(tok.node.clone()));
  }

  return Err(UnexpectedEOF);
}

fn op_precedence(op: &Token) -> Op {
  match *op {
    Token::Add | Token::Sub => Op::Left(10),
    Token::Div | Token::Mul => Op::Left(20),
    Token::Car => Op::Right(30),
    _ => Op::None,
  }
}

fn parse_bin_expr(it: &mut ParseIter) -> Parse {
  let mut expr = parse_unexpr(it)?;

  // prevents this from breaking the LHS until we know we made it
  // otherwise, things like (2 + 3) * 4 get restructured into 2 + (3 * 4)
  let mut break_left = false;

  while let Some(&tok) = it.peek() {
    let prec = op_precedence(&tok.node);

    if let Op::None = prec {
      break;
    }

    it.next();

    let rhs = parse_unexpr(it)?;

    expr = match (break_left, expr.clone()) {
      (true, Node::BinExpr{lhs: cur_lhs, op: cur_op, rhs: cur_rhs}) => {
        let cur_prec = op_precedence(&cur_op);
        match (cur_prec, prec) {
          // these should never happen
          (_, Op::None) => {break}
          (Op::None, _) => {break}

          // left-to-right
          // there has to be a better way to handle this, no?
          (Op::Left(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Right(m)) if n > m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Left(n), Op::Right(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
          }

          // right-to-left
          _ => {
            Node::BinExpr {
              lhs: cur_lhs,
              op: cur_op,
              rhs: Box::new(Node::BinExpr {lhs: cur_rhs, op: tok.node.clone(), rhs: Box::new(rhs)}),
            }
          }
        }
      }
      _ => {
        Node::BinExpr {lhs: Box::new(expr), op: tok.node.clone(), rhs: Box::new(rhs)}
      }
    };

    break_left = true;
  };

  Ok(expr)
}

fn parse_unexpr(it: &mut ParseIter) -> Parse {
  parse_simple(it)
}

/* unused, here for reference
fn parse_index(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Sql => {
        it.next();
        let idx = parse_bin_expr(it)?;
        require_token(it, Token::Sqr)?;
        Ok(idx)
      }
      Token::Dot => {
        it.next();
        parse_name_as_str(it)
      }
      _ => {
        Err(UnexpectedToken(c.node.clone()))
      }
    };
  }

  Err(UnexpectedEOF)
}
*/

fn parse_fn_args(it: &mut ParseIter) -> Result<Vec<Node>, ParseErrorKind> {
  require_token(it, Token::Pal)?;
  // FIXME
  require_token(it, Token::Par)?;
  Ok(Vec::new())
}

fn parse_simple(it: &mut ParseIter) -> Parse {
  let mut atom = parse_atom(it)?;
  while let Some(&tok) = it.peek() {
    match tok.node {
      Token::Col => {
        it.next();
        let meth = parse_name_as_str(it)?;
        let args = parse_fn_args(it)?;
        atom = Node::Method {
          owner: Box::new(atom),
          meth: Box::new(meth),
          args: args,
        };
      }

      Token::Pal => {
        let args = parse_fn_args(it)?;
        atom = Node::Func {
          func: Box::new(atom),
          args: args,
        };
      }

      Token::Sql => {
        it.next();
        let idx = parse_bin_expr(it)?;
        require_token(it, Token::Sqr)?;
        atom = Node::Index {
          lhs: Box::new(atom),
          rhs: Box::new(idx),
        };
      }

      Token::Dot => {
        it.next();
        let idx = parse_name_as_str(it)?;
        atom = Node::Index {
          lhs: Box::new(atom),
          rhs: Box::new(idx),
        };
      }

      _ => {
        break
      }
    }
  }

  Ok(atom)
}

fn parse_atom(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Pal => {
        it.next();
        let out = parse_bin_expr(it)?;
        require_token(it, Token::Par)?;
        Ok(out)
      }
      _ => parse_quark(it),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_name_as_str(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Name(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_name(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_quark(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Null => {it.next(); Ok(Node::Null)},
      Token::Bool(x) => {it.next(); Ok(Node::Bool(x))},
      Token::Float(x) => {it.next(); Ok(Node::Float(x))},
      Token::Int(x) => {it.next(); Ok(Node::Int(x))},
      Token::Str(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      Token::Table => {it.next(); Ok(Node::Table)},
      ref x => Err(UnexpectedToken(x.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_stmt(it: &mut ParseIter) -> Parse {
  if let Some(&tok) = it.peek() {
    return match tok.node {
      Token::Break => {
        it.next();
        Ok(Node::Break)
      }

      Token::Continue => {
        it.next();
        Ok(Node::Continue)
      }

      Token::Pass => {
        it.next();
        Ok(Node::Pass)
      }

      _ => {
        parse_bin_expr(it).map(|expr| Node::Stmt(Box::new(expr)))
      }
    }
  }

  Err(UnexpectedEOF)
}

pub fn parse(tokens: Vec<Spanned<Token>>) -> Parse {
  let mut it: ParseIter = tokens.iter().peekable();
  let mut nodes: Vec<Node> = vec![];

  while !peek_token(&mut it, Token::EOF) {
    nodes.push(parse_stmt(&mut it)?);
    require_token(&mut it, Token::End)?;
  }

  Ok(Node::Block(nodes))
}
<|endoftext|>