<commit_before>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnconsumedTokens,
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
}

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Expr, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    if let Some(result) = parse_expression(&mut iter) {
        match result {
            Ok(expr) => {
                if let None = iter.next() {
                    Ok(expr)
                } else {
                    Err(vec![ParseError::UnconsumedTokens])
                }
            }
            Err(error) => {
                {
                    synchronise(&mut iter);
                }
                Err(vec![error])
            }
        }

    } else {
        Err(vec![ParseError::UnexpectedEndOfFile])
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum TokenKind {
        StartOfConstruct,
        BodyOfConstruct,
        EndOfConstruct,
    };
    fn classify(token: &Token) -> TokenKind {
        match token {
            &Token::Semicolon => TokenKind::EndOfConstruct,
            &Token::Class => TokenKind::StartOfConstruct,
            &Token::Fun => TokenKind::StartOfConstruct,
            &Token::Var => TokenKind::StartOfConstruct,
            &Token::For => TokenKind::StartOfConstruct,
            &Token::If => TokenKind::StartOfConstruct,
            &Token::While => TokenKind::StartOfConstruct,
            &Token::Print => TokenKind::StartOfConstruct,
            &Token::Return => TokenKind::StartOfConstruct,
            _ => TokenKind::BodyOfConstruct,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            TokenKind::StartOfConstruct => {
                break;
            }
            TokenKind::BodyOfConstruct => {
                let _ = tokens.next();
            }
            TokenKind::EndOfConstruct => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right;
        {
            if let Some(result) = parse_subexpression(tokens) {
                right = try_wrap_err!(result);
            } else {
                return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                                 operator.position)));
            }
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse(tokens).unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(tokens).is_err());
    }
}
<commit_msg>[clippy] unnecessary operation<commit_after>use ast::*;
use scanner::{Lexeme, Position, Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub enum ParseError {
    UnconsumedTokens,
    UnexpectedEndOfFile,
    UnexpectedToken(Lexeme, Position),
    MissingSubexpression(Lexeme, Position),
    MissingClosingParen(Lexeme, Position),
}

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Expr, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    if let Some(result) = parse_expression(&mut iter) {
        match result {
            Ok(expr) => {
                if let None = iter.next() {
                    Ok(expr)
                } else {
                    Err(vec![ParseError::UnconsumedTokens])
                }
            }
            Err(error) => {
                {
                    synchronise(&mut iter);
                }
                Err(vec![error])
            }
        }

    } else {
        Err(vec![ParseError::UnexpectedEndOfFile])
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum TokenKind {
        StartOfConstruct,
        BodyOfConstruct,
        EndOfConstruct,
    };
    fn classify(token: &Token) -> TokenKind {
        match token {
            &Token::Semicolon => TokenKind::EndOfConstruct,
            &Token::Class => TokenKind::StartOfConstruct,
            &Token::Fun => TokenKind::StartOfConstruct,
            &Token::Var => TokenKind::StartOfConstruct,
            &Token::For => TokenKind::StartOfConstruct,
            &Token::If => TokenKind::StartOfConstruct,
            &Token::While => TokenKind::StartOfConstruct,
            &Token::Print => TokenKind::StartOfConstruct,
            &Token::Return => TokenKind::StartOfConstruct,
            _ => TokenKind::BodyOfConstruct,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            TokenKind::StartOfConstruct => {
                break;
            }
            TokenKind::BodyOfConstruct => {
                let _ = tokens.next();
            }
            TokenKind::EndOfConstruct => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<BinaryOperator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_subexpression(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::BangEqual => Some(BinaryOperator::NotEqual),
            Token::EqualEqual => Some(BinaryOperator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Greater => Some(BinaryOperator::Greater),
            Token::GreaterEqual => Some(BinaryOperator::GreaterEqual),
            Token::Less => Some(BinaryOperator::Less),
            Token::LessEqual => Some(BinaryOperator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Minus => Some(BinaryOperator::Minus),
            Token::Plus => Some(BinaryOperator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<BinaryOperator> {
        match *token {
            Token::Slash => Some(BinaryOperator::Slash),
            Token::Star => Some(BinaryOperator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<UnaryOperator> {
        match *token {
            Token::Minus => Some(UnaryOperator::Minus),
            Token::Bang => Some(UnaryOperator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right = if let Some(result) = parse_unary(tokens) {
            try_wrap_err!(result)
        } else {
            return Some(Err(ParseError::MissingSubexpression(operator.lexeme.clone(),
                                                             operator.position)));
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr = if let Some(result) = parse_expression(tokens) {
                    try_wrap_err!(result)
                } else {
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::MissingClosingParen(token.lexeme.clone(),
                                                                            token.position)));
                        }

                    }
                    return Some(Err(ParseError::UnexpectedEndOfFile));
                }
            }
            _ => {
                return Some(Err(ParseError::UnexpectedToken(primary_token.lexeme.clone(),
                                                            primary_token.position)));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let string = String::from("123");
        let (tokens, _) = scan(&string);
        let expr = parse(tokens).unwrap();
        assert_eq!(&string, &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789");
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2");
        assert!(parse(tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)");
        assert!(parse(tokens).is_err());
    }
}
<|endoftext|>


<commit_before>
use pair::Pair;
use token::{Token, TokenKind};
use error::{NcclError, ErrorKind};

use std::error::Error;

#[derive(Debug)]
pub struct Parser {
    current: usize,
    path: Vec<String>,
    indent: usize,
    tokens: Vec<Token>,
    pair: Pair,
    line: u64,
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: Pair::new("__top_level__"),
            line: 1,
        }
    }

    pub fn new_with(tokens: Vec<Token>, pair: Pair) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: pair,
            line: 1
        }
    }

    // faked you out with that Scanner, didn't I?
    // you thought this was going to be recursive descent. YOU WERE WRONG!
    pub fn parse(mut self) -> Result<Pair, Vec<Box<Error>>> {
        let mut errors: Vec<Box<Error>> = vec![];
        let mut prev_indent = 0;
        let mut i = 0;

        while i < self.tokens.len() {
            match self.tokens[i].kind {
                TokenKind::Value => { // add to path respective of self.index
                    if self.indent <= self.path.len() {
                        let mut new = self.path[0..self.indent].to_owned();
                        new.push(self.tokens[i].lexeme.clone());
                        self.path = new;
                    } else {
                        self.path.push(self.tokens[i].lexeme.clone());
                    }

                    self.pair.add_slice(&self.path);

                    if i + 2 <= self.tokens.len() && self.tokens[i + 2].kind == TokenKind::Value {
                        self.path.clear();
                        self.indent = 0;
                    }
                },

                TokenKind::Indent => { // set new self.index
                    let mut indent = 0;

                    while self.tokens[i].kind == TokenKind::Indent {
                        indent += 1;
                        i += 1;
                    }

                    i -= 1;

                    if prev_indent > indent {
                        if prev_indent - indent == 1 || prev_indent - indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    } else {
                        if indent - prev_indent == 1 || indent - prev_indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    }
                },

                TokenKind::Newline => { // reset self.index
                    prev_indent = self.indent;
                    self.indent = 0;
                    self.line += 1;
                },

                TokenKind::EOF => break,
            }
            i += 1;
        }

        if errors.len() == 0 {
            Ok(self.pair)
        } else {
            Err(errors)
        }
    }
}

<commit_msg>fix OBOE<commit_after>
use pair::Pair;
use token::{Token, TokenKind};
use error::{NcclError, ErrorKind};

use std::error::Error;

#[derive(Debug)]
pub struct Parser {
    current: usize,
    path: Vec<String>,
    indent: usize,
    tokens: Vec<Token>,
    pair: Pair,
    line: u64,
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: Pair::new("__top_level__"),
            line: 1,
        }
    }

    pub fn new_with(tokens: Vec<Token>, pair: Pair) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: pair,
            line: 1
        }
    }

    // faked you out with that Scanner, didn't I?
    // you thought this was going to be recursive descent. YOU WERE WRONG!
    pub fn parse(mut self) -> Result<Pair, Vec<Box<Error>>> {
        let mut errors: Vec<Box<Error>> = vec![];
        let mut prev_indent = 0;
        let mut i = 0;

        while i < self.tokens.len() {
            match self.tokens[i].kind {
                TokenKind::Value => { // add to path respective of self.index
                    if self.indent <= self.path.len() {
                        let mut new = self.path[0..self.indent].to_owned();
                        new.push(self.tokens[i].lexeme.clone());
                        self.path = new;
                    } else {
                        self.path.push(self.tokens[i].lexeme.clone());
                    }

                    self.pair.add_slice(&self.path);

                    if i + 2 < self.tokens.len() && self.tokens[i + 2].kind == TokenKind::Value {
                        self.path.clear();
                        self.indent = 0;
                    }
                },

                TokenKind::Indent => { // set new self.index
                    let mut indent = 0;

                    while self.tokens[i].kind == TokenKind::Indent {
                        indent += 1;
                        i += 1;
                    }

                    i -= 1;

                    if prev_indent > indent {
                        if prev_indent - indent == 1 || prev_indent - indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    } else {
                        if indent - prev_indent == 1 || indent - prev_indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    }
                },

                TokenKind::Newline => { // reset self.index
                    prev_indent = self.indent;
                    self.indent = 0;
                    self.line += 1;
                },

                TokenKind::EOF => break,
            }
            i += 1;
        }

        if errors.len() == 0 {
            Ok(self.pair)
        } else {
            Err(errors)
        }
    }
}

<|endoftext|>


<commit_before>use ast::{Token, Expr};
use ast::{BlockExpr, ConstExpr, ThrowExpr, ReturnExpr, CallExpr, ConstructExpr, IfExpr, WhileLoopExpr, SwitchExpr, FunctionDeclExpr, LocalExpr, ArrayDeclExpr, ObjectDeclExpr, GetConstFieldExpr, GetFieldExpr, NumOpExpr, ConstExpr};
use ast::{CBool, CNull, CUndefined, CString, CNum};
use ast::{TIdent, TNumber, TString, TSemicolon, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TNumOp};
use collections::treemap::TreeMap;
use std::fmt;
use std::vec::Vec;
#[deriving(Clone)]
#[deriving(Eq)]
/// An error encountered during parsing an expression
pub enum ParseError {
	/// When it expected a certain kind of token, but got another
	Expected(Vec<Token>, Token),
	/// When it expected a certain expression, but got another
	ExpectedExpr(~str, Expr)
}
impl fmt::Show for ParseError {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		return match *self {
			Expected(ref wanted, ref got) if wanted.len() == 0 => write!(f.buf, "Expected expression, got {}", got),
			Expected(ref wanted, ref got) => {
				try!(write!(f.buf, "Expected "));
				let last = wanted.last().unwrap();
				for wanted_token in wanted.iter() {
					match write!(f.buf, "'{}'{}", wanted_token, if wanted_token == last {""} else {", "}) {
						Err(err) => return Err(err),
						Ok(_) => ()
					}
				}
				write!(f.buf, ", got {}", got)
			},
			ExpectedExpr(ref wanted, ref got) => {
				write!(f.buf, "Expected {}, got {}", wanted, got)
			}
		}
	}
}
/// A result which gives a pretty-printed error if it fails
pub trait VerboseResult<T> {
	/// Unwrap the value if it's Ok and if not pretty-print the error
	fn v_unwrap(&self) -> T;
}
impl<T : Clone, E : fmt::Show> VerboseResult<T> for Result<T, E> {
	fn v_unwrap(&self) -> T {
		match *self {
			Ok(ref val) => val.clone(),
			Err(ref err) => fail!("{}", err)
		}
	}
}
pub type ParseResult = Result<~Expr, ParseError>;
pub type ParseStructResult = Result<Option<~Expr>, ParseError>;
/// A Javascript parser
pub struct Parser {
	/// The tokens being input
	tokens: Vec<Token>,
	/// The current position within the tokens
	pos: uint
}
impl Parser {
	/// Creates a new parser, using [tokens] as input
	pub fn new(tokens: Vec<Token>) -> ~Parser {
		return ~Parser {tokens: tokens, pos: 0};
	}
	/// Parse all expressions in the token array
	pub fn parse_all(&mut self) -> ParseResult {
		let mut exprs = Vec::new();
		while self.pos < self.tokens.len() {
			let result = try!(self.parse());
			exprs.push(result);
		}
		return Ok(~BlockExpr(exprs));
	}
	fn parse_struct(&mut self, name:&str) -> ParseStructResult {
		match name {
			"true" => Ok(Some(~ConstExpr(CBool(true)))),
			"false" => Ok(Some(~ConstExpr(CBool(false)))),
			"null" => Ok(Some(~ConstExpr(CNull))),
			"undefined" => Ok(Some(~ConstExpr(CUndefined))),
			"throw" => {
				let thrown = try!(self.parse());
				Ok(Some(~ThrowExpr(thrown)))
			},
			"return" => Ok(Some(~ReturnExpr(Some(try!(self.parse()))))),
			"new" => {
				let call = try!(self.parse());
				print!("new {}", call);
				match *call {
					CallExpr(ref func, ref args) => Ok(Some(~ConstructExpr(func.clone(), args.clone()))),
					_ => Err(ExpectedExpr(~"constructor", *call))
				}
			}
			"if" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				let next = self.tokens.get(self.pos + 1).clone();
				Ok(Some(~IfExpr(cond, expr, if next == TIdent(~"else") {
					self.pos += 2;
					Some(try!(self.parse()))
				} else {
					None
				})))
			},
			"while" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				Ok(Some(~WhileLoopExpr(cond, expr)))
			},
			"switch" => {
				try!(self.expect(TOpenParen));
				let value = self.parse();
				try!(self.expect(TCloseParen));
				try!(self.expect(TOpenBlock));
				let mut cases = Vec::new();
				let mut default = None;
				while self.pos + 1 < self.tokens.len() {
					let tok = self.tokens.get(self.pos).clone();
					self.pos += 1;
					match tok {
						TIdent(ref id) if *id == ~"case" => {
							let cond = self.parse();
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							cases.push((cond.unwrap(), block));
						},
						TIdent(ref id) if *id == ~"default" => {
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							default = Some(~BlockExpr(block));
						},
						TCloseBlock => break,
						_ => fail!("Expected 'case', 'default' or '{}', not '{}'", '}', tok)
					}
				}
				try!(self.expect(TCloseBlock));
				Ok(Some(~SwitchExpr(value.unwrap(), cases, default)))
			},
			"function" => {
				let tk = self.tokens.get(self.pos).clone();
				let name : Option<~str> = match tk {
					TIdent(ref name) => {
						self.pos += 1;
						Some(name.clone())
					},
					TOpenParen => None,
					_ => return Err(Expected(vec!(TIdent(~"identifier")), tk.clone()))
				};
				try!(self.expect(TOpenParen));
				let mut args:Vec<~str> = Vec::new();
				let mut tk = self.tokens.get(self.pos).clone();
				while tk != TCloseParen {
					match tk {
						TIdent(ref id) => args.push(id.to_owned()),
						_ => return Err(Expected(vec!(TIdent(~"identifier")), tk))
					}
					self.pos += 1;
					if *self.tokens.get(self.pos) == TComma {
						self.pos += 1;
					}
					tk = self.tokens.get(self.pos).clone();
				}
				self.pos += 1;
				let block = try!(self.parse());
				Ok(Some(~FunctionDeclExpr(name, args, block)))
			},
			_ => Ok(None)
		}
	}
	/// Parse a single expression
	pub fn parse(&mut self) -> ParseResult {
		let token = self.tokens.get(self.pos).clone();
		self.pos += 1;
		let expr : ~Expr = match token {
			TSemicolon => try!(self.parse()),
			TIdent(ref s) => {
				let structure = try!(self.parse_struct(s.clone()));
				match structure {
					Some(v) => v,
					None => ~LocalExpr(s.to_owned())
				}
			},
			TString(ref s) => ~ConstExpr(CString(s.to_owned())),
			TOpenParen => {
				let nexte = try!(self.parse());
				try!(self.expect(TCloseParen));
				nexte
			},
			TOpenArray => {
				self.pos += 1;
				let mut array = Vec::new();
				while self.tokens.get(self.pos) == &TComma || array.len() == 0 {
					array.push(match *self.tokens.get(self.pos) {
						TComma => ~ConstExpr(CNull),
						_ => try!(self.parse())
					});
					self.pos += 1;
				}
				self.pos -= 1;
				try!(self.expect(TCloseArray));
				~ArrayDeclExpr(array)
			},
			TOpenBlock if self.tokens.get(self.pos) == &TCloseBlock => {
				self.pos += 1;
				~ObjectDeclExpr(~TreeMap::new())
			},
			TOpenBlock if self.tokens.get(self.pos + 1) == &TColon => {
				let mut map = ~TreeMap::new();
				while self.tokens.get(self.pos - 1) == &TComma || map.len() == 0 {
					let tk = self.tokens.get(self.pos).clone();
					let name = match tk {
						TIdent(ref id) => id,
						TString(ref str) => str,
						_ => return Err(Expected(vec!(TIdent(~"identifier"), TString(~"string")), tk))
					};
					self.pos += 1;
					try!(self.expect(TColon));
					let value = try!(self.parse());
					map.insert(name.to_owned(), value);
					self.pos += 1;
				}
				~ObjectDeclExpr(map)
			},
			TOpenBlock => {
				let mut exprs = Vec::new();
				loop {
					if *self.tokens.get(self.pos) == TCloseBlock {
						break;
					} else {
						exprs.push(try!(self.parse()));
					}
				}
				self.pos += 1;
				~BlockExpr(exprs)
			},
			TNumber(num) => ~ConstExpr(CNum(num)),
			_ => return Err(Expected(Vec::new(), token.clone()))
		};
		return if self.pos >= self.tokens.len() { Ok(expr) } else {self.parse_next(expr)};
	}
	fn parse_next(&mut self, expr:~Expr) -> ParseResult {
		let next = self.tokens.get(self.pos).clone();
		let mut carry_on = true;
		let mut result = expr.clone();
		match next {
			TDot => {
				self.pos += 1;
				let tk = self.tokens.get(self.pos).clone();
				match tk {
					TIdent(ref s) => result = ~GetConstFieldExpr(expr, s.to_owned()),
					_ => return Err(Expected(vec!(TIdent(~"identifier")), next))
				}
				self.pos += 1;
			},
			TOpenParen => {
				let mut args = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos + 1) == TCloseParen;
				loop {
					self.pos += 1;
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseParen && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseParen), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						args.push(parsed);
						expect_comma_or_end = true;
					}
				}
				result = ~CallExpr(expr, args);
			},
			TQuestion => {
				self.pos += 1;
				let if_e = try!(self.parse());
				try!(self.expect(TColon));
				let else_e = try!(self.parse());
				result = ~IfExpr(expr, if_e, Some(else_e));
			},
			TOpenArray => {
				self.pos += 1;
				let index = try!(self.parse());
				try!(self.expect(TCloseArray));
				result = ~GetFieldExpr(expr, index);
			},
			TSemicolon => {
				self.pos += 1;
			},
			TNumOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~NumOpExpr(op, expr, next);
			},
			_ => carry_on = false
		};
		return if carry_on && self.pos < self.tokens.len() { self.parse_next(result) } else {Ok(result)};
	}
	/// Returns an error if the next symbol is not [tk]
	fn expect(&mut self, tk:Token) -> Result<(), ParseError> {
		self.pos += 1;
		let curr_tk = self.tokens.get(self.pos - 1).clone();
		return if curr_tk != tk {
			Err(Expected(vec!(tk), curr_tk))
		} else {
			Ok(())
		};
	}
}<commit_msg>Fix array parsing<commit_after>use ast::{Token, Expr};
use ast::{BlockExpr, ConstExpr, ThrowExpr, ReturnExpr, CallExpr, ConstructExpr, IfExpr, WhileLoopExpr, SwitchExpr, FunctionDeclExpr, LocalExpr, ArrayDeclExpr, ObjectDeclExpr, GetConstFieldExpr, GetFieldExpr, NumOpExpr, ConstExpr};
use ast::{CBool, CNull, CUndefined, CString, CNum};
use ast::{TIdent, TNumber, TString, TSemicolon, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TNumOp};
use collections::treemap::TreeMap;
use std::fmt;
use std::vec::Vec;
#[deriving(Clone)]
#[deriving(Eq)]
/// An error encountered during parsing an expression
pub enum ParseError {
	/// When it expected a certain kind of token, but got another
	Expected(Vec<Token>, Token),
	/// When it expected a certain expression, but got another
	ExpectedExpr(~str, Expr)
}
impl fmt::Show for ParseError {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		return match *self {
			Expected(ref wanted, ref got) if wanted.len() == 0 => write!(f.buf, "Expected expression, got {}", got),
			Expected(ref wanted, ref got) => {
				try!(write!(f.buf, "Expected "));
				let last = wanted.last().unwrap();
				for wanted_token in wanted.iter() {
					match write!(f.buf, "'{}'{}", wanted_token, if wanted_token == last {""} else {", "}) {
						Err(err) => return Err(err),
						Ok(_) => ()
					}
				}
				write!(f.buf, ", got {}", got)
			},
			ExpectedExpr(ref wanted, ref got) => {
				write!(f.buf, "Expected {}, got {}", wanted, got)
			}
		}
	}
}
/// A result which gives a pretty-printed error if it fails
pub trait VerboseResult<T> {
	/// Unwrap the value if it's Ok and if not pretty-print the error
	fn v_unwrap(&self) -> T;
}
impl<T : Clone, E : fmt::Show> VerboseResult<T> for Result<T, E> {
	fn v_unwrap(&self) -> T {
		match *self {
			Ok(ref val) => val.clone(),
			Err(ref err) => fail!("{}", err)
		}
	}
}
pub type ParseResult = Result<~Expr, ParseError>;
pub type ParseStructResult = Result<Option<~Expr>, ParseError>;
/// A Javascript parser
pub struct Parser {
	/// The tokens being input
	tokens: Vec<Token>,
	/// The current position within the tokens
	pos: uint
}
impl Parser {
	/// Creates a new parser, using [tokens] as input
	pub fn new(tokens: Vec<Token>) -> ~Parser {
		return ~Parser {tokens: tokens, pos: 0};
	}
	/// Parse all expressions in the token array
	pub fn parse_all(&mut self) -> ParseResult {
		let mut exprs = Vec::new();
		while self.pos < self.tokens.len() {
			let result = try!(self.parse());
			exprs.push(result);
		}
		return Ok(~BlockExpr(exprs));
	}
	fn parse_struct(&mut self, name:&str) -> ParseStructResult {
		match name {
			"true" => Ok(Some(~ConstExpr(CBool(true)))),
			"false" => Ok(Some(~ConstExpr(CBool(false)))),
			"null" => Ok(Some(~ConstExpr(CNull))),
			"undefined" => Ok(Some(~ConstExpr(CUndefined))),
			"throw" => {
				let thrown = try!(self.parse());
				Ok(Some(~ThrowExpr(thrown)))
			},
			"return" => Ok(Some(~ReturnExpr(Some(try!(self.parse()))))),
			"new" => {
				let call = try!(self.parse());
				print!("new {}", call);
				match *call {
					CallExpr(ref func, ref args) => Ok(Some(~ConstructExpr(func.clone(), args.clone()))),
					_ => Err(ExpectedExpr(~"constructor", *call))
				}
			}
			"if" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				let next = self.tokens.get(self.pos + 1).clone();
				Ok(Some(~IfExpr(cond, expr, if next == TIdent(~"else") {
					self.pos += 2;
					Some(try!(self.parse()))
				} else {
					None
				})))
			},
			"while" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				Ok(Some(~WhileLoopExpr(cond, expr)))
			},
			"switch" => {
				try!(self.expect(TOpenParen));
				let value = self.parse();
				try!(self.expect(TCloseParen));
				try!(self.expect(TOpenBlock));
				let mut cases = Vec::new();
				let mut default = None;
				while self.pos + 1 < self.tokens.len() {
					let tok = self.tokens.get(self.pos).clone();
					self.pos += 1;
					match tok {
						TIdent(ref id) if *id == ~"case" => {
							let cond = self.parse();
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							cases.push((cond.unwrap(), block));
						},
						TIdent(ref id) if *id == ~"default" => {
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							default = Some(~BlockExpr(block));
						},
						TCloseBlock => break,
						_ => fail!("Expected 'case', 'default' or '{}', not '{}'", '}', tok)
					}
				}
				try!(self.expect(TCloseBlock));
				Ok(Some(~SwitchExpr(value.unwrap(), cases, default)))
			},
			"function" => {
				let tk = self.tokens.get(self.pos).clone();
				let name : Option<~str> = match tk {
					TIdent(ref name) => {
						self.pos += 1;
						Some(name.clone())
					},
					TOpenParen => None,
					_ => return Err(Expected(vec!(TIdent(~"identifier")), tk.clone()))
				};
				try!(self.expect(TOpenParen));
				let mut args:Vec<~str> = Vec::new();
				let mut tk = self.tokens.get(self.pos).clone();
				while tk != TCloseParen {
					match tk {
						TIdent(ref id) => args.push(id.to_owned()),
						_ => return Err(Expected(vec!(TIdent(~"identifier")), tk))
					}
					self.pos += 1;
					if *self.tokens.get(self.pos) == TComma {
						self.pos += 1;
					}
					tk = self.tokens.get(self.pos).clone();
				}
				self.pos += 1;
				let block = try!(self.parse());
				Ok(Some(~FunctionDeclExpr(name, args, block)))
			},
			_ => Ok(None)
		}
	}
	/// Parse a single expression
	pub fn parse(&mut self) -> ParseResult {
		let token = self.tokens.get(self.pos).clone();
		self.pos += 1;
		let expr : ~Expr = match token {
			TSemicolon => try!(self.parse()),
			TIdent(ref s) => {
				let structure = try!(self.parse_struct(s.clone()));
				match structure {
					Some(v) => v,
					None => ~LocalExpr(s.to_owned())
				}
			},
			TString(ref s) => ~ConstExpr(CString(s.to_owned())),
			TOpenParen => {
				let nexte = try!(self.parse());
				try!(self.expect(TCloseParen));
				nexte
			},
			TOpenArray => {
				let mut array = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos) == TCloseArray;
				loop {
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseArray && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseArray), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						array.push(parsed);
						expect_comma_or_end = true;
					}
					self.pos += 1;
				}
				~ArrayDeclExpr(array)
			},
			TOpenBlock if self.tokens.get(self.pos) == &TCloseBlock => {
				self.pos += 1;
				~ObjectDeclExpr(~TreeMap::new())
			},
			TOpenBlock if self.tokens.get(self.pos + 1) == &TColon => {
				let mut map = ~TreeMap::new();
				while self.tokens.get(self.pos - 1) == &TComma || map.len() == 0 {
					let tk = self.tokens.get(self.pos).clone();
					let name = match tk {
						TIdent(ref id) => id,
						TString(ref str) => str,
						_ => return Err(Expected(vec!(TIdent(~"identifier"), TString(~"string")), tk))
					};
					self.pos += 1;
					try!(self.expect(TColon));
					let value = try!(self.parse());
					map.insert(name.to_owned(), value);
					self.pos += 1;
				}
				~ObjectDeclExpr(map)
			},
			TOpenBlock => {
				let mut exprs = Vec::new();
				loop {
					if *self.tokens.get(self.pos) == TCloseBlock {
						break;
					} else {
						exprs.push(try!(self.parse()));
					}
				}
				self.pos += 1;
				~BlockExpr(exprs)
			},
			TNumber(num) => ~ConstExpr(CNum(num)),
			_ => return Err(Expected(Vec::new(), token.clone()))
		};
		return if self.pos >= self.tokens.len() { Ok(expr) } else {self.parse_next(expr)};
	}
	fn parse_next(&mut self, expr:~Expr) -> ParseResult {
		let next = self.tokens.get(self.pos).clone();
		let mut carry_on = true;
		let mut result = expr.clone();
		match next {
			TDot => {
				self.pos += 1;
				let tk = self.tokens.get(self.pos).clone();
				match tk {
					TIdent(ref s) => result = ~GetConstFieldExpr(expr, s.to_owned()),
					_ => return Err(Expected(vec!(TIdent(~"identifier")), next))
				}
				self.pos += 1;
			},
			TOpenParen => {
				let mut args = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos + 1) == TCloseParen;
				loop {
					self.pos += 1;
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseParen && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseParen), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						args.push(parsed);
						expect_comma_or_end = true;
					}
				}
				result = ~CallExpr(expr, args);
			},
			TQuestion => {
				self.pos += 1;
				let if_e = try!(self.parse());
				try!(self.expect(TColon));
				let else_e = try!(self.parse());
				result = ~IfExpr(expr, if_e, Some(else_e));
			},
			TOpenArray => {
				self.pos += 1;
				let index = try!(self.parse());
				try!(self.expect(TCloseArray));
				result = ~GetFieldExpr(expr, index);
			},
			TSemicolon => {
				self.pos += 1;
			},
			TNumOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~NumOpExpr(op, expr, next);
			},
			_ => carry_on = false
		};
		return if carry_on && self.pos < self.tokens.len() { self.parse_next(result) } else {Ok(result)};
	}
	/// Returns an error if the next symbol is not [tk]
	fn expect(&mut self, tk:Token) -> Result<(), ParseError> {
		self.pos += 1;
		let curr_tk = self.tokens.get(self.pos - 1).clone();
		return if curr_tk != tk {
			Err(Expected(vec!(tk), curr_tk))
		} else {
			Ok(())
		};
	}
}<|endoftext|>


<commit_before>use std::mem;

pub use self::Token::*;

use self::TokenClass::*;
use self::ParserState::*;

/// `Token` is a section of a compiled mustache string.
#[derive(Clone, Debug)]
pub enum Token {
    Text(String),
    ETag(Vec<String>, String),
    UTag(Vec<String>, String),
    Section(Vec<String>, bool, Vec<Token>, String, String, String, String, String),
    IncompleteSection(Vec<String>, bool, String, bool),
    Partial(String, String, String),
}

enum TokenClass {
    Normal,
    StandAlone,
    WhiteSpace(String, usize),
    NewLineWhiteSpace(String, usize),
}

/// `Parser` parses a string into a series of `Token`s.
pub struct Parser<'a, T: 'a> {
    reader: &'a mut T,
    ch: Option<char>,
    lookahead: Option<char>,
    line: usize,
    col: usize,
    content: String,
    state: ParserState,
    otag: String,
    ctag: String,
    otag_chars: Vec<char>,
    ctag_chars: Vec<char>,
    tag_position: usize,
    tokens: Vec<Token>,
    partials: Vec<String>,
}

enum ParserState { TEXT, OTAG, TAG, CTAG }

impl<'a, T: Iterator<Item=char>> Parser<'a, T> {
    pub fn new(reader: &'a mut T, otag: &str, ctag: &str) -> Parser<'a, T> {
        let mut parser = Parser {
            reader: reader,
            ch: None,
            lookahead: None,
            line: 1,
            col: 1,
            content: String::new(),
            state: TEXT,
            otag: otag.to_string(),
            ctag: ctag.to_string(),
            otag_chars: otag.chars().collect(),
            ctag_chars: ctag.chars().collect(),
            tag_position: 0,
            tokens: Vec::new(),
            partials: Vec::new(),
        };

        parser.bump();
        parser
    }

    fn bump(&mut self) {
        match self.lookahead.take() {
            None => { self.ch = self.reader.next(); }
            Some(ch) => { self.ch = Some(ch); }
        }

        if let Some(ch) = self.ch {
            if ch == '\n' {
                self.line += 1;
                self.col = 1;
            } else {
                self.col += 1;
            }
        }
    }

    fn peek(&mut self) -> Option<char> {
        match self.lookahead {
            None => {
                self.lookahead = self.reader.next();
                self.lookahead
            }
            Some(ch) => Some(ch),
        }
    }

    fn ch_is(&self, ch: char) -> bool {
        match self.ch {
            Some(c) => c == ch,
            None => false,
        }
    }

    /// Parse the template into tokens and a list of partial files.
    pub fn parse(mut self) -> (Vec<Token>, Vec<String>) {
        let mut curly_brace_tag = false;

        while let Some(ch) = self.ch {
            match self.state {
                TEXT => {
                    if ch == self.otag_chars[0] {
                        if self.otag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = OTAG;
                        } else {
                            self.add_text();
                            self.state = TAG;
                        }
                    } else {
                        self.content.push(ch);
                    }
                    self.bump();
                }
                OTAG => {
                    if ch == self.otag_chars[self.tag_position] {
                        if self.tag_position == self.otag_chars.len() - 1 {
                            self.add_text();
                            curly_brace_tag = false;
                            self.state = TAG;
                        } else {
                            self.tag_position = self.tag_position + 1;
                        }
                    } else {
                        // We don't have a tag, so add all the tag parts we've seen
                        // so far to the string.
                        self.state = TEXT;
                        self.not_otag();
                        self.content.push(ch);
                    }
                    self.bump();
                }
                TAG => {
                    if self.content.is_empty() && ch == '{' {
                        curly_brace_tag = true;
                        self.content.push(ch);
                        self.bump();
                    } else if curly_brace_tag && ch == '}' {
                        curly_brace_tag = false;
                        self.content.push(ch);
                        self.bump();
                    } else if ch == self.ctag_chars[0] {
                        if self.ctag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = CTAG;
                            self.bump();
                        } else {
                            self.add_tag();
                            self.state = TEXT;
                        }
                    } else {
                        self.content.push(ch);
                        self.bump();
                    }
                }
                CTAG => {
                    if ch == self.ctag_chars[self.tag_position] {
                        if self.tag_position == self.ctag_chars.len() - 1 {
                            self.add_tag();
                            self.state = TEXT;
                        } else {
                            self.state = TAG;
                            self.not_ctag();
                            self.content.push(ch);
                            self.bump();
                        }
                    } else {
                        panic!("character {} is not part of CTAG: {}",
                              ch,
                              self.ctag_chars[self.tag_position]);
                    }
                }
            }
        }

        match self.state {
            TEXT => { self.add_text(); }
            OTAG => { self.not_otag(); self.add_text(); }
            TAG => { panic!("unclosed tag"); }
            CTAG => { self.not_ctag(); self.add_text(); }
        }

        // Check that we don't have any incomplete sections.
        for token in self.tokens.iter() {
            if let IncompleteSection(ref path, _, _, _) = *token {
                panic!("Unclosed mustache section {}", path.join("."));
            }
        }

        let Parser { tokens, partials, .. } = self;

        (tokens, partials)
    }

    fn add_text(&mut self) {
        if !self.content.is_empty() {
            let mut content = String::new();
            mem::swap(&mut content, &mut self.content);

            self.tokens.push(Text(content.to_string()));
        }
    }

    // This function classifies whether or not a token is standalone, or if it
    // has trailing whitespace. It's looking for this pattern:
    //
    //   ("\n" | "\r\n") whitespace* token ("\n" | "\r\n")
    //
    fn classify_token(&mut self) -> TokenClass {
        // Exit early if the next character is not '\n' or '\r\n'.
        if let Some(ch) = self.ch {
            if !(ch == '\n' || (ch == '\r' && self.peek() == Some('\n'))) {
                return Normal;
            }
        }

        match self.tokens.last() {
            // If the last token ends with a newline (or there is no previous
            // token), then this token is standalone.
            None => { StandAlone }

            Some(&IncompleteSection(_, _, _, true)) => { StandAlone }

            Some(&Text(ref s)) if !s.is_empty() => {
                // Look for the last newline character that may have whitespace
                // following it.
                match s.rfind(| c:char| c == '\n' || !c.is_whitespace()) {
                    // It's all whitespace.
                    None => {
                        if self.tokens.len() == 1 {
                            WhiteSpace(s.to_string(), 0)
                        } else {
                            Normal
                        }
                    }
                    Some(pos) => {
                        if s.as_bytes()[pos] == b'\n' {
                            if pos == s.len() - 1 {
                                StandAlone
                            } else {
                                WhiteSpace(s.to_string(), pos + 1)
                            }
                        } else { Normal }
                    }
                }
            }
            Some(_) => Normal,
        }
    }

    fn eat_whitespace(&mut self) -> bool {
        // If the next character is a newline, and the last token ends with a
        // newline and whitespace, clear out the whitespace.

        match self.classify_token() {
            Normal => { false }
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                true
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                true
            }
        }
    }

    fn add_tag(&mut self) {
        self.bump();

        let tag = self.otag.clone() + &self.content + &self.ctag;

        // Move the content to avoid a copy.
        let mut content = String::new();
        mem::swap(&mut content, &mut self.content);
        let len = content.len();
        let content = content;

        match content.as_bytes()[0] as char {
            '!' => {
                // ignore comments
                self.eat_whitespace();
            }
            '&' => {
                let name = &content[1..len];
                let name = self.check_content(name);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(UTag(name, tag));
            }
            '{' => {
                if content.ends_with("}") {
                    let name = &content[1..len - 1];
                    let name = self.check_content(name);
                    let name = name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect();
                    self.tokens.push(UTag(name, tag));
                } else { panic!("unbalanced \"{\" in tag"); }
            }
            '#' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, false, tag, newlined));
            }
            '^' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, true, tag, newlined));
            }
            '/' => {
                self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                let mut children: Vec<Token> = Vec::new();

                loop {
                    if self.tokens.is_empty() {
                        panic!("closing unopened section");
                    }

                    let last = self.tokens.pop();

                    match last {
                        Some(IncompleteSection(section_name, inverted, osection, _)) => {
                            children.reverse();

                            // Collect all the children's sources.
                            let mut srcs = Vec::new();
                            for child in children.iter() {
                                match *child {
                                    Text(ref s)
                                    | ETag(_, ref s)
                                    | UTag(_, ref s)
                                    | Partial(_, _, ref s) => {
                                        srcs.push(s.clone())
                                    }
                                    Section(_, _, _, _, ref osection, ref src, ref csection, _) => {
                                        srcs.push(osection.clone());
                                        srcs.push(src.clone());
                                        srcs.push(csection.clone());
                                    }
                                    _ => panic!(),
                                }
                            }

                            if section_name == name {
                                // Cache the combination of all the sources in the
                                // section. It's unfortunate, but we need to do this in
                                // case the user uses a function to instantiate the
                                // tag.
                                let mut src = String::new();
                                for s in srcs.iter() { src.push_str(s); }

                                self.tokens.push(
                                    Section(
                                        name,
                                        inverted,
                                        children,
                                        self.otag.to_string(),
                                        osection,
                                        src.to_string(),
                                        tag,
                                        self.ctag.to_string()));
                                break;
                            } else {
                                panic!("Unclosed section");
                            }
                        }
                        Some(last_token) => children.push(last_token),
                        None => (),
                    }
                }
            }
            '>' => { self.add_partial(&content, tag); }
            '=' => {
                self.eat_whitespace();

                if len > 2usize && content.ends_with("=") {
                    let s = self.check_content(&content[1..len - 1]);

                    fn is_whitespace(c: char) -> bool { c.is_whitespace() }
                    let pos = s.find(is_whitespace);
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.otag = s[0..pos].to_string();
                    self.otag_chars = self.otag.chars().collect();

                    let s2 = &s[pos..];
                    let pos = s2.find(| c : char| !c.is_whitespace());
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.ctag = s2[pos..].to_string();
                    self.ctag_chars = self.ctag.chars().collect();
                } else {
                    panic!("invalid change delimiter tag content");
                }
            }
            _ => {
                // If the name is "." then we want the top element, which we represent with
                // an empty name.
                let name = self.check_content(&content);
                let name = if name == "." {
                    Vec::new()
                } else {
                    name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect()
                };

                self.tokens.push(ETag(name, tag));
            }
        }
    }

    fn add_partial(&mut self, content: &str, tag: String) {
        let indent = match self.classify_token() {
            Normal => "".to_string(),
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                "".to_string()
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                let ws = &s[pos..];

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                ws.to_string()
            }
        };

        // We can't inline the tokens directly as we may have a recursive
        // partial. So instead, we'll cache the partials we used and look them
        // up later.
        let name = &content[1..content.len()];
        let name = self.check_content(name);

        self.tokens.push(Partial(name.to_string(), indent, tag));
        self.partials.push(name);
    }

    fn not_otag(&mut self) {
        for (i, ch) in self.otag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn not_ctag(&mut self) {
        for (i, ch) in self.ctag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn check_content(&self, content: &str) -> String {
        let trimmed = content.trim();
        if trimmed.is_empty() {
            panic!("empty tag");
        }
        trimmed.to_string()
    }
}
<commit_msg>chore(clippy): fix Clippy’s `single_char_pattern` warnings<commit_after>use std::mem;

pub use self::Token::*;

use self::TokenClass::*;
use self::ParserState::*;

/// `Token` is a section of a compiled mustache string.
#[derive(Clone, Debug)]
pub enum Token {
    Text(String),
    ETag(Vec<String>, String),
    UTag(Vec<String>, String),
    Section(Vec<String>, bool, Vec<Token>, String, String, String, String, String),
    IncompleteSection(Vec<String>, bool, String, bool),
    Partial(String, String, String),
}

enum TokenClass {
    Normal,
    StandAlone,
    WhiteSpace(String, usize),
    NewLineWhiteSpace(String, usize),
}

/// `Parser` parses a string into a series of `Token`s.
pub struct Parser<'a, T: 'a> {
    reader: &'a mut T,
    ch: Option<char>,
    lookahead: Option<char>,
    line: usize,
    col: usize,
    content: String,
    state: ParserState,
    otag: String,
    ctag: String,
    otag_chars: Vec<char>,
    ctag_chars: Vec<char>,
    tag_position: usize,
    tokens: Vec<Token>,
    partials: Vec<String>,
}

enum ParserState { TEXT, OTAG, TAG, CTAG }

impl<'a, T: Iterator<Item=char>> Parser<'a, T> {
    pub fn new(reader: &'a mut T, otag: &str, ctag: &str) -> Parser<'a, T> {
        let mut parser = Parser {
            reader: reader,
            ch: None,
            lookahead: None,
            line: 1,
            col: 1,
            content: String::new(),
            state: TEXT,
            otag: otag.to_string(),
            ctag: ctag.to_string(),
            otag_chars: otag.chars().collect(),
            ctag_chars: ctag.chars().collect(),
            tag_position: 0,
            tokens: Vec::new(),
            partials: Vec::new(),
        };

        parser.bump();
        parser
    }

    fn bump(&mut self) {
        match self.lookahead.take() {
            None => { self.ch = self.reader.next(); }
            Some(ch) => { self.ch = Some(ch); }
        }

        if let Some(ch) = self.ch {
            if ch == '\n' {
                self.line += 1;
                self.col = 1;
            } else {
                self.col += 1;
            }
        }
    }

    fn peek(&mut self) -> Option<char> {
        match self.lookahead {
            None => {
                self.lookahead = self.reader.next();
                self.lookahead
            }
            Some(ch) => Some(ch),
        }
    }

    fn ch_is(&self, ch: char) -> bool {
        match self.ch {
            Some(c) => c == ch,
            None => false,
        }
    }

    /// Parse the template into tokens and a list of partial files.
    pub fn parse(mut self) -> (Vec<Token>, Vec<String>) {
        let mut curly_brace_tag = false;

        while let Some(ch) = self.ch {
            match self.state {
                TEXT => {
                    if ch == self.otag_chars[0] {
                        if self.otag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = OTAG;
                        } else {
                            self.add_text();
                            self.state = TAG;
                        }
                    } else {
                        self.content.push(ch);
                    }
                    self.bump();
                }
                OTAG => {
                    if ch == self.otag_chars[self.tag_position] {
                        if self.tag_position == self.otag_chars.len() - 1 {
                            self.add_text();
                            curly_brace_tag = false;
                            self.state = TAG;
                        } else {
                            self.tag_position = self.tag_position + 1;
                        }
                    } else {
                        // We don't have a tag, so add all the tag parts we've seen
                        // so far to the string.
                        self.state = TEXT;
                        self.not_otag();
                        self.content.push(ch);
                    }
                    self.bump();
                }
                TAG => {
                    if self.content.is_empty() && ch == '{' {
                        curly_brace_tag = true;
                        self.content.push(ch);
                        self.bump();
                    } else if curly_brace_tag && ch == '}' {
                        curly_brace_tag = false;
                        self.content.push(ch);
                        self.bump();
                    } else if ch == self.ctag_chars[0] {
                        if self.ctag_chars.len() > 1 {
                            self.tag_position = 1;
                            self.state = CTAG;
                            self.bump();
                        } else {
                            self.add_tag();
                            self.state = TEXT;
                        }
                    } else {
                        self.content.push(ch);
                        self.bump();
                    }
                }
                CTAG => {
                    if ch == self.ctag_chars[self.tag_position] {
                        if self.tag_position == self.ctag_chars.len() - 1 {
                            self.add_tag();
                            self.state = TEXT;
                        } else {
                            self.state = TAG;
                            self.not_ctag();
                            self.content.push(ch);
                            self.bump();
                        }
                    } else {
                        panic!("character {} is not part of CTAG: {}",
                              ch,
                              self.ctag_chars[self.tag_position]);
                    }
                }
            }
        }

        match self.state {
            TEXT => { self.add_text(); }
            OTAG => { self.not_otag(); self.add_text(); }
            TAG => { panic!("unclosed tag"); }
            CTAG => { self.not_ctag(); self.add_text(); }
        }

        // Check that we don't have any incomplete sections.
        for token in self.tokens.iter() {
            if let IncompleteSection(ref path, _, _, _) = *token {
                panic!("Unclosed mustache section {}", path.join("."));
            }
        }

        let Parser { tokens, partials, .. } = self;

        (tokens, partials)
    }

    fn add_text(&mut self) {
        if !self.content.is_empty() {
            let mut content = String::new();
            mem::swap(&mut content, &mut self.content);

            self.tokens.push(Text(content.to_string()));
        }
    }

    // This function classifies whether or not a token is standalone, or if it
    // has trailing whitespace. It's looking for this pattern:
    //
    //   ("\n" | "\r\n") whitespace* token ("\n" | "\r\n")
    //
    fn classify_token(&mut self) -> TokenClass {
        // Exit early if the next character is not '\n' or '\r\n'.
        if let Some(ch) = self.ch {
            if !(ch == '\n' || (ch == '\r' && self.peek() == Some('\n'))) {
                return Normal;
            }
        }

        match self.tokens.last() {
            // If the last token ends with a newline (or there is no previous
            // token), then this token is standalone.
            None => { StandAlone }

            Some(&IncompleteSection(_, _, _, true)) => { StandAlone }

            Some(&Text(ref s)) if !s.is_empty() => {
                // Look for the last newline character that may have whitespace
                // following it.
                match s.rfind(| c:char| c == '\n' || !c.is_whitespace()) {
                    // It's all whitespace.
                    None => {
                        if self.tokens.len() == 1 {
                            WhiteSpace(s.to_string(), 0)
                        } else {
                            Normal
                        }
                    }
                    Some(pos) => {
                        if s.as_bytes()[pos] == b'\n' {
                            if pos == s.len() - 1 {
                                StandAlone
                            } else {
                                WhiteSpace(s.to_string(), pos + 1)
                            }
                        } else { Normal }
                    }
                }
            }
            Some(_) => Normal,
        }
    }

    fn eat_whitespace(&mut self) -> bool {
        // If the next character is a newline, and the last token ends with a
        // newline and whitespace, clear out the whitespace.

        match self.classify_token() {
            Normal => { false }
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                true
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                true
            }
        }
    }

    fn add_tag(&mut self) {
        self.bump();

        let tag = self.otag.clone() + &self.content + &self.ctag;

        // Move the content to avoid a copy.
        let mut content = String::new();
        mem::swap(&mut content, &mut self.content);
        let len = content.len();
        let content = content;

        match content.as_bytes()[0] as char {
            '!' => {
                // ignore comments
                self.eat_whitespace();
            }
            '&' => {
                let name = &content[1..len];
                let name = self.check_content(name);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(UTag(name, tag));
            }
            '{' => {
                if content.ends_with('}') {
                    let name = &content[1..len - 1];
                    let name = self.check_content(name);
                    let name = name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect();
                    self.tokens.push(UTag(name, tag));
                } else { panic!("unbalanced \"{\" in tag"); }
            }
            '#' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, false, tag, newlined));
            }
            '^' => {
                let newlined = self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                self.tokens.push(IncompleteSection(name, true, tag, newlined));
            }
            '/' => {
                self.eat_whitespace();

                let name = self.check_content(&content[1..len]);
                let name = name.split_terminator('.')
                    .map(|x| x.to_string())
                    .collect();
                let mut children: Vec<Token> = Vec::new();

                loop {
                    if self.tokens.is_empty() {
                        panic!("closing unopened section");
                    }

                    let last = self.tokens.pop();

                    match last {
                        Some(IncompleteSection(section_name, inverted, osection, _)) => {
                            children.reverse();

                            // Collect all the children's sources.
                            let mut srcs = Vec::new();
                            for child in children.iter() {
                                match *child {
                                    Text(ref s)
                                    | ETag(_, ref s)
                                    | UTag(_, ref s)
                                    | Partial(_, _, ref s) => {
                                        srcs.push(s.clone())
                                    }
                                    Section(_, _, _, _, ref osection, ref src, ref csection, _) => {
                                        srcs.push(osection.clone());
                                        srcs.push(src.clone());
                                        srcs.push(csection.clone());
                                    }
                                    _ => panic!(),
                                }
                            }

                            if section_name == name {
                                // Cache the combination of all the sources in the
                                // section. It's unfortunate, but we need to do this in
                                // case the user uses a function to instantiate the
                                // tag.
                                let mut src = String::new();
                                for s in srcs.iter() { src.push_str(s); }

                                self.tokens.push(
                                    Section(
                                        name,
                                        inverted,
                                        children,
                                        self.otag.to_string(),
                                        osection,
                                        src.to_string(),
                                        tag,
                                        self.ctag.to_string()));
                                break;
                            } else {
                                panic!("Unclosed section");
                            }
                        }
                        Some(last_token) => children.push(last_token),
                        None => (),
                    }
                }
            }
            '>' => { self.add_partial(&content, tag); }
            '=' => {
                self.eat_whitespace();

                if len > 2usize && content.ends_with('=') {
                    let s = self.check_content(&content[1..len - 1]);

                    fn is_whitespace(c: char) -> bool { c.is_whitespace() }
                    let pos = s.find(is_whitespace);
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.otag = s[0..pos].to_string();
                    self.otag_chars = self.otag.chars().collect();

                    let s2 = &s[pos..];
                    let pos = s2.find(| c : char| !c.is_whitespace());
                    let pos = match pos {
                      None => { panic!("invalid change delimiter tag content"); }
                      Some(pos) => { pos }
                    };

                    self.ctag = s2[pos..].to_string();
                    self.ctag_chars = self.ctag.chars().collect();
                } else {
                    panic!("invalid change delimiter tag content");
                }
            }
            _ => {
                // If the name is "." then we want the top element, which we represent with
                // an empty name.
                let name = self.check_content(&content);
                let name = if name == "." {
                    Vec::new()
                } else {
                    name.split_terminator('.')
                        .map(|x| x.to_string())
                        .collect()
                };

                self.tokens.push(ETag(name, tag));
            }
        }
    }

    fn add_partial(&mut self, content: &str, tag: String) {
        let indent = match self.classify_token() {
            Normal => "".to_string(),
            StandAlone => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();
                "".to_string()
            }
            WhiteSpace(s, pos) | NewLineWhiteSpace(s, pos) => {
                if self.ch_is('\r') { self.bump(); }
                self.bump();

                let ws = &s[pos..];

                // Trim the whitespace from the last token.
                self.tokens.pop();
                self.tokens.push(Text(s[0..pos].to_string()));

                ws.to_string()
            }
        };

        // We can't inline the tokens directly as we may have a recursive
        // partial. So instead, we'll cache the partials we used and look them
        // up later.
        let name = &content[1..content.len()];
        let name = self.check_content(name);

        self.tokens.push(Partial(name.to_string(), indent, tag));
        self.partials.push(name);
    }

    fn not_otag(&mut self) {
        for (i, ch) in self.otag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn not_ctag(&mut self) {
        for (i, ch) in self.ctag_chars.iter().enumerate() {
            if !(i < self.tag_position) {
                break
            }
            self.content.push(*ch);
        }
    }

    fn check_content(&self, content: &str) -> String {
        let trimmed = content.trim();
        if trimmed.is_empty() {
            panic!("empty tag");
        }
        trimmed.to_string()
    }
}
<|endoftext|>