<commit_before>use std::iter::{Peekable};
use ast;

/* Grammar
    main {  }

*/

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(usize, usize),
    Letter(u8)
}

fn tokenize(grammar : &String) -> Vec<Token> {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = (-1, -1);
    let mut i = 0;
    let mut in_quote = false;
    let mut escaped = false;

    while let Some(item) = iterator.next() {
        if in_quote {
            if escaped {
                tokens.push(Token::Letter(item as u8));
                escaped = false;
            } else {
                match item {
                    '\'' => {
                        tokens.push(Token::SingleQuote);
                        in_quote = false;
                    },
                    '\"' => {
                        tokens.push(Token::DoubleQuote);
                        in_quote = false;
                    },
                    '\\' => { escaped = true; },
                    _ => tokens.push(Token::Letter(item as u8))
                }
            }
        } else {
            match item {
                '{' => tokens.push(Token::OpenBrace),
                '}' => tokens.push(Token::CloseBrace),
                '(' => tokens.push(Token::OpenParen),
                ')' => tokens.push(Token::CloseParen),
                '[' => tokens.push(Token::OpenBracket),
                ']' => tokens.push(Token::CloseBracket),
                '.' => tokens.push(Token::Dot),
                '+' => tokens.push(Token::Plus),
                '*' => tokens.push(Token::Asterik),
                '?' => tokens.push(Token::Question),
                '!' => tokens.push(Token::Exclamation),
                '&' => tokens.push(Token::Ambersand),
                '/' => tokens.push(Token::Slash),
                '\'' => {
                    tokens.push(Token::SingleQuote);
                    in_quote = true;
                },
                '\"' => {
                    tokens.push(Token::DoubleQuote);
                    in_quote = true;
                },
                _ if item.is_alphanumeric() => {
                    if name == (-1, -1) {
                        name = (i, i);
                    } else {
                        name.1 += 1;
                    }
                },
                _ if item.is_whitespace() && name != (-1, -1) => {
                    tokens.push(Token::Name(name.0 as usize, name.1 as usize));
                    name = (-1, -1);
                }
                _ => { }
            }
        }
        i += 1;
    }
    tokens
}

fn parse(grammar : &String, tokens : Vec<Token>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter().peekable();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };

    while let Some(token) = iterator.next() {
        if let &Token::Name(name_start, name_end) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator, false);
                    match pattern {
                        Ok(p) => grammar_object.rules.push(p),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }
    Ok(grammar_object)
}

fn parse_pattern<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut sequence = vec![];

    while let Some(token) = iterator.next() {
        let mut initial_pattern = Err(0);
        match token {
            &Token::OpenParen => { initial_pattern = initial_pattern.or(parse_pattern(grammar, iterator, true)); },
            &Token::OpenBracket => { initial_pattern = initial_pattern.or(parse_char_class(grammar, iterator)); },
            &Token::DoubleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, false)); },
            &Token::SingleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, true)); },
            &Token::Dot => { initial_pattern = initial_pattern.or(Ok(ast::Pattern::CharAny)); },
            &Token::Name(le, ri) => { initial_pattern = initial_pattern.or(parse_variable(grammar, le, ri)); },
            &Token::Ambersand => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, true)); },
            &Token::Exclamation => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, false)); },
            _ => { }
        }

        if initial_pattern.is_ok() {
            if let Some(&suffix_token) = iterator.peek() {
                match suffix_token {
                    &Token::Plus | &Token::Asterik | &Token::Question => {
                        initial_pattern = parse_suffix(grammar, iterator, initial_pattern.ok().unwrap());
                    }
                    _ => { }
                }
            }
        }

        match initial_pattern {
            Ok(p) => sequence.push(p),
            Err(x) => return Err(x)
        }

        if let Some(&test_token) = iterator.peek() {
            match test_token {
                &Token::CloseParen => if is_subpattern {
                    iterator.next();
                } else {
                    return Err(0);
                },
                &Token::CloseBrace => if !is_subpattern {
                    iterator.next();
                    break;
                } else {
                    return Err(0);
                },
                &Token::Slash => {
                    let sub_pattern = parse_choice(grammar, iterator, &mut sequence, is_subpattern);
                    match sub_pattern {
                        Ok(p) => sequence.push(p),
                        Err(x) => return Err(x)
                    }
                },
                _ => { }
            }
        }
    }

    let mut boxed_vec = vec![];
    while let Some(item) = sequence.pop() {
        boxed_vec.push(Box::new(item));
    }

    Ok(ast::Pattern::Sequence(boxed_vec))
}

fn parse_choice<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, patterns : &mut Vec<ast::Pattern>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut boxed_vec = vec![];
    while let Some(item) = patterns.pop() {
        boxed_vec.push(Box::new(item));
    }

    iterator.next();

    let right = parse_pattern(grammar, iterator, is_subpattern);

    match right {
        Ok(p) => Ok(ast::Pattern::Choice(
            Box::new(ast::Pattern::Sequence(boxed_vec)),
            Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_suffix<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, pattern : ast::Pattern) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    if let Some(token) = iterator.next() {
        match token {
            &Token::Asterik => Ok(ast::Pattern::ZeroOrMore(Box::new(pattern))),
            &Token::Plus => Ok(ast::Pattern::OneOrMore(Box::new(pattern))),
            &Token::Question => Ok(ast::Pattern::Optional(Box::new(pattern))),
            _ => Err(0)
        }
    } else {
        Err(0)
    }
}

fn parse_lookahead<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, lookahead : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let sub_pattern = parse_pattern(grammar, iterator, false);
    match sub_pattern {
        Ok(p) => Ok(ast::Pattern::Lookahead(lookahead, Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_char_class<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_char_sequence<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, single_quote : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{

    Err(0)
}

fn parse_variable(grammar : &String, left : usize, right : usize) -> Result<ast::Pattern, u8>
{
    Err(0)
}

<commit_msg>Add variable parser and ordering<commit_after>use std::collections::{HashMap};
use std::iter::{Peekable};
use std::str;
use ast;

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(i32),
    Letter(u8)
}

fn tokenize(grammar : &String) -> (Vec<Token>, HashMap<String, i32>) {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = vec![];
    let mut i = 1;
    let mut in_quote = false;
    let mut escaped = false;
    let mut map = HashMap::new();
    map.insert("main".to_string(), 0);

    while let Some(item) = iterator.next() {
        if in_quote {
            if escaped {
                tokens.push(Token::Letter(item as u8));
                escaped = false;
            } else {
                match item {
                    '\'' => {
                        tokens.push(Token::SingleQuote);
                        in_quote = false;
                    },
                    '\"' => {
                        tokens.push(Token::DoubleQuote);
                        in_quote = false;
                    },
                    '\\' => { escaped = true; },
                    _ => tokens.push(Token::Letter(item as u8))
                }
            }
        } else {
            match item {
                '{' => tokens.push(Token::OpenBrace),
                '}' => tokens.push(Token::CloseBrace),
                '(' => tokens.push(Token::OpenParen),
                ')' => tokens.push(Token::CloseParen),
                '[' => tokens.push(Token::OpenBracket),
                ']' => tokens.push(Token::CloseBracket),
                '.' => tokens.push(Token::Dot),
                '+' => tokens.push(Token::Plus),
                '*' => tokens.push(Token::Asterik),
                '?' => tokens.push(Token::Question),
                '!' => tokens.push(Token::Exclamation),
                '&' => tokens.push(Token::Ambersand),
                '/' => tokens.push(Token::Slash),
                '\'' => {
                    tokens.push(Token::SingleQuote);
                    in_quote = true;
                },
                '\"' => {
                    tokens.push(Token::DoubleQuote);
                    in_quote = true;
                },
                _ if item.is_alphanumeric() => name.push(item as u8),
                _ if item.is_whitespace() && name.len() != 0 => {
                    let id = map.entry(String::from_utf8(name.clone()).unwrap()).or_insert(i);
                    tokens.push(Token::Name(*id));
                    name.clear();
                    i += 1;
                }
                _ => { }
            }
        }
    }
    (tokens, map)
}

fn parse(grammar : &String, tokens : Vec<Token>, rule_map : HashMap<String, i32>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter().peekable();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };
    let mut insert_order = vec![];

    while let Some(token) = iterator.next() {
        if let &Token::Name(id) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator, false);
                    match pattern {
                        Ok(p) => insert_order.push((id, p)),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }

    insert_order.sort_by(|a, b| a.0.cmp(&b.0));
    grammar_object.rules = insert_order.drain(..).map(|x| x.1).collect();

    Ok(grammar_object)
}

fn parse_pattern<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut sequence = vec![];

    while let Some(token) = iterator.next() {
        let mut initial_pattern = Err(0);
        match token {
            &Token::OpenParen => { initial_pattern = initial_pattern.or(parse_pattern(grammar, iterator, true)); },
            &Token::OpenBracket => { initial_pattern = initial_pattern.or(parse_char_class(grammar, iterator)); },
            &Token::DoubleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, false)); },
            &Token::SingleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator, true)); },
            &Token::Dot => { initial_pattern = initial_pattern.or(Ok(ast::Pattern::CharAny)); },
            &Token::Name(id) => { initial_pattern = initial_pattern.or(parse_variable(grammar, id)); },
            &Token::Ambersand => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, true)); },
            &Token::Exclamation => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, false)); },
            _ => { }
        }

        if initial_pattern.is_ok() {
            if let Some(&suffix_token) = iterator.peek() {
                match suffix_token {
                    &Token::Plus | &Token::Asterik | &Token::Question => {
                        initial_pattern = parse_suffix(grammar, iterator, initial_pattern.ok().unwrap());
                    }
                    _ => { }
                }
            }
        }

        match initial_pattern {
            Ok(p) => sequence.push(p),
            Err(x) => return Err(x)
        }

        if let Some(&test_token) = iterator.peek() {
            match test_token {
                &Token::CloseParen => if is_subpattern {
                    iterator.next();
                } else {
                    return Err(0);
                },
                &Token::CloseBrace => if !is_subpattern {
                    iterator.next();
                    break;
                } else {
                    return Err(0);
                },
                &Token::Slash => {
                    let sub_pattern = parse_choice(grammar, iterator, &mut sequence, is_subpattern);
                    match sub_pattern {
                        Ok(p) => sequence.push(p),
                        Err(x) => return Err(x)
                    }
                },
                _ => { }
            }
        }
    }

    let mut boxed_vec = vec![];
    while let Some(item) = sequence.pop() {
        boxed_vec.push(Box::new(item));
    }

    Ok(ast::Pattern::Sequence(boxed_vec))
}

fn parse_choice<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, patterns : &mut Vec<ast::Pattern>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut boxed_vec = vec![];
    while let Some(item) = patterns.pop() {
        boxed_vec.push(Box::new(item));
    }

    iterator.next();

    let right = parse_pattern(grammar, iterator, is_subpattern);

    match right {
        Ok(p) => Ok(ast::Pattern::Choice(
            Box::new(ast::Pattern::Sequence(boxed_vec)),
            Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_suffix<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, pattern : ast::Pattern) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    if let Some(token) = iterator.next() {
        match token {
            &Token::Asterik => Ok(ast::Pattern::ZeroOrMore(Box::new(pattern))),
            &Token::Plus => Ok(ast::Pattern::OneOrMore(Box::new(pattern))),
            &Token::Question => Ok(ast::Pattern::Optional(Box::new(pattern))),
            _ => Err(0)
        }
    } else {
        Err(0)
    }
}

fn parse_lookahead<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, lookahead : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let sub_pattern = parse_pattern(grammar, iterator, false);
    match sub_pattern {
        Ok(p) => Ok(ast::Pattern::Lookahead(lookahead, Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_char_class<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut tuples = vec![];

    while let Some(&token) = iterator.peek() {
        if token == &Token::CloseBracket {
            iterator.next();
            break;
        }
        let letter = parse_char_class_element(grammar, iterator);

        if let Some(&dot_token) = iterator.peek() {
            if dot_token == &Token::Dot {
                let range_letter = parse_char_class_element(grammar, iterator);
                if letter.is_ok() && range_letter.is_ok() {
                    tuples.push((letter.ok().unwrap(), Some(range_letter.ok().unwrap())));
                } else {
                    return Err(0);
                }
            }
        }

        match letter {
            Ok(x) => tuples.push((x, None)),
            Err(x) => return Err(x)
        };
    }

    Ok(ast::Pattern::CharClass(tuples))
}

fn parse_char_class_element<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<u8, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let first_quote = iterator.next();
    let utf8_byte = iterator.next();
    let second_quote = iterator.next();

    if first_quote.is_some() && utf8_byte.is_some() && second_quote.is_some() {
        let single_quote_check = *first_quote.unwrap() == Token::SingleQuote
            && first_quote.unwrap() == second_quote.unwrap();
        let double_quote_check = *first_quote.unwrap() == Token::DoubleQuote
            && first_quote.unwrap() == second_quote.unwrap();

        if single_quote_check || double_quote_check {
            match utf8_byte.unwrap() {
                &Token::Letter(x) => Ok(x),
                _ => Err(0)
            }
        } else {
            Err(0)
        }
    } else {
        Err(0)
    }
}

fn parse_char_sequence<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_single_quote : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut characters = vec![];
    while let Some(item) = iterator.next() {
        match item {
            &Token::Letter(x) => { characters.push(x); },
            &Token::SingleQuote if is_single_quote => { break; },
            &Token::DoubleQuote if !is_single_quote => { break; },
            _ => { return Err(0); }
        }
    }
    Ok(ast::Pattern::CharSequence(characters))
}

fn parse_variable(grammar : &String, id : i32) -> Result<ast::Pattern, u8>
{
    Ok(ast::Pattern::Variable(id))
}

<|endoftext|>


<commit_before>use ast::*;
use scanner::{Token, TokenWithContext};
use std::iter::Peekable;

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Option<Expr>, String> {
    let mut iter = tokens.iter().peekable();
    parse_expression(&mut iter)
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<Operator>,
                       parse_subexpression: &Fn(&mut Peekable<I>) -> Result<Option<Expr>, String>)
                       -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        if let Some(e) = try!(parse_subexpression(tokens)) {
            expr = e;
        } else {
            return Ok(None);
        }
    };
    let peeked_token;
    {
        peeked_token = tokens.peek().cloned(); // Can I avoid this?
    };
    while let Some(peeked_token) = peeked_token {
        if let Some(mapped_operator) = map_operator(&peeked_token.token) {
            {
                // Just advance, we know all we need from the peeked value
                let _ = tokens.next();
            }
            let right;
            {
                if let Some(e) = try!(parse_subexpression(tokens)) {
                    right = e
                } else {
                    // TODO add context
                    return Err("Expected subexpression".into());
                }
            };
            let binary_expression = BinaryExpr {
                left: expr,
                operator: mapped_operator,
                right: right,
            };
            expr = Expr::Binary(Box::new(binary_expression));
        } else {
            break;
        }
    }
    Ok(Some(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::BangEqual => Some(Operator::NotEqual),
            &Token::EqualEqual => Some(Operator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Greater => Some(Operator::Greater),
            &Token::GreaterEqual => Some(Operator::GreaterEqual),
            &Token::Less => Some(Operator::Less),
            &Token::LessEqual => Some(Operator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Plus => Some(Operator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Slash => Some(Operator::Slash),
            &Token::Star => Some(Operator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Bang => Some(Operator::Bang),
            _ => None,
        }
    }
    let peeked_token;
    {
        peeked_token = tokens.peek().cloned(); // Can I avoid this?
    };
    if let Some(Some(mapped_operator)) =
        peeked_token.map(|pt| &pt.token)
            .map(map_operator) {
        {
            // Just advance, we know all we need from the peeked value
            let _ = tokens.next();
        }
        let right;
        {
            if let Some(e) = try!(parse_unary(tokens)) {
                right = e;
            } else {
                // TODO: add context
                return Err("Expected right side of unary".into());
            }
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Ok(Some(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr;
                {
                    if let Some(e) = try!(parse_expression(tokens)) {
                        expr = e;
                    } else {
                        // TODO add context
                        return Err("Unfinished grouping expression".into());
                    }
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::LeftParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Ok(Some(Expr::Grouping(Box::new(grouping_expression))));
                        }
                    }
                    // TODO: fill with context
                    return Err("Missing )".into());
                }
            }
            _ => {
                // TODO: fill with context
                return Err("Unexpected token".into());
            }
        };
        Ok(Some(parsed_expression))
    } else {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let tokens = scan(&"123".into()).unwrap();
        let expr = parse(tokens).unwrap().unwrap();
        assert_eq!("123", &expr.pretty_print());
    }

    #[test]
    fn binary() {
    let tokens = scan(&"123+456".into()).unwrap();
    let expr = parse(tokens).unwrap().unwrap();
    assert_eq!("123 + 456", &expr.pretty_print());
    }
}
<commit_msg>Fixed parsing<commit_after>use ast::*;
use scanner::{Token, TokenWithContext};
use std::iter::Peekable;

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Option<Expr>, String> {
    let mut iter = tokens.iter().peekable();
    parse_expression(&mut iter)
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<Operator>,
                       parse_subexpression: &Fn(&mut Peekable<I>) -> Result<Option<Expr>, String>)
                       -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        if let Some(e) = try!(parse_subexpression(tokens)) {
            expr = e;
        } else {
            return Ok(None);
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        {
            // Just advance, we know all we need from the peeked value
            let _ = tokens.next();
        }
        let right;
        {
            if let Some(e) = try!(parse_subexpression(tokens)) {
                right = e
            } else {
                // TODO add context
                return Err("Expected subexpression".into());
            }
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Ok(Some(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::BangEqual => Some(Operator::NotEqual),
            &Token::EqualEqual => Some(Operator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Greater => Some(Operator::Greater),
            &Token::GreaterEqual => Some(Operator::GreaterEqual),
            &Token::Less => Some(Operator::Less),
            &Token::LessEqual => Some(Operator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Plus => Some(Operator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Slash => Some(Operator::Slash),
            &Token::Star => Some(Operator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Bang => Some(Operator::Bang),
            _ => None,
        }
    }
    let peeked_token;
    {
        peeked_token = tokens.peek().cloned(); // Can I avoid this?
    };
    if let Some(Some(mapped_operator)) =
        peeked_token.map(|pt| &pt.token)
            .map(map_operator) {
        {
            // Just advance, we know all we need from the peeked value
            let _ = tokens.next();
        }
        let right;
        {
            if let Some(e) = try!(parse_unary(tokens)) {
                right = e;
            } else {
                // TODO: add context
                return Err("Expected right side of unary".into());
            }
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Ok(Some(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Result<Option<Expr>, String>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr;
                {
                    if let Some(e) = try!(parse_expression(tokens)) {
                        expr = e;
                    } else {
                        // TODO add context
                        return Err("Unfinished grouping expression".into());
                    }
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::LeftParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Ok(Some(Expr::Grouping(Box::new(grouping_expression))));
                        }
                    }
                    // TODO: fill with context
                    return Err("Missing )".into());
                }
            }
            _ => {
                // TODO: fill with context
                return Err("Unexpected token".into());
            }
        };
        Ok(Some(parsed_expression))
    } else {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let tokens = scan(&"123".into()).unwrap();
        let expr = parse(tokens).unwrap().unwrap();
        assert_eq!("123", &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let tokens = scan(&"123+456".into()).unwrap();
        let expr = parse(tokens).unwrap().unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }
}
<|endoftext|>


<commit_before>use std::str::from_utf8;
use std::result::Result;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: TypeFlag,
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    GlobalExtendedHeaderWithMetadata,
    ExtendedHeaderWithMetadataForNext,
    VendorSpecific,
    Invalid
}

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u += f * ((c as u64) - ('0' as u64));
        f *= 8;
    }

    Ok(u)
}

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::GlobalExtendedHeaderWithMetadata,
        'x' => TypeFlag::ExtendedHeaderWithMetadataForNext,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::Invalid
    }
}

macro_rules! take_str_eat_garbage (
 ( $i:expr, $size:expr ) => ( chain!( $i, s: map_res!(take_until!("\0"), from_utf8) ~ take!($size - s.len()), ||{ s } ));
);

fn parse_ustar00(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("00")                                                 ~
        uname:    take_str_eat_garbage!(32)                        ~
        gname:    take_str_eat_garbage!(32)                        ~
        devmajor: map_res!(take_str_eat_garbage!(8), octal_to_u64) ~
        devminor: map_res!(take_str_eat_garbage!(8), octal_to_u64) ~
        prefix:   take_str_eat_garbage!(155)                       ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                prefix:   prefix
            })
        }
    )
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: parse_ustar00,
        ||{
            ustar
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str_eat_garbage!(100)                        ~
        mode:     take_str_eat_garbage!(8)                          ~
        uid:      map_res!(take_str_eat_garbage!(8),  octal_to_u64) ~
        gid:      map_res!(take_str_eat_garbage!(8),  octal_to_u64) ~
        size:     map_res!(take_str_eat_garbage!(12), octal_to_u64) ~
        mtime:    map_res!(take_str_eat_garbage!(12), octal_to_u64) ~
        chksum:   take_str_eat_garbage!(8)                          ~
        typeflag: take!(1)                                          ~
        linkname: take_str_eat_garbage!(100)                        ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: char_to_type_flag(typeflag[0] as char),
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    map_res!(i, many0!(parse_entry), filter_entries)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }
}
<commit_msg>add test for take_str_eat_garbage<commit_after>use std::str::from_utf8;
use std::result::Result;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: TypeFlag,
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    GlobalExtendedHeaderWithMetadata,
    ExtendedHeaderWithMetadataForNext,
    VendorSpecific,
    Invalid
}

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u += f * ((c as u64) - ('0' as u64));
        f *= 8;
    }

    Ok(u)
}

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::GlobalExtendedHeaderWithMetadata,
        'x' => TypeFlag::ExtendedHeaderWithMetadataForNext,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::Invalid
    }
}

macro_rules! take_str_eat_garbage (
 ( $i:expr, $size:expr ) => ( chain!( $i, s: map_res!(take_until!("\0"), from_utf8) ~ take!($size - s.len()), ||{ s } ));
);

fn parse_ustar00(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("00")                                                 ~
        uname:    take_str_eat_garbage!(32)                        ~
        gname:    take_str_eat_garbage!(32)                        ~
        devmajor: map_res!(take_str_eat_garbage!(8), octal_to_u64) ~
        devminor: map_res!(take_str_eat_garbage!(8), octal_to_u64) ~
        prefix:   take_str_eat_garbage!(155)                       ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                prefix:   prefix
            })
        }
    )
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: parse_ustar00,
        ||{
            ustar
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str_eat_garbage!(100)                        ~
        mode:     take_str_eat_garbage!(8)                          ~
        uid:      map_res!(take_str_eat_garbage!(8),  octal_to_u64) ~
        gid:      map_res!(take_str_eat_garbage!(8),  octal_to_u64) ~
        size:     map_res!(take_str_eat_garbage!(12), octal_to_u64) ~
        mtime:    map_res!(take_str_eat_garbage!(12), octal_to_u64) ~
        chksum:   take_str_eat_garbage!(8)                          ~
        typeflag: take!(1)                                          ~
        linkname: take_str_eat_garbage!(100)                        ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: char_to_type_flag(typeflag[0] as char),
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

fn filter_entries(entries: Vec<TarEntry>) -> Result<Vec<TarEntry>, &'static str> {
    Ok(entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>())
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    map_res!(i, many0!(parse_entry), filter_entries)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fs::File;
use std::iter::Peekable;
use std::str::Chars;

use super::sql::*;

//pub enum Keyword {
//    Select,
//    From,
//    Where
//}

#[derive(Debug,Clone,PartialEq)]
pub enum Token {
    Identifier(String),
    Keyword(String),
    Operator(String),
    Number(String),
    Comma,
    Whitespace,
    Eq,
    Neq,
    Lt,
    Gt,
    LtEq,
    GtEq,
    Plus,
    Minus,
    Mult,
    Div,
    LParen,
    RParen,

    //Operator(String)
}

#[derive(Debug,Clone)]
pub enum ParserError {
    TokenizerError(String),
    ParserError(String),
}

struct Tokenizer {
    query: String,
}

impl Tokenizer {

    pub fn tokenize(&mut self) -> Result<Vec<Token>, ParserError> {

        let mut peekable = self.query.chars().peekable();

        let mut tokens : Vec<Token> = vec![];

        while let Some(token) = self.next_token(&mut peekable)? {
            tokens.push(token);
        }

        Ok(tokens.into_iter().filter(|t| match t {
            &Token::Whitespace => false,
            _ => true
        }).collect())
    }

    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<Token>, ParserError> {
        match chars.peek() {
            Some(&ch) => match ch {
                // whitespace
                ' ' | '\t' | '\n' => {
                    chars.next(); // consume
                    Ok(Some(Token::Whitespace))
                },
                // identifier or keyword
                'a' ... 'z' | 'A' ... 'Z' | '_' | '@' => {
                    let mut s = String::new();
                    while let Some(&ch) = chars.peek() {
                        match ch {
                            'a' ... 'z' | 'A' ... 'Z' | '_' | '0' ... '9' => {
                                chars.next(); // consume
                                s.push(ch);
                            },
                            _ => break
                        }
                    }
                    match s.to_uppercase().as_ref() {
                        "SELECT" | "FROM" | "WHERE" | "LIMIT" | "ORDER" | "GROUP" | "BY" |
                        "UNION" | "ALL"| "UPDATE" | "DELETE" | "IN" | "NOT" | "NULL" |
                        "SET" => Ok(Some(Token::Keyword(s))),
                        _ => Ok(Some(Token::Identifier(s))),
                    }
                },
                // numbers
                '0' ... '9' => {
                    let mut s = String::new();
                    while let Some(&ch) = chars.peek() {
                        match ch {
                            '0' ... '9' => {
                                chars.next(); // consume
                                s.push(ch);
                            },
                            _ => break
                        }
                    }
                    Ok(Some(Token::Number(s)))
                },
                // operators
                '+' => { chars.next(); Ok(Some(Token::Plus)) },
                '-' => { chars.next(); Ok(Some(Token::Minus)) },
                '*' => { chars.next(); Ok(Some(Token::Mult)) },
                '/' => { chars.next(); Ok(Some(Token::Div)) },
                '=' => { chars.next(); Ok(Some(Token::Eq)) },
                '<' => {
                    chars.next(); // consume
                    match chars.peek() {
                        Some(&ch) => match ch {
                            '=' => {
                                chars.next();
                                Ok(Some(Token::LtEq))
                            },
                            '>' => {
                                chars.next();
                                Ok(Some(Token::Neq))
                            },
                            _ => Ok(Some(Token::Lt))
                        },
                        None => Ok(Some(Token::Lt))
                    }
                },
                '>' => {
                    chars.next(); // consume
                    match chars.peek() {
                        Some(&ch) => match ch {
                            '=' => {
                                chars.next();
                                Ok(Some(Token::GtEq))
                            },
                            _ => Ok(Some(Token::Gt))
                        },
                        None => Ok(Some(Token::Gt))
                    }
                },
                _ => Err(ParserError::TokenizerError(
                    String::from(format!("unhandled char '{}' in tokenizer", ch))))
            },
            None => Ok(None)
        }
    }
}

pub struct Parser {
    tokens: Vec<Token>,
    index: usize
}

impl Parser {

    pub fn new(tokens: Vec<Token>) -> Self {
        Parser { tokens: tokens, index: 0 }
    }

    pub fn parse(&mut self) -> Result<ASTNode, ParserError> {
        self.parse_expr(0)
    }

    fn parse_expr(&mut self, precedence: u8) -> Result<ASTNode, ParserError> {

        let mut expr = self.parse_prefix()?;

        while let Some(tok) = self.next_token() {

            let next_precedence = self.get_precedence(&tok)?;
            if precedence >= next_precedence {
                break;
            }

            if let Some(new_expr) = self.parse_infix(expr.clone(), next_precedence)? {
                expr = new_expr;
            }
        }

        Ok(expr)
    }

    fn parse_prefix(&mut self) -> Result<ASTNode, ParserError> {
        match self.next_token() {
            Some(Token::Keyword(k)) => {
                match k.to_uppercase().as_ref() {
                    "SELECT" => Ok(self.parse_select()?),
                    _ => Err(ParserError::ParserError(
                        format!("No prefix parser for keyword {}", k))),
                }
            },
            _ => Err(ParserError::ParserError(
                format!("Prefix parser expected a keyword")))
        }
    }

    fn parse_infix(&mut self, expr: ASTNode, precedence: u8) -> Result<Option<ASTNode>, ParserError> {
        match self.next_token() {
            Some(tok) => {
                match tok {
                    Token::Eq => Ok(Some(ASTNode::SQLBinaryExpr {
                        left: Box::new(expr),
                        op: SQLOperator::EQ,
                        right: Box::new(self.parse_expr(precedence)?)
                    })),
                    _ => Err(ParserError::ParserError(
                        format!("No infix parser for token {:?}", tok))),
                }
            },
            None => Ok(None)
        }
    }

    fn get_precedence(&self, tok: &Token) -> Result<u8, ParserError> {
        match tok {
            &Token::Eq | &Token::Lt | & Token::LtEq |
            &Token::Neq | &Token::Gt | & Token::GtEq => Ok(20),
            &Token::Plus | &Token::Minus => Ok(30),
            &Token::Mult | &Token::Div => Ok(40),
            _ => Err(ParserError::TokenizerError(String::from("invalid token for get_precedence")))
        }
    }

    fn peek_token(&mut self) -> Option<Token> {
        if self.index < self.tokens.len() {
            Some(self.tokens[self.index].clone())
        } else {
            None
        }
    }

    fn next_token(&mut self) -> Option<Token> {
        if self.index < self.tokens.len() {
            self.index = self.index + 1;
            Some(self.tokens[self.index-1].clone())
        } else {
            None
        }
    }

    // specific methods

    fn parse_select(&self) -> Result<ASTNode, ParserError> {

        //TODO: parse expression list
        //TODO: parse FROM
        //TODO: parse relation
        //TODO: parse WHERE

        Ok(ASTNode::SQLSelect {
            projection: vec![],
            selection: None,
            relation: None,
            limit: None,
            order: None,
        })
    }

}

#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn tokenize_select_1()  {
        let sql = String::from("SELECT 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        println!("tokens = {:?}", tokens);
        assert_eq!(2, tokens.len());
        assert_eq!(Token::Keyword(String::from("SELECT")), tokens[0]);
        assert_eq!(Token::Number(String::from("1")), tokens[1]);
    }

    #[test]
    fn tokenize_simple_select()  {
        let sql = String::from("SELECT * FROM customer WHERE id = 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        println!("tokens = {:?}", tokens);
        assert_eq!(8, tokens.len());
        assert_eq!(Token::Keyword(String::from("SELECT")), tokens[0]);
        assert_eq!(Token::Mult, tokens[1]);
        assert_eq!(Token::Keyword(String::from("FROM")), tokens[2]);
        assert_eq!(Token::Identifier(String::from("customer")), tokens[3]);
        assert_eq!(Token::Keyword(String::from("WHERE")), tokens[4]);
        assert_eq!(Token::Identifier(String::from("id")), tokens[5]);
        assert_eq!(Token::Eq, tokens[6]);
        assert_eq!(Token::Number(String::from("1")), tokens[7]);
    }

    #[test]
    fn parse_simple_select() {
        let sql = String::from("SELECT * FROM customer WHERE id = 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        let mut parser = Parser::new(tokens);
        let ast = parser.parse().unwrap();
        println!("AST = {:?}", ast);
    }
}

<commit_msg>can now parse expression list<commit_after>use std::error::Error;
use std::fs::File;
use std::iter::Peekable;
use std::str::Chars;

use super::sql::*;

//pub enum Keyword {
//    Select,
//    From,
//    Where
//}

#[derive(Debug,Clone,PartialEq)]
pub enum Token {
    Identifier(String),
    Keyword(String),
    Operator(String),
    Number(String),
    Comma,
    Whitespace,
    Eq,
    Neq,
    Lt,
    Gt,
    LtEq,
    GtEq,
    Plus,
    Minus,
    Mult,
    Div,
    LParen,
    RParen,

    //Operator(String)
}

#[derive(Debug,Clone)]
pub enum ParserError {
    TokenizerError(String),
    ParserError(String),
}

struct Tokenizer {
    query: String,
}

impl Tokenizer {

    pub fn tokenize(&mut self) -> Result<Vec<Token>, ParserError> {

        let mut peekable = self.query.chars().peekable();

        let mut tokens : Vec<Token> = vec![];

        while let Some(token) = self.next_token(&mut peekable)? {
            tokens.push(token);
        }

        Ok(tokens.into_iter().filter(|t| match t {
            &Token::Whitespace => false,
            _ => true
        }).collect())
    }

    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<Token>, ParserError> {
        match chars.peek() {
            Some(&ch) => match ch {
                // whitespace
                ' ' | '\t' | '\n' => {
                    chars.next(); // consume
                    Ok(Some(Token::Whitespace))
                },
                // identifier or keyword
                'a' ... 'z' | 'A' ... 'Z' | '_' | '@' => {
                    let mut s = String::new();
                    while let Some(&ch) = chars.peek() {
                        match ch {
                            'a' ... 'z' | 'A' ... 'Z' | '_' | '0' ... '9' => {
                                chars.next(); // consume
                                s.push(ch);
                            },
                            _ => break
                        }
                    }
                    match s.to_uppercase().as_ref() {
                        "SELECT" | "FROM" | "WHERE" | "LIMIT" | "ORDER" | "GROUP" | "BY" |
                        "UNION" | "ALL"| "UPDATE" | "DELETE" | "IN" | "NOT" | "NULL" |
                        "SET" => Ok(Some(Token::Keyword(s))),
                        _ => Ok(Some(Token::Identifier(s))),
                    }
                },
                // numbers
                '0' ... '9' => {
                    let mut s = String::new();
                    while let Some(&ch) = chars.peek() {
                        match ch {
                            '0' ... '9' => {
                                chars.next(); // consume
                                s.push(ch);
                            },
                            _ => break
                        }
                    }
                    Ok(Some(Token::Number(s)))
                },
                // punctuation
                ',' => { chars.next(); Ok(Some(Token::Comma)) },
                '(' => { chars.next(); Ok(Some(Token::LParen)) },
                ')' => { chars.next(); Ok(Some(Token::RParen)) },
                // operators
                '+' => { chars.next(); Ok(Some(Token::Plus)) },
                '-' => { chars.next(); Ok(Some(Token::Minus)) },
                '*' => { chars.next(); Ok(Some(Token::Mult)) },
                '/' => { chars.next(); Ok(Some(Token::Div)) },
                '=' => { chars.next(); Ok(Some(Token::Eq)) },
                '<' => {
                    chars.next(); // consume
                    match chars.peek() {
                        Some(&ch) => match ch {
                            '=' => {
                                chars.next();
                                Ok(Some(Token::LtEq))
                            },
                            '>' => {
                                chars.next();
                                Ok(Some(Token::Neq))
                            },
                            _ => Ok(Some(Token::Lt))
                        },
                        None => Ok(Some(Token::Lt))
                    }
                },
                '>' => {
                    chars.next(); // consume
                    match chars.peek() {
                        Some(&ch) => match ch {
                            '=' => {
                                chars.next();
                                Ok(Some(Token::GtEq))
                            },
                            _ => Ok(Some(Token::Gt))
                        },
                        None => Ok(Some(Token::Gt))
                    }
                },
                _ => Err(ParserError::TokenizerError(
                    String::from(format!("unhandled char '{}' in tokenizer", ch))))
            },
            None => Ok(None)
        }
    }
}

pub struct Parser {
    tokens: Vec<Token>,
    index: usize
}

impl Parser {

    pub fn new(tokens: Vec<Token>) -> Self {
        Parser { tokens: tokens, index: 0 }
    }

    pub fn parse(&mut self) -> Result<ASTNode, ParserError> {
        self.parse_expr(0)
    }

    fn parse_expr(&mut self, precedence: u8) -> Result<ASTNode, ParserError> {

        let mut expr = self.parse_prefix()?;

        while let Some(tok) = self.peek_token() {

            let next_precedence = self.get_precedence(&tok)?;
            if precedence >= next_precedence {
                break;
            }

            if let Some(new_expr) = self.parse_infix(expr.clone(), next_precedence)? {
                expr = new_expr;
            }
        }

        Ok(expr)
    }

    fn parse_prefix(&mut self) -> Result<ASTNode, ParserError> {
        match self.next_token() {
            Some(t) => {
                match t {
                    Token::Keyword(k) => {
                        match k.to_uppercase().as_ref() {
                            "SELECT" => Ok(self.parse_select()?),
                            _ => Err(ParserError::ParserError(
                                format!("No prefix parser for keyword {}", k))),
                        }
                    },
                    Token::Identifier(id) =>
                        Ok(ASTNode::SQLIdentifier { id: id, parts: vec![] }),
                    _ => Err(ParserError::ParserError(
                        format!("Prefix parser expected a keyword but found {:?}", t)))
                }
            },
            None => Err(ParserError::ParserError(
                format!("Prefix parser expected a keyword but hit EOF")))
        }
    }

    fn parse_infix(&mut self, expr: ASTNode, precedence: u8) -> Result<Option<ASTNode>, ParserError> {
        match self.next_token() {
            Some(tok) => {
                match tok {
                    Token::Eq => Ok(Some(ASTNode::SQLBinaryExpr {
                        left: Box::new(expr),
                        op: SQLOperator::EQ,
                        right: Box::new(self.parse_expr(precedence)?)
                    })),
                    _ => Err(ParserError::ParserError(
                        format!("No infix parser for token {:?}", tok))),
                }
            },
            None => Ok(None)
        }
    }

    fn get_precedence(&self, tok: &Token) -> Result<u8, ParserError> {
        match tok {
            &Token::Eq | &Token::Lt | & Token::LtEq |
            &Token::Neq | &Token::Gt | & Token::GtEq => Ok(20),
            &Token::Plus | &Token::Minus => Ok(30),
            &Token::Mult | &Token::Div => Ok(40),
            _ => Ok(0)
                /*Err(ParserError::TokenizerError(
                format!("invalid token {:?} for get_precedence", tok)))*/
        }
    }

    fn peek_token(&mut self) -> Option<Token> {
        if self.index < self.tokens.len() {
            Some(self.tokens[self.index].clone())
        } else {
            None
        }
    }

    fn next_token(&mut self) -> Option<Token> {
        if self.index < self.tokens.len() {
            self.index = self.index + 1;
            Some(self.tokens[self.index-1].clone())
        } else {
            None
        }
    }

    // specific methods

    fn parse_select(&mut self) -> Result<ASTNode, ParserError> {

        let projection = self.parse_expr_list()?;

        //TODO: parse FROM
        //TODO: parse relation
        //TODO: parse WHERE

        Ok(ASTNode::SQLSelect {
            projection: projection,
            selection: None,
            relation: None,
            limit: None,
            order: None,
        })
    }

    fn parse_expr_list(&mut self) -> Result<Vec<ASTNode>, ParserError> {
        println!("parse_expr_list()");
        let mut expr_list : Vec<ASTNode> = vec![];
        loop {
            println!("parse_expr_list() top of loop");
            expr_list.push(self.parse_expr(0)?);
            if let Some(t) = self.peek_token() {
                if t == Token::Comma {
                    self.next_token();
                } else {
                    println!("parse_expr_list() BREAK on token={:?}", t);
                    break;
                }
            }
        }
        Ok(expr_list)
    }

}

#[cfg(test)]
mod tests {

    use super::*;

    #[test]
    fn tokenize_select_1()  {
        let sql = String::from("SELECT 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        println!("tokens = {:?}", tokens);
        assert_eq!(2, tokens.len());
        assert_eq!(Token::Keyword(String::from("SELECT")), tokens[0]);
        assert_eq!(Token::Number(String::from("1")), tokens[1]);
    }

    #[test]
    fn tokenize_simple_select()  {
        let sql = String::from("SELECT * FROM customer WHERE id = 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        println!("tokens = {:?}", tokens);
        assert_eq!(8, tokens.len());
        assert_eq!(Token::Keyword(String::from("SELECT")), tokens[0]);
        assert_eq!(Token::Mult, tokens[1]);
        assert_eq!(Token::Keyword(String::from("FROM")), tokens[2]);
        assert_eq!(Token::Identifier(String::from("customer")), tokens[3]);
        assert_eq!(Token::Keyword(String::from("WHERE")), tokens[4]);
        assert_eq!(Token::Identifier(String::from("id")), tokens[5]);
        assert_eq!(Token::Eq, tokens[6]);
        assert_eq!(Token::Number(String::from("1")), tokens[7]);
    }

    #[test]
    fn parse_simple_select() {
        let sql = String::from("SELECT id, fname, lname FROM customer WHERE id = 1");
        let mut tokenizer = Tokenizer { query: sql };
        let tokens = tokenizer.tokenize().unwrap();
        let mut parser = Parser::new(tokens);
        let ast = parser.parse().unwrap();
        println!("AST = {:?}", ast);
        match ast {
            ASTNode::SQLSelect { projection, .. } => {
                assert_eq!(3, projection.len());
            },
            _ => assert!(false)
        }
    }
}

<|endoftext|>


<commit_before>use std::str::{self, FromStr};

use pest::prelude::*;

#[derive(Debug, PartialEq, Clone)]
pub struct Program(Vec<Statement>);

impl IntoIterator for Program {
    type Item = Statement;
    type IntoIter = ::std::vec::IntoIter<Statement>;

    fn into_iter(self) -> Self::IntoIter {
        self.0.into_iter()
    }
}

impl FromStr for Program {
    type Err = String;
    fn from_str(input: &str) -> Result<Self, Self::Err> {
        let mut parser = Rdp::new(StringInput::new(input));

        println!("{:?}", (parser.program(), parser.end()));
        println!("{:#?}", parser.queue());
        println!("{:?}", parser.stack());
        let (expected, pos) = parser.expected();
        let (line, col) = parser.input().line_col(pos);
        println!("Expected: {:?} at line {} col {}", expected, line, col);
        unimplemented!();
    }
}

#[derive(Debug, PartialEq, Clone)]
pub enum Statement {
    Comment(String),
    Declaration {
        pattern: Pattern,
        type_def: TypeDefinition,
        expr: Expression,
    },
    Assignment {
        lhs: Identifier,
        expr: Expression,
    },
    WhileLoop {
        condition: Expression,
        body: Block,
    },
    ConditionGroup {
        // Condition expression and body block
        branches: Vec<(Expression, Block)>,
        // "default" or "else" branch body
        default: Option<Block>,
    },
    Expression {
        expr: Expression,
    },
}

#[derive(Debug, PartialEq, Clone)]
pub enum Pattern {
    Identifier(Identifier),
}

#[derive(Debug, PartialEq, Clone)]
pub enum TypeDefinition {
    Name {
        name: Identifier,
    },
    Array {
        type_def: Box<TypeDefinition>,
        //TODO: This probably isn't the right type since this should accept anything and then get
        // statically checked to ensure the correct number was put here
        size: usize,
    },
}

#[derive(Debug, PartialEq, Clone)]
pub enum Expression {
    StringLiteral(String),
    Identifier(Identifier),
}

pub type Identifier = String;
pub type Block = Vec<Statement>;

impl_rdp! {
    grammar! {
        program = _{ statement* ~ eoi }

        statement = _{ declaration | assignment | while_loop | for_loop | conditional | (expr ~ semi) | comment }

        comment = @{ block_comment | line_comment }
        line_comment = _{ ["//"] ~ (!(["\r"] | ["\n"]) ~ any)* ~ (["\n"] | ["\r\n"] | ["\r"] | eoi) }
        block_comment = _{ ["/*"] ~ ((!(["*/"]) ~ any) | block_comment)* ~ ["*/"] }

        assignment = { identifier ~ ["="] ~ expr ~ semi}
        declaration = { ["let"] ~ pattern ~ [":"] ~ type_def ~ (["="] ~ expr)? ~ semi}
        pattern = { identifier }

        type_def = { identifier | array_type }
        array_type = { ["["] ~ identifier ~ semi ~ array_size ~ ["]"] }
        array_size = { ["_"] | number }

        while_loop = { ["while"] ~ expr ~ block }
        for_loop = { ["for"] ~ pattern ~ ["in"] ~ expr ~ block }

        expr = {
            { block | group | constant | func_call | call_chain | conditional | bool_not | string_literal | range | number }

            // Ordered from lowest precedence to highest precedence
            bool_or = {< ["||"] }
            bool_and = {< ["&&"] }
            // NOTE: Order matters! { ["<"] | ["<="] } will never match "<="
            comparison = { ["=="] | ["!="] | [">="] | ["<="] | [">"] | ["<"] }
            concatenation = {< ["++"] }
            term = { ["+"] | ["-"] }
            factor = { ["/"] | ["*"] | ["%"] }
            pow = { ["**"] }
        }

        conditional = { ["if"] ~ expr ~ block ~ (["else"] ~ conditional)? ~ (["else"] ~ block)? }

        // This allows {} and {statement; statement; statement;} and {statement; expr} and {expr}
        block = { ["{"] ~ statement* ~ expr? ~ ["}"] }

        group = { ["("] ~ expr ~ [")"] }
        bool_not = { ["!"] ~ expr }
        range = { number ~ (comma ~ number)? ~ [".."] ~ number }

        func_call = { identifier ~ func_args }

        call_chain = { identifier ~ prop_get_call* }
        prop_get_call = { ["."] ~ identifier ~ func_args? }

        // This allows () and (func_arg, func_arg) and (func_arg) and (func_arg,)
        func_args = { ["("] ~ (func_arg ~ comma)* ~ func_arg? ~ [")"] }
        func_arg = { expr }

        string_literal = { ["\""] ~ literal_char* ~ ["\""] }
        literal_char = _{ escape_sequence | (!["\""] ~ any) }
        escape_sequence = _{ ["\\\\"] | ["\\\""] | ["\\\'"] | ["\\n"] | ["\\r"] | ["\\t"] | ["\\0"] | ["\\f"] | ["\\v"] | ["\\e"] }

        identifier = @{ !keyword ~ (alpha | ["_"]) ~ (alphanumeric | ["_"])* }
        alpha = _{ ['a'..'z'] | ['A'..'Z'] }
        alphanumeric = _{ alpha | ['0'..'9'] }

        number = @{ (["-"] | ["+"])? ~ positive_integer }
        positive_integer = _{ ["0"] | (nonzero ~ digit*) }
        // Allow "_" in numbers for grouping: 1_000_000 == 1000000
        digit = _{ ["0"] | nonzero | ["_"] }
        nonzero = _{ ['1'..'9'] }

        constant = @{ ["true"] | ["false"] }

        whitespace = _{ [" "] | ["\t"] | ["\u{000C}"] | ["\r"] | ["\n"] }
        // NOTE: When changing this code, make sure you don't have a subset of a word before
        // another word. For example: { ["type"] | ["typeof"] } will never match "typeof"
        keyword = @{
            ["abstract"] | ["as"] | ["become"] | ["break"] | ["byte"] | ["class"] | ["clear"] |
            ["const"] | ["continue"] | ["do"] | ["else"] | ["enum"] | ["eval"] | ["export"] |
            ["extern"] | ["false"] | ["final"] | ["fn"] | ["for"] | ["if"] | ["impl"] | ["import"] |
            ["in"] | ["let"] | ["loop"] | ["match"] | ["mod"] | ["move"] | ["mut"] | ["of"] |
            ["out"] | ["pub"] | ["raw"] | ["ref"] | ["return"] | ["self"] | ["static"] |
            ["struct"] | ["super"] | ["trait"] | ["true"] | ["typeof"] | ["type"] | ["unsafe"] |
            ["use"] | ["where"] | ["while"] | ["yield"]
        }

        // These are separate rules because we can use the generated rules and tokens to provide
        // better error messages
        comma = { [","] }
        semi = { [";"] }
    }
}
<commit_msg>Starting to implement code that transforms tokens into ast<commit_after>use std::str::{self, FromStr};
use std::collections::VecDeque;

use pest::prelude::*;

#[derive(Debug, PartialEq, Clone)]
pub struct Program(Vec<Statement>);

impl IntoIterator for Program {
    type Item = Statement;
    type IntoIter = ::std::vec::IntoIter<Statement>;

    fn into_iter(self) -> Self::IntoIter {
        self.0.into_iter()
    }
}

impl FromStr for Program {
    type Err = String;
    fn from_str(input: &str) -> Result<Self, Self::Err> {
        let mut parser = Rdp::new(StringInput::new(input));

        println!("{:?}", (parser.program(), parser.end()));
        println!("{:#?}", parser.queue());
        println!("{:?}", parser.stack());
        let (expected, pos) = parser.expected();
        let (line, col) = parser.input().line_col(pos);
        println!("Expected: {:?} at line {} col {}", expected, line, col);
        unimplemented!();
    }
}

#[derive(Debug, PartialEq, Clone)]
pub enum Statement {
    Comment(String),
    Declaration {
        pattern: Pattern,
        type_def: TypeDefinition,
        expr: Option<Expression>,
    },
    Assignment {
        lhs: Identifier,
        expr: Expression,
    },
    WhileLoop {
        condition: Expression,
        body: Block,
    },
    ConditionGroup {
        // Condition expression and body block
        branches: Vec<(Expression, Block)>,
        // "default" or "else" branch body
        default: Option<Block>,
    },
    Expression {
        expr: Expression,
    },
}

#[derive(Debug, PartialEq, Clone)]
pub enum Pattern {
    Identifier(Identifier),
}

#[derive(Debug, PartialEq, Clone)]
pub enum TypeDefinition {
    Name {
        name: Identifier,
    },
    Array {
        type_def: Box<TypeDefinition>,
        //TODO: This probably isn't the right type since this should accept anything and then get
        // statically checked to ensure the correct number was put here
        size: Option<Number>,
    },
}

#[derive(Debug, PartialEq, Clone)]
pub enum Expression {
    StringLiteral(String),
    Identifier(Identifier),
}

pub type Identifier = String;
pub type Block = Vec<Statement>;
pub type Number = i32;

impl_rdp! {
    grammar! {
        program = _{ statement* ~ eoi }

        statement = _{ declaration | assignment | while_loop | for_loop | conditional | (expr ~ semi) | comment }

        comment = @{ block_comment | line_comment }
        line_comment = _{ ["//"] ~ (!(["\r"] | ["\n"]) ~ any)* ~ (["\n"] | ["\r\n"] | ["\r"] | eoi) }
        block_comment = _{ ["/*"] ~ ((!(["*/"]) ~ any) | block_comment)* ~ ["*/"] }

        assignment = { identifier ~ ["="] ~ expr ~ semi}
        declaration = { ["let"] ~ pattern ~ [":"] ~ type_def ~ (["="] ~ expr)? ~ semi}
        pattern = { identifier }

        type_def = _{ identifier | array_type }
        array_type = { ["["] ~ type_def ~ semi ~ array_size ~ ["]"] }
        array_size = _{ unspecified | number }
        unspecified = { ["_"] }

        while_loop = { ["while"] ~ expr ~ block }
        for_loop = { ["for"] ~ pattern ~ ["in"] ~ expr ~ block }

        expr = {
            { block | group | constant | func_call | call_chain | conditional | bool_not | string_literal | range | number }

            // Ordered from lowest precedence to highest precedence
            bool_or = {< ["||"] }
            bool_and = {< ["&&"] }
            // NOTE: Order matters! { ["<"] | ["<="] } will never match "<="
            comparison = { ["=="] | ["!="] | [">="] | ["<="] | [">"] | ["<"] }
            concatenation = {< ["++"] }
            term = { ["+"] | ["-"] }
            factor = { ["/"] | ["*"] | ["%"] }
            pow = { ["**"] }
        }

        conditional = { ["if"] ~ expr ~ block ~ (["else"] ~ conditional)? ~ (["else"] ~ block)? }

        // This allows {} and {statement; statement; statement;} and {statement; expr} and {expr}
        block = { ["{"] ~ statement* ~ expr? ~ ["}"] }

        group = { ["("] ~ expr ~ [")"] }
        bool_not = { ["!"] ~ expr }
        range = { number ~ (comma ~ number)? ~ [".."] ~ number }

        func_call = { identifier ~ func_args }

        call_chain = { identifier ~ prop_get_call* }
        prop_get_call = { ["."] ~ identifier ~ func_args? }

        // This allows () and (func_arg, func_arg) and (func_arg) and (func_arg,)
        func_args = { ["("] ~ (func_arg ~ comma)* ~ func_arg? ~ [")"] }
        func_arg = { expr }

        string_literal = { ["\""] ~ literal_char* ~ ["\""] }
        literal_char = _{ escape_sequence | (!["\""] ~ any) }
        escape_sequence = _{ ["\\\\"] | ["\\\""] | ["\\\'"] | ["\\n"] | ["\\r"] | ["\\t"] | ["\\0"] | ["\\f"] | ["\\v"] | ["\\e"] }

        identifier = @{ !keyword ~ (alpha | ["_"]) ~ (alphanumeric | ["_"])* }
        alpha = _{ ['a'..'z'] | ['A'..'Z'] }
        alphanumeric = _{ alpha | ['0'..'9'] }

        number = @{ (["-"] | ["+"])? ~ positive_integer }
        positive_integer = _{ ["0"] | (nonzero ~ digit*) }
        // Allow "_" in numbers for grouping: 1_000_000 == 1000000
        digit = _{ ["0"] | nonzero | ["_"] }
        nonzero = _{ ['1'..'9'] }

        constant = @{ ["true"] | ["false"] }

        whitespace = _{ [" "] | ["\t"] | ["\u{000C}"] | ["\r"] | ["\n"] }
        // NOTE: When changing this code, make sure you don't have a subset of a word before
        // another word. For example: { ["type"] | ["typeof"] } will never match "typeof"
        keyword = @{
            ["abstract"] | ["as"] | ["become"] | ["break"] | ["byte"] | ["class"] | ["clear"] |
            ["const"] | ["continue"] | ["do"] | ["else"] | ["enum"] | ["eval"] | ["export"] |
            ["extern"] | ["false"] | ["final"] | ["fn"] | ["for"] | ["if"] | ["impl"] | ["import"] |
            ["in"] | ["let"] | ["loop"] | ["match"] | ["mod"] | ["move"] | ["mut"] | ["of"] |
            ["out"] | ["pub"] | ["raw"] | ["ref"] | ["return"] | ["self"] | ["static"] |
            ["struct"] | ["super"] | ["trait"] | ["true"] | ["typeof"] | ["type"] | ["unsafe"] |
            ["use"] | ["where"] | ["while"] | ["yield"]
        }

        // These are separate rules because we can use the generated rules and tokens to provide
        // better error messages
        comma = { [","] }
        semi = { [";"] }
    }

    process! {
        parse_program(&self) -> Program {
            (list: _program()) => {
                Program(list.into_iter().collect())
            },
        }

        _program(&self) -> VecDeque<Statement> {
            (head: _statement(), mut tail: _program()) => {
                tail.push_front(head);

                tail
            },
            () => {
                VecDeque::new()
            },
        }

        _statement(&self) -> Statement {
            (&text: comment) => Statement::Comment(text.to_owned()),
            (_: declaration, pattern: _pattern(), type_def: _type_def(), expr: _optional_expr(), _: semi) => {
                Statement::Declaration {pattern: pattern, type_def: type_def, expr: expr}
            },
        }

        _pattern(&self) -> Pattern {
            (_: pattern, ident: _identifier()) => {
                Pattern::Identifier(ident)
            },
        }

        _type_def(&self) -> TypeDefinition {
            (ident: _identifier()) => {
                TypeDefinition::Name {name: ident}
            },
            (_: array_type, type_def: _type_def(), _: semi, size: _array_size()) => {
                TypeDefinition::Array {type_def: Box::new(type_def), size: size}
            },
        }

        _array_size(&self) -> Option<Number> {
            (_: unspecified) => {
                None
            },
            (n: _number()) => {
                Some(n)
            }
        }

        _optional_expr(&self) -> Option<Expression> {
            (e: _expr()) => Some(e),
            () => None,
        }

        _expr(&self) -> Expression {
            () => unimplemented!(),
        }

        _identifier(&self) -> Identifier {
            (&ident: identifier) => {
                ident.into()
            }
        }

        _number(&self) -> Number {
            (&s: number) => {
                // If our grammar is correct, we are guarenteed that this will work
                s.parse().unwrap()
            }
        }
    }
}
<|endoftext|>


<commit_before>use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span, tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

#[cfg(feature="with-rustc-serialize")]
fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::DelimToken;

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_pat_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}

#[cfg(feature="with-serde")]
fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::DelimToken;

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket,
                                                comma_sep!(),
                                                |p| Ok(parse_json(cx, p)))
                .ok()
                .unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                serde_json::Value::Array(xs.into_vec())
            })
        }
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            })
                .ok()
                .unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::serde_json::Value::Object(_ob)
            });
            expr
        }
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                ::serde_json::to_value(&$expr)
            }})
        }
        &Token::Ident(id) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, {
                ::serde_json::Value::Null
            })
        }
        _ => {
            // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_pat_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                ::serde_json::to_value(&$expr)
            }})
        }
    }
}
<commit_msg>Use `TokenTree` from its new module.<commit_after>use syntax::codemap::Span;
use syntax::ptr::P;
use syntax::tokenstream::TokenTree;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span, tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

#[cfg(feature="with-rustc-serialize")]
fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::DelimToken;

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_pat_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}

#[cfg(feature="with-serde")]
fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::DelimToken;

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket,
                                                comma_sep!(),
                                                |p| Ok(parse_json(cx, p)))
                .ok()
                .unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                serde_json::Value::Array(xs.into_vec())
            })
        }
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            })
                .ok()
                .unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::serde_json::Value::Object(_ob)
            });
            expr
        }
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                ::serde_json::to_value(&$expr)
            }})
        }
        &Token::Ident(id) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, {
                ::serde_json::Value::Null
            })
        }
        _ => {
            // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_pat_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                ::serde_json::to_value(&$expr)
            }})
        }
    }
}
<|endoftext|>


<commit_before>use std::io::Read;
use serde::de::Deserialize;
use serde_protobuf::descriptor::Descriptors;
use serde_protobuf::de::Deserializer;
use serde_protobuf::value::Message;
use serde_value::Value;
use protobuf::{CodedInputStream, parse_from_reader};

static UNKNOWN_FDSET: &'static [u8] = b"
*
\runknown.protoxyz.sevag.pqrs\"
Unknown";

pub fn process_single(read: &mut Read) {
    let proto = parse_from_reader(&mut UNKNOWN_FDSET.clone()).unwrap();
    let descriptors = Descriptors::from_proto(&proto);

    let byte_is = CodedInputStream::new(read);

    let mut deserializer = Deserializer::for_named_message(&descriptors, ".xyz.sevag.pqrs.Unknown", byte_is).unwrap();
    let value = Value::deserialize(&mut deserializer).unwrap();
    println!("{:?}", value);
}

pub fn process_stream(read: &mut Read) {
    let mut stream = CodedInputStream::new(read);

    loop {
        match stream.eof() {
            Err(e) => panic!(e),
            Ok(true) => break,
            Ok(false) => break,
            //todo: actually do deserialization here
        }
    }
}
<commit_msg>Apparently it's impossible to decode without fdsets<commit_after>use std::env;
use std::fs::File;
use std::io::Read;
use serde::de::Deserialize;
use serde_protobuf::descriptor::Descriptors;
use serde_protobuf::de::Deserializer;
use serde_value::Value;
use protobuf::{CodedInputStream, parse_from_reader};

pub fn process_single(read: &mut Read) {
    let proto = parse_from_reader(&mut open_combined_fdset()).unwrap();
    let descriptors = Descriptors::from_proto(&proto);
    let byte_is = CodedInputStream::new(read);

    let mut deserializer = Deserializer::for_named_message(&descriptors, ".com.example.dog.Dog", byte_is).unwrap();
    let value = Value::deserialize(&mut deserializer).unwrap();
    println!("{:?}", value);
}

pub fn process_stream(read: &mut Read) {
    let mut stream = CodedInputStream::new(read);

    loop {
        match stream.eof() {
            Err(e) => panic!(e),
            Ok(true) => break,
            Ok(false) => break,
            //todo: actually do deserialization here
        }
    }
}

fn open_combined_fdset() -> File {
    let mut home = match env::home_dir() {
        Some(home) => home,
        None => panic!("Could not find $HOME"),
    };

    home.push(".pq/combined.fdset");

    match File::open(home.as_path()) {
        Ok(x) => return x,
        Err(e) => panic!(e),
    }
}
<|endoftext|>