<commit_before>
use pair::Pair;
use token::{Token, TokenKind};

use std::error::Error;

#[derive(Debug)]
pub struct Parser {
    current: usize,
    path: Vec<String>,
    indent: usize,
    tokens: Vec<Token>,
    pair: Pair,
    line: u64,
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: Pair::new("__top_level__"),
            line: 1,
        }
    }

    pub fn new_with(tokens: Vec<Token>, pair: Pair) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: pair,
            line: 1
        }
    }

    // faked you out with that Scanner, didn't I?
    // you thought this was going to be recursive descent. YOU WERE WRONG!
    pub fn parse(mut self) -> Result<Pair, Vec<Box<Error>>> {
        let mut i = 0;

        while i < self.tokens.len() {
            match self.tokens[i].kind {
                TokenKind::Value => { // add to path respective of self.index
                    if self.indent <= self.path.len() {
                        let mut new = self.path[0..self.indent].to_owned();
                        new.push(self.tokens[i].lexeme.clone());
                        self.path = new;
                    } else {
                        self.path.push(self.tokens[i].lexeme.clone());
                    }

                    self.pair.add_slice(&self.path);

                    if i + 2 <= self.tokens.len() && self.tokens[i + 2].kind == TokenKind::Value {
                        self.path.clear();
                        self.indent = 0;
                    }
                },

                TokenKind::Indent => { // set new self.index
                    let mut indent = 0;

                    while self.tokens[i].kind == TokenKind::Indent {
                        indent += 1;
                        i += 1;
                    }

                    i -= 1;

                    self.indent = indent;
                },

                TokenKind::Newline => { // reset self.index
                    self.indent = 0;
                    self.line += 1;
                },

                TokenKind::EOF => break,
            }
            i += 1;
        }

        Ok(self.pair)
    }
}

<commit_msg>error if incorrect indentation<commit_after>
use pair::Pair;
use token::{Token, TokenKind};
use error::{NcclError, ErrorKind};

use std::error::Error;

#[derive(Debug)]
pub struct Parser {
    current: usize,
    path: Vec<String>,
    indent: usize,
    tokens: Vec<Token>,
    pair: Pair,
    line: u64,
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: Pair::new("__top_level__"),
            line: 1,
        }
    }

    pub fn new_with(tokens: Vec<Token>, pair: Pair) -> Self {
        Parser {
            current: 0,
            path: vec![],
            indent: 0,
            tokens: tokens,
            pair: pair,
            line: 1
        }
    }

    // faked you out with that Scanner, didn't I?
    // you thought this was going to be recursive descent. YOU WERE WRONG!
    pub fn parse(mut self) -> Result<Pair, Vec<Box<Error>>> {
        let mut errors: Vec<Box<Error>> = vec![];
        let mut prev_indent = 0;
        let mut i = 0;

        while i < self.tokens.len() {
            match self.tokens[i].kind {
                TokenKind::Value => { // add to path respective of self.index
                    if self.indent <= self.path.len() {
                        let mut new = self.path[0..self.indent].to_owned();
                        new.push(self.tokens[i].lexeme.clone());
                        self.path = new;
                    } else {
                        self.path.push(self.tokens[i].lexeme.clone());
                    }

                    self.pair.add_slice(&self.path);

                    if i + 2 <= self.tokens.len() && self.tokens[i + 2].kind == TokenKind::Value {
                        self.path.clear();
                        self.indent = 0;
                    }
                },

                TokenKind::Indent => { // set new self.index
                    let mut indent = 0;

                    while self.tokens[i].kind == TokenKind::Indent {
                        indent += 1;
                        i += 1;
                    }

                    i -= 1;

                    if prev_indent > indent {
                        if prev_indent - indent == 1 || prev_indent - indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    } else {
                        if indent - prev_indent == 1 || indent - prev_indent == 0 {
                            self.indent = indent;
                        } else {
                            errors.push(Box::new(NcclError::new(ErrorKind::IndentationError, "Incorrect level of indentation found", self.line)));
                            self.indent = prev_indent;
                        }
                    }
                },

                TokenKind::Newline => { // reset self.index
                    prev_indent = self.indent;
                    self.indent = 0;
                    self.line += 1;
                },

                TokenKind::EOF => break,
            }
            i += 1;
        }

        if errors.len() == 0 {
            Ok(self.pair)
        } else {
            Err(errors)
        }
    }
}

<|endoftext|>


<commit_before>use ast::*;

#[derive(Clone, Show)]
pub struct Parser {
    code: String,
    cur: usize,
    col: usize,
    line: usize
}

#[allow(dead_code)]
pub struct ParserError {
    line: usize,
    col: usize,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: usize, col: usize, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0, cur: 0, col: 0
        }
    }

    pub fn load(&mut self, code: String) {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
    }

    //============= private methods =================
    pub fn read_exp(&mut self) -> Option<ExprAst> {
        if self.eof() { return None }
        self.skip_space();
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return Some(ExprAst::Bool(BoolNode::new(true))),
                'f' => return Some(ExprAst::Bool(BoolNode::new(false))),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if cur.is_numeric() ||
            (cur == '-' && (self.peekc().is_numeric())) {
                let mut sign = 1is;
                let mut num = 0is;
                if cur == '-' {
                    sign = -1;
                } else {
                    self.unread();
                }
                loop {
                    cur = self.readc();
                    if !cur.is_numeric() {
                        break;
                    }
                    num = (num * 10is) + (cur as isize - '0' as isize);
                }
                num *= sign;
                if self.is_delimiter(cur) {
                    self.unread();
                }
                return Some(ExprAst::Int(IntNode::new(num)));
            } else if cur == '\"' {
                let mut buf = String::new();
                loop {
                    cur = self.readc();
                    if cur == '\"' {
                        break;
                    }
                    buf.push(cur);
                }
                return Some(ExprAst::Str(StrNode::new(buf.as_slice())));
            } else if cur == '(' {
                return self.read_pair();
            } else if self.is_initial(cur) {
                let mut buf = String::new();
                buf.push(cur);
                loop {
                    cur = self.readc();
                    if !(self.is_initial(cur) || cur.is_numeric()) {
                        break;
                    }
                    buf.push(cur);
                }
                if self.is_delimiter(cur) {
                    self.unread();
                }
                return Some(ExprAst::Symbol(SymbolNode::new(buf.as_slice())));
            } else if cur == '\'' {
                let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
                let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp().unwrap(),
                                                            ExprAst::Nil));
                return Some(ExprAst::Pair(PairNode::new(quote_sym, quote_exp)));
            }
        None
    }

    fn read_pair(&mut self) -> Option<ExprAst> {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return Some(ExprAst::Nil);
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return Some(ExprAst::Pair(PairNode::new(car_obj.unwrap(),
                                                    cdr_obj.unwrap())));
        } else {
            return Some(ExprAst::Nil);
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';'
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn eof(&self) -> bool {
        self.cur >= self.code.len()
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            return 0 as char
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> Option<ExprAst> {
        Some(ExprAst::Char(CharNode::new('a')))
    }
}

#[test]
fn test_parser() {
    // let mut parser = Parser::new();
    // let res = parser.load("11".to_string());
    // assert!(res.as_int() == 11);
    // res.print();

    // let res = parser.load("-11".to_string());
    // assert!(res.as_int() == -11);
    // res.print();

    // let res = parser.load(r#""hello""#.to_string());
    // assert!(res.as_str() == "hello");
    // res.print();

    // let res = parser.load("()".to_string());
    // assert!(res.is_empty());
    // res.print();

    // let res = parser.load("(1 2)".to_string());
    // assert!(res.is_pair());
    // assert!(res.car().as_int() == 1);
    // assert!(res.cdr().car().as_int() == 2);
    // assert!(res.cdr().cdr().is_empty());

    // let res = parser.load("(+ 1 2)".to_string());
    // assert!(res.is_pair());
    // assert!(res.car().is_symbol());
    // assert!(res.cdr().car().as_int() == 1);
    // assert!(res.cdr().cdr().car().as_int() == 2);
}
<commit_msg>refactor parser testing<commit_after>use ast::*;

#[derive(Clone, Show)]
pub struct Parser {
    code: String,
    cur: usize,
    col: usize,
    line: usize
}

#[allow(dead_code)]
pub struct ParserError {
    line: usize,
    col: usize,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: usize, col: usize, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0, cur: 0, col: 0
        }
    }

    pub fn load(&mut self, code: String) {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
    }

    //============= private methods =================
    pub fn read_exp(&mut self) -> Option<ExprAst> {
        if self.eof() { return None }
        self.skip_space();
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return Some(ExprAst::Bool(BoolNode::new(true))),
                'f' => return Some(ExprAst::Bool(BoolNode::new(false))),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if cur.is_numeric() ||
            (cur == '-' && (self.peekc().is_numeric())) {
                let mut sign = 1is;
                let mut num = 0is;
                if cur == '-' {
                    sign = -1;
                } else {
                    self.unread();
                }
                loop {
                    cur = self.readc();
                    if !cur.is_numeric() {
                        break;
                    }
                    num = (num * 10is) + (cur as isize - '0' as isize);
                }
                num *= sign;
                if self.is_delimiter(cur) {
                    self.unread();
                }
                return Some(ExprAst::Int(IntNode::new(num)));
            } else if cur == '\"' {
                let mut buf = String::new();
                loop {
                    cur = self.readc();
                    if cur == '\"' {
                        break;
                    }
                    buf.push(cur);
                }
                return Some(ExprAst::Str(StrNode::new(buf.as_slice())));
            } else if cur == '(' {
                return self.read_pair();
            } else if self.is_initial(cur) {
                let mut buf = String::new();
                buf.push(cur);
                loop {
                    cur = self.readc();
                    if !(self.is_initial(cur) || cur.is_numeric()) {
                        break;
                    }
                    buf.push(cur);
                }
                if self.is_delimiter(cur) {
                    self.unread();
                }
                return Some(ExprAst::Symbol(SymbolNode::new(buf.as_slice())));
            } else if cur == '\'' {
                let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
                let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp().unwrap(),
                                                            ExprAst::Nil));
                return Some(ExprAst::Pair(PairNode::new(quote_sym, quote_exp)));
            }
        None
    }

    fn read_pair(&mut self) -> Option<ExprAst> {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return Some(ExprAst::Nil);
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return Some(ExprAst::Pair(PairNode::new(car_obj.unwrap(),
                                                    cdr_obj.unwrap())));
        } else {
            return Some(ExprAst::Nil);
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';'
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn eof(&self) -> bool {
        self.cur >= self.code.len()
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            return 0 as char
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> Option<ExprAst> {
        Some(ExprAst::Char(CharNode::new('a')))
    }
}

#[test]
fn test_parser() {
    macro_rules! test_case {
        ($test_str:expr, $expect_type:ident, $expect_val:expr) => { {
            let mut parser = Parser::new();
            parser.load($test_str.to_string());
            let res = parser.read_exp().unwrap();
            if res.$expect_type() != $expect_val {
                assert!(false);
            }
        }}
    }

    macro_rules! test_res {
        ($test_str:expr) => { {
            let mut parser = Parser::new();
            parser.load($test_str.to_string());
            parser.read_exp().unwrap()
        }}
    }

    test_case!("11", as_int, 11);
    test_case!("-11", as_int, -11);
    test_case!(r#""hello""#, as_str, "hello");

    let res = test_res!("()");
    assert!(res.is_empty());

    let res = test_res!("(1 2)");
    assert!(res.is_pair());
    assert!(res.car().as_int() == 1);
    assert!(res.cdr().car().as_int() == 2);
    assert!(res.cdr().cdr().is_empty());

    let res = test_res!("(+ 1 2)");
    assert!(res.is_pair());
    assert!(res.car().is_symbol());
    assert!(res.cdr().car().as_int() == 1);
    assert!(res.cdr().cdr().car().as_int() == 2);
}
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::Tokenizer;

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

#[derive(PartialEq, Debug)]
pub enum ParseError {
    SyntaxError,
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        Err(ParseError::SyntaxError)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
defineUnit "km/h"

my_var = 30 km/h
f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
print (f 30)

operator INFIX 6 "$$" (\x y = x * y + x)
"#);
        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Better formatting in test<commit_after>use token::Token;
use tokenizer::Tokenizer;

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

#[derive(PartialEq, Debug)]
pub enum ParseError {
    SyntaxError,
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        Err(ParseError::SyntaxError)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>


<commit_before>use std::mem;

use num::bigint::{self, ToBigInt};
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use consts::*;
use error::{Result, BertError};

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if !self.eof() {
            return Err(BertError::ExtraData(self.pos));
        }
        return Ok(term);
    }

    pub fn parse_bert2(&mut self) -> Result<Vec<BertTerm>> {
        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let _ = self.parse_varint()?;
            let _ = self.magic_number()?;
            let t = self.bert_term()?;
            terms.push(t);
        }
        return Ok(terms);
    }

    pub fn parse_disk_log(&mut self) -> Result<Vec<BertTerm>> {
        self.disk_log_magic()?;
        self.disk_log_opened_status()?;

        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let t = self.disk_log_term()?;
            terms.push(t);
        }
        return Ok(terms);
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(offset));
        }
        return Ok(());
    }

    fn disk_log_magic(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u32_be()?;
        if magic != DISK_LOG_MAGIC {
            return Err(BertError::InvalidMagicNumber(offset));
        }
        return Ok(());
    }

    fn disk_log_opened_status(&mut self) -> Result<()> {
        let offset = self.pos;
        let status = self.eat_u32_be()?;
        if status != DISK_LOG_OPENED && status != DISK_LOG_CLOSED {
            return Err(BertError::InvalidDiskLogOpenedStatus(offset));
        }
        return Ok(());
    }

    fn disk_log_term(&mut self) -> Result<BertTerm> {
        // XXX(vfoley): should we check that the correct length was read?
        let _len_offset = self.pos;
        let _len = self.eat_u32_be()?;

        let magic_offset = self.pos;
        let magic = self.eat_u32_be()?;
        if magic != DISK_LOG_TERM_MAGIC {
            return Err(BertError::InvalidMagicNumber(magic_offset));
        }

        let magic_offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(magic_offset));
        }

        return self.bert_term();
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        match self.eat_u8()? {
            SMALL_INTEGER_EXT => { self.small_integer() }
            INTEGER_EXT => { self.integer() }
            FLOAT_EXT => { self.old_float() }
            NEW_FLOAT_EXT => { self.new_float() }
            ATOM_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom(len)
            }
            SMALL_ATOM_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom(len)
            }
            ATOM_UTF8_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom_utf8(len)
            }
            SMALL_ATOM_UTF8_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom_utf8(len)
            }
            SMALL_TUPLE_EXT => {
                let len = self.eat_u8()? as usize;
                self.tuple(len)
            }
            LARGE_TUPLE_EXT => {
                let len = self.eat_u32_be()? as usize;
                self.tuple(len)
            }
            NIL_EXT => { Ok(BertTerm::Nil) }
            LIST_EXT => { self.list() }
            STRING_EXT => { self.string() }
            BINARY_EXT => { self.binary() }
            SMALL_BIG_EXT => {
                let len = self.eat_u8()?;
                self.bigint(len as usize)
            }
            LARGE_BIG_EXT => {
                let len = self.eat_u32_be()?;
                self.bigint(len as usize)
            }
            MAP_EXT => {
                self.map()
            }
            tag => { Err(BertError::InvalidTag(offset, tag)) }
        }
    }

    fn small_integer(&mut self) -> Result<BertTerm> {
        let b = self.eat_u8()?;
        Ok(BertTerm::Int(b as i32))
    }

    fn integer(&mut self) -> Result<BertTerm> {
        let n = self.eat_i32_be()?;
        Ok(BertTerm::Int(n))
    }

    fn old_float(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        let mut s = String::new();
        while !self.eof() && self.peek()? != 0 {
            s.push(self.eat_char()?);
        }

        while !self.eof() && self.peek()? == 0 {
            let _ = self.eat_u8()?;
        }

        s.parse::<f64>()
            .map_err(|_| BertError::InvalidFloat(offset))
            .map(|f| BertTerm::Float(f))
    }

    fn new_float(&mut self) -> Result<BertTerm> {
        let raw_bytes = self.eat_u64_be()?;
        let f: f64 = f64::from_bits(raw_bytes);
        Ok(BertTerm::Float(f))
    }

    fn atom(&mut self, len: usize) -> Result<BertTerm> {
        let initial_pos = self.pos;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        let is_ascii = bytes.iter().all(|byte| *byte < 128);

        // Optimization: ASCII atoms represent the overwhelming
        // majority of use cases of atoms. When we read the bytes
        // of the atom, we record whether they are all ASCII
        // (i.e., small than 128); if it's the case, we don't
        // need to bother with latin-1 decoding. We use an unsafe
        // method because ASCII strings are guaranteed to be valid
        // UTF-8 strings.
        if is_ascii {
            let s = unsafe { String::from_utf8_unchecked(bytes) };
            Ok(BertTerm::Atom(s))
        } else {
            ISO_8859_1.decode(&bytes, DecoderTrap::Strict)
                .map(|s| BertTerm::Atom(s))
                .map_err(|_| BertError::InvalidLatin1Atom(initial_pos))
        }
    }

    fn atom_utf8(&mut self, len: usize) -> Result<BertTerm> {
        let initial_pos = self.pos;
        let buf: Vec<u8> = self.eat_slice(len)?.to_owned();
        String::from_utf8(buf)
            .map(|s| BertTerm::Atom(s))
            .map_err(|_| BertError::InvalidUTF8Atom(initial_pos))
    }

    fn tuple(&mut self, len: usize) -> Result<BertTerm> {
        let mut terms = Vec::with_capacity(len);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        Ok(BertTerm::Tuple(terms))
    }

    fn string(&mut self) -> Result<BertTerm> {
        let len = self.eat_u16_be()? as usize;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        Ok(BertTerm::String(bytes))
    }

    fn binary(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        Ok(BertTerm::Binary(bytes))
    }

    fn list(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()?;
        let mut terms = Vec::with_capacity(len as usize + 1);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        let tail = self.bert_term()?;
        match tail {
            BertTerm::Nil => (),
            last_term => { terms.push(last_term); }
        };
        Ok(BertTerm::List(terms))
    }

    fn bigint(&mut self, len: usize) -> Result<BertTerm> {
        let sign = self.eat_u8()?;
        let mut sum: bigint::BigInt = Zero::zero();
        let mut pos: bigint::BigInt = One::one();
        for _ in 0 .. len {
            let d = self.eat_u8()?;
            let t = &pos * &(d.to_bigint().unwrap());
            sum = sum + &t;
            pos = pos * (256).to_bigint().unwrap();
        }
        if sign == 1 {
            sum = -sum;
        }
        Ok(BertTerm::BigInt(sum))
    }

    // TODO(vfoley): ensure no duplicate keys
    fn map(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let mut keys = Vec::with_capacity(len);
        let mut vals = Vec::with_capacity(len);
        for _ in 0 .. len {
            keys.push(self.bert_term()?);
            vals.push(self.bert_term()?);
        }
        Ok(BertTerm::Map(keys, vals))
    }

    // Low-level parsing methods
    fn eof(&self) -> bool {
        self.pos >= self.contents.len()
    }

    fn peek(&self) -> Result<u8> {
        if self.eof() {
            return Err(BertError::NotEnoughData {
                offset: self.pos,
                needed: 1,
                available: 0
            });
        } else {
            return Ok(self.contents[self.pos]);
        }
    }

    fn can_read(&self, n: usize) -> bool {
        (self.pos + n) <= self.contents.len()
    }

    fn eat_slice(&mut self, len: usize) -> Result<&[u8]> {
        if !self.can_read(len) {
            return Err(BertError::NotEnoughData {
                offset: self.pos,
                needed: len,
                available: self.contents.len() - self.pos
            })
        }
        let slice = &self.contents[self.pos .. self.pos + len];
        self.pos += len;
        return Ok(slice);
    }

    fn eat_u8(&mut self) -> Result<u8> {
        let b = self.peek()?;
        self.pos += 1;
        return Ok(b);
    }

    fn eat_char(&mut self) -> Result<char> {
        let b = self.eat_u8()?;
        return Ok(b as char);
    }

    fn eat_u16_be(&mut self) -> Result<u16> {
        let bytes = self.eat_slice(2)?;
        let b0 = bytes[0] as u16;
        let b1 = bytes[1] as u16;
        return Ok((b0 << 8) + b1)
    }

    fn eat_i32_be(&mut self) -> Result<i32> {
        let bytes = self.eat_slice(4)?;
        let b0 = bytes[0] as i32;
        let b1 = bytes[1] as i32;
        let b2 = bytes[2] as i32;
        let b3 = bytes[3] as i32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u32_be(&mut self) -> Result<u32> {
        let bytes = self.eat_slice(4)?;
        let b0 = bytes[0] as u32;
        let b1 = bytes[1] as u32;
        let b2 = bytes[2] as u32;
        let b3 = bytes[3] as u32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u64_be(&mut self) -> Result<u64> {
        let bytes = self.eat_slice(8)?;
        let b0 = bytes[0] as u64;
        let b1 = bytes[1] as u64;
        let b2 = bytes[2] as u64;
        let b3 = bytes[3] as u64;
        let b4 = bytes[4] as u64;
        let b5 = bytes[5] as u64;
        let b6 = bytes[6] as u64;
        let b7 = bytes[7] as u64;
        return Ok((b0 << 56) + (b1 << 48) + (b2 << 40) + (b3 << 32) +
                  (b4 << 24) + (b5 << 16) + (b6 << 8) + b7);
    }

    // https://developers.google.com/protocol-buffers/docs/encoding#varints
    fn parse_varint(&mut self) -> Result<u64> {
        let start_pos = self.pos;

        let mut bytes = Vec::with_capacity(mem::size_of::<u64>());
        let mut i = 0;

        while !self.eof() && i < bytes.capacity() {
            let b = self.eat_u8()?;
            bytes.push(b);
            if b & 0x80 == 0 {
                break;
            }
            i += 1;
        }

        if i >= bytes.capacity() {
            return Err(BertError::VarintTooLarge(start_pos));
        }

        let mut x: u64 = 0;
        for (i, byte) in bytes.iter().rev().enumerate() {
            x = (x << (7*i) as u64) | (*byte as u64 & 0x7f);
        }

        return Ok(x);
    }
}

#[test]
fn test_varint() {
    assert_eq!(1, {
        match Parser::new(vec![1]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert_eq!(300, {
        match Parser::new(vec![0b1010_1100, 0b0000_0010]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x7f]).parse_varint().is_ok());
    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x80]).parse_varint().is_err());
}
<commit_msg>Rename offset to initial_pos<commit_after>use std::mem;

use num::bigint::{self, ToBigInt};
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use consts::*;
use error::{Result, BertError};

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if !self.eof() {
            return Err(BertError::ExtraData(self.pos));
        }
        return Ok(term);
    }

    pub fn parse_bert2(&mut self) -> Result<Vec<BertTerm>> {
        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let _ = self.parse_varint()?;
            let _ = self.magic_number()?;
            let t = self.bert_term()?;
            terms.push(t);
        }
        return Ok(terms);
    }

    pub fn parse_disk_log(&mut self) -> Result<Vec<BertTerm>> {
        self.disk_log_magic()?;
        self.disk_log_opened_status()?;

        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let t = self.disk_log_term()?;
            terms.push(t);
        }
        return Ok(terms);
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let initial_pos = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(initial_pos));
        }
        return Ok(());
    }

    fn disk_log_magic(&mut self) -> Result<()> {
        let initial_pos = self.pos;
        let magic = self.eat_u32_be()?;
        if magic != DISK_LOG_MAGIC {
            return Err(BertError::InvalidMagicNumber(initial_pos));
        }
        return Ok(());
    }

    fn disk_log_opened_status(&mut self) -> Result<()> {
        let initial_pos = self.pos;
        let status = self.eat_u32_be()?;
        if status != DISK_LOG_OPENED && status != DISK_LOG_CLOSED {
            return Err(BertError::InvalidDiskLogOpenedStatus(initial_pos));
        }
        return Ok(());
    }

    fn disk_log_term(&mut self) -> Result<BertTerm> {
        // XXX(vfoley): should we check that the correct length was read?
        let _len_offset = self.pos;
        let _len = self.eat_u32_be()?;

        let magic_pos = self.pos;
        let magic = self.eat_u32_be()?;
        if magic != DISK_LOG_TERM_MAGIC {
            return Err(BertError::InvalidMagicNumber(magic_pos));
        }

        let magic_pos = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(magic_pos));
        }

        return self.bert_term();
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let initial_pos = self.pos;
        match self.eat_u8()? {
            SMALL_INTEGER_EXT => { self.small_integer() }
            INTEGER_EXT => { self.integer() }
            FLOAT_EXT => { self.old_float() }
            NEW_FLOAT_EXT => { self.new_float() }
            ATOM_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom(len)
            }
            SMALL_ATOM_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom(len)
            }
            ATOM_UTF8_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom_utf8(len)
            }
            SMALL_ATOM_UTF8_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom_utf8(len)
            }
            SMALL_TUPLE_EXT => {
                let len = self.eat_u8()? as usize;
                self.tuple(len)
            }
            LARGE_TUPLE_EXT => {
                let len = self.eat_u32_be()? as usize;
                self.tuple(len)
            }
            NIL_EXT => { Ok(BertTerm::Nil) }
            LIST_EXT => { self.list() }
            STRING_EXT => { self.string() }
            BINARY_EXT => { self.binary() }
            SMALL_BIG_EXT => {
                let len = self.eat_u8()?;
                self.bigint(len as usize)
            }
            LARGE_BIG_EXT => {
                let len = self.eat_u32_be()?;
                self.bigint(len as usize)
            }
            MAP_EXT => {
                self.map()
            }
            tag => { Err(BertError::InvalidTag(initial_pos, tag)) }
        }
    }

    fn small_integer(&mut self) -> Result<BertTerm> {
        let b = self.eat_u8()?;
        Ok(BertTerm::Int(b as i32))
    }

    fn integer(&mut self) -> Result<BertTerm> {
        let n = self.eat_i32_be()?;
        Ok(BertTerm::Int(n))
    }

    fn old_float(&mut self) -> Result<BertTerm> {
        let initial_pos = self.pos;
        let mut s = String::new();
        while !self.eof() && self.peek()? != 0 {
            s.push(self.eat_char()?);
        }

        while !self.eof() && self.peek()? == 0 {
            let _ = self.eat_u8()?;
        }

        s.parse::<f64>()
            .map_err(|_| BertError::InvalidFloat(initial_pos))
            .map(|f| BertTerm::Float(f))
    }

    fn new_float(&mut self) -> Result<BertTerm> {
        let raw_bytes = self.eat_u64_be()?;
        let f: f64 = f64::from_bits(raw_bytes);
        Ok(BertTerm::Float(f))
    }

    fn atom(&mut self, len: usize) -> Result<BertTerm> {
        let initial_pos = self.pos;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        let is_ascii = bytes.iter().all(|byte| *byte < 128);

        // Optimization: ASCII atoms represent the overwhelming
        // majority of use cases of atoms. When we read the bytes
        // of the atom, we record whether they are all ASCII
        // (i.e., small than 128); if it's the case, we don't
        // need to bother with latin-1 decoding. We use an unsafe
        // method because ASCII strings are guaranteed to be valid
        // UTF-8 strings.
        if is_ascii {
            let s = unsafe { String::from_utf8_unchecked(bytes) };
            Ok(BertTerm::Atom(s))
        } else {
            ISO_8859_1.decode(&bytes, DecoderTrap::Strict)
                .map(|s| BertTerm::Atom(s))
                .map_err(|_| BertError::InvalidLatin1Atom(initial_pos))
        }
    }

    fn atom_utf8(&mut self, len: usize) -> Result<BertTerm> {
        let initial_pos = self.pos;
        let buf: Vec<u8> = self.eat_slice(len)?.to_owned();
        String::from_utf8(buf)
            .map(|s| BertTerm::Atom(s))
            .map_err(|_| BertError::InvalidUTF8Atom(initial_pos))
    }

    fn tuple(&mut self, len: usize) -> Result<BertTerm> {
        let mut terms = Vec::with_capacity(len);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        Ok(BertTerm::Tuple(terms))
    }

    fn string(&mut self) -> Result<BertTerm> {
        let len = self.eat_u16_be()? as usize;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        Ok(BertTerm::String(bytes))
    }

    fn binary(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let bytes: Vec<u8> = self.eat_slice(len)?.to_owned();
        Ok(BertTerm::Binary(bytes))
    }

    fn list(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()?;
        let mut terms = Vec::with_capacity(len as usize + 1);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        let tail = self.bert_term()?;
        match tail {
            BertTerm::Nil => (),
            last_term => { terms.push(last_term); }
        };
        Ok(BertTerm::List(terms))
    }

    fn bigint(&mut self, len: usize) -> Result<BertTerm> {
        let sign = self.eat_u8()?;
        let mut sum: bigint::BigInt = Zero::zero();
        let mut pos: bigint::BigInt = One::one();
        for _ in 0 .. len {
            let d = self.eat_u8()?;
            let t = &pos * &(d.to_bigint().unwrap());
            sum = sum + &t;
            pos = pos * (256).to_bigint().unwrap();
        }
        if sign == 1 {
            sum = -sum;
        }
        Ok(BertTerm::BigInt(sum))
    }

    // TODO(vfoley): ensure no duplicate keys
    fn map(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let mut keys = Vec::with_capacity(len);
        let mut vals = Vec::with_capacity(len);
        for _ in 0 .. len {
            keys.push(self.bert_term()?);
            vals.push(self.bert_term()?);
        }
        Ok(BertTerm::Map(keys, vals))
    }

    // Low-level parsing methods
    fn eof(&self) -> bool {
        self.pos >= self.contents.len()
    }

    fn peek(&self) -> Result<u8> {
        if self.eof() {
            return Err(BertError::NotEnoughData {
                offset: self.pos,
                needed: 1,
                available: 0
            });
        } else {
            return Ok(self.contents[self.pos]);
        }
    }

    fn can_read(&self, n: usize) -> bool {
        (self.pos + n) <= self.contents.len()
    }

    fn eat_slice(&mut self, len: usize) -> Result<&[u8]> {
        if !self.can_read(len) {
            return Err(BertError::NotEnoughData {
                offset: self.pos,
                needed: len,
                available: self.contents.len() - self.pos
            })
        }
        let slice = &self.contents[self.pos .. self.pos + len];
        self.pos += len;
        return Ok(slice);
    }

    fn eat_u8(&mut self) -> Result<u8> {
        let b = self.peek()?;
        self.pos += 1;
        return Ok(b);
    }

    fn eat_char(&mut self) -> Result<char> {
        let b = self.eat_u8()?;
        return Ok(b as char);
    }

    fn eat_u16_be(&mut self) -> Result<u16> {
        let bytes = self.eat_slice(2)?;
        let b0 = bytes[0] as u16;
        let b1 = bytes[1] as u16;
        return Ok((b0 << 8) + b1)
    }

    fn eat_i32_be(&mut self) -> Result<i32> {
        let bytes = self.eat_slice(4)?;
        let b0 = bytes[0] as i32;
        let b1 = bytes[1] as i32;
        let b2 = bytes[2] as i32;
        let b3 = bytes[3] as i32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u32_be(&mut self) -> Result<u32> {
        let bytes = self.eat_slice(4)?;
        let b0 = bytes[0] as u32;
        let b1 = bytes[1] as u32;
        let b2 = bytes[2] as u32;
        let b3 = bytes[3] as u32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u64_be(&mut self) -> Result<u64> {
        let bytes = self.eat_slice(8)?;
        let b0 = bytes[0] as u64;
        let b1 = bytes[1] as u64;
        let b2 = bytes[2] as u64;
        let b3 = bytes[3] as u64;
        let b4 = bytes[4] as u64;
        let b5 = bytes[5] as u64;
        let b6 = bytes[6] as u64;
        let b7 = bytes[7] as u64;
        return Ok((b0 << 56) + (b1 << 48) + (b2 << 40) + (b3 << 32) +
                  (b4 << 24) + (b5 << 16) + (b6 << 8) + b7);
    }

    // https://developers.google.com/protocol-buffers/docs/encoding#varints
    fn parse_varint(&mut self) -> Result<u64> {
        let start_pos = self.pos;

        let mut bytes = Vec::with_capacity(mem::size_of::<u64>());
        let mut i = 0;

        while !self.eof() && i < bytes.capacity() {
            let b = self.eat_u8()?;
            bytes.push(b);
            if b & 0x80 == 0 {
                break;
            }
            i += 1;
        }

        if i >= bytes.capacity() {
            return Err(BertError::VarintTooLarge(start_pos));
        }

        let mut x: u64 = 0;
        for (i, byte) in bytes.iter().rev().enumerate() {
            x = (x << (7*i) as u64) | (*byte as u64 & 0x7f);
        }

        return Ok(x);
    }
}

#[test]
fn test_varint() {
    assert_eq!(1, {
        match Parser::new(vec![1]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert_eq!(300, {
        match Parser::new(vec![0b1010_1100, 0b0000_0010]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x7f]).parse_varint().is_ok());
    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x80]).parse_varint().is_err());
}
<|endoftext|>


<commit_before>use std::fmt::Show;
use peruse::parsers::*;
use grammar::*;

pub type LParser<'a, T> = Box<Parser<'a, &'a [Token], T> + 'a>;

fn assign<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(variable(), literal(Equals), expr()),
    |&: (var,( _, expr))| match var{
      Variable(name) => Assign(name, expr),
      _ => unreachable!()
    }
  )
}

fn output<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(literal(OutputCmd), expr()),
    |&: (_, var): (Token, Expr)| Output(var)
  )
}

fn if_stmt<'a>() -> LParser<'a, Statement> {
  //todo: optional else
  map!(
    seq!(
      literal(IfKeyword),
      expr(),
      comparator(),
      expr(),
      braced_block(),
      opt!(map!(
        seq!(literal(ElseKeyword), braced_block()),
        |&: (_, else_block)| else_block
      ))
    ),
    |&: (_, (lhs, (comp, (rhs, (then_block, else_block_opt)))))| If(lhs, comp, rhs, then_block, else_block_opt)
  )
}

fn while_stmt<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(literal(WhileKeyword), expr(), comparator(), expr(), braced_block()),
    |&: (_, (lhs, (comp, (rhs, block))))| While(lhs, comp, rhs, block)
  )
}

pub fn block<'a>() -> LParser<'a, Block> {
  map!(
    rep!(
      map!(
        seq!(
          or!(output(), if_stmt(), while_stmt(), assign()),
          literal(NewLine)
        ),
        |&: (stmt, _)| stmt
      )
    ),
    |&: stmts| Block(stmts)
  )

}

pub fn braced_block<'a>() -> LParser<'a, Block> {
  map!(
    seq!(literal(OpenBrace), block(), literal(CloseBrace)),
    |&: (_, (block, _))| block
  )
}

pub fn comparator<'a>() -> LParser<'a, Comparator> {
  box MatchParser{
    matcher: box |&: input: &Token| match *input {
      Cmp(c) => Ok(c),
      ref other => Err(format!("Expected comparator, got {}", other))
    }
  }
}

fn variable<'a>() -> LParser<'a, Expr> {
  box MatchParser{
    matcher: box |&: input: &Token| match input {
      &Ident(ref str) => Ok(Variable(str.clone())),
      other => Err(format!("Expected variable, got {}", other))
    }
  }
}

type EParser<'a> = LParser<'a, Expr>;

fn expr<'a>() -> EParser<'a> {

  fn number<'a>() -> EParser<'a> {
    box MatchParser{matcher: box |&: input: &Token| -> Result<Expr, String> match *input {
      Number(num) => Ok(Num(num)),
      ref other => Err(format!("wrong type, expected number, got {}", other))
    }}
  }

  fn term<'a>() -> EParser<'a> {
    or!(paren_expr(), variable(), number())
  }

  fn mult<'a>() -> EParser<'a> {
    map!(
      seq!(
        term(),
        rep!(seq!(or!(literal(MultSign), literal(DivideSign)), term()))
      ),
      |&: (first, rest): (Expr, Vec<(Token, Expr)>)| {
        if rest.len() == 0 {
          first
        } else {
          let mut f = Vec::new();
          f.push(MultTerm(Multiply, first));
          for &(ref sign, ref value) in rest.iter() {
            let s = match *sign {
              MultSign => Multiply,
              DivideSign => Divide,
              _ => panic!("not allowed")
            };
            f.push(MultTerm(s, value.clone())); //maybe box the value instead
          }
          MultDiv(f)
        }
      }
    )
  }

  fn simple_expr<'a>() -> EParser<'a> {
    or!(mult(), term())
  }

  fn plus<'a>() -> EParser<'a> {
    map!(
      seq!(
        simple_expr(),
        rep!(seq!(or!(literal(PlusSign), literal(MinusSign)), simple_expr()))
      ),
      |&: (first, rest): (Expr, Vec<(Token, Expr)>)| {
        if rest.len() == 0 {
          first
        } else {
          let mut f = Vec::new();
          f.push(AddTerm(Add, first));
          for &(ref sign, ref value) in rest.iter() {
            let s = match *sign {
              PlusSign => Add,
              MinusSign => Subtract,
              _ => panic!("not allowed")
            };
            f.push(AddTerm(s, value.clone()));
          }
          AddSub(f)
        }
      }
    )
  }

  fn paren_expr<'a>() -> EParser<'a> {
    map!(
      seq!(literal(OpenParen), expr(), literal(CloseParen)),
      |&: (_, (expr, _))| expr
    )
  }

  plus()
}

fn test_parser<'a, I, O: PartialEq + Show>(input: I, parser: &Parser<'a, I, O>, expected: O) {
  match parser.parse(input) {
    Ok((output, rest)) => {
      assert_eq!(output, expected);
    },
    Err(err) => panic!(err)
  }
}

#[test]
fn test_term() {
  let parser = expr();
  let input = [Number(5)];
  let expected = Num(5);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_plus_sequence() {
  let parser = expr();
  let input = [Number(5), PlusSign, Number(3)];
  let expected = AddSub(vec![AddTerm(Add, Num(5)), AddTerm(Add, Num(3))]);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_plus_mult_sequence() {
  let parser = expr();
  let input = [Number(3), PlusSign, Number(4), MultSign, Number(5)];
  let expected = AddSub(vec![AddTerm(Add, Num(3)), AddTerm(Add, MultDiv(vec![MultTerm(Multiply, Num(4)), MultTerm(Multiply, Num(5))]))]);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_simple_assign() {
  let parser = assign();
  let input = [Ident(from_str("x").unwrap()), Equals, Number(7)];
  let expected = Assign(from_str("x").unwrap(), Num(7));
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_simple_output() {
  let parser = output();
  let input = [OutputCmd, Number(4)];
  let expected = Output(Num(4));
  test_parser(input.as_slice(), &*parser, expected);
}
<commit_msg>if chaining<commit_after>use std::fmt::Show;
use peruse::parsers::*;
use grammar::*;

pub type LParser<'a, T> = Box<Parser<'a, &'a [Token], T> + 'a>;

fn assign<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(variable(), literal(Equals), expr()),
    |&: (var,( _, expr))| match var{
      Variable(name) => Assign(name, expr),
      _ => unreachable!()
    }
  )
}

fn output<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(literal(OutputCmd), expr()),
    |&: (_, var): (Token, Expr)| Output(var)
  )
}

fn if_stmt<'a>() -> LParser<'a, Statement> {
  //todo: optional else
  map!(
    seq!(
      literal(IfKeyword),
      expr(),
      comparator(),
      expr(),
      braced_block(),
      opt!(map!(
        seq!(
          literal(ElseKeyword),
          or!(
            braced_block(),
            coerce(map!(if_stmt(), |&: if_stmt| Block(vec![if_stmt]))) //else if...
          )
        ),
        |&: (_, else_block)| else_block
      ))
    ),
    |&: (_, (lhs, (comp, (rhs, (then_block, else_block_opt)))))| If(lhs, comp, rhs, then_block, else_block_opt)
  )
}

fn while_stmt<'a>() -> LParser<'a, Statement> {
  map!(
    seq!(literal(WhileKeyword), expr(), comparator(), expr(), braced_block()),
    |&: (_, (lhs, (comp, (rhs, block))))| While(lhs, comp, rhs, block)
  )
}

pub fn block<'a>() -> LParser<'a, Block> {
  map!(
    rep!(
      map!(
        seq!(
          or!(output(), if_stmt(), while_stmt(), assign()),
          literal(NewLine)
        ),
        |&: (stmt, _)| stmt
      )
    ),
    |&: stmts| Block(stmts)
  )

}

pub fn braced_block<'a>() -> LParser<'a, Block> {
  map!(
    seq!(literal(OpenBrace), block(), literal(CloseBrace)),
    |&: (_, (block, _))| block
  )
}

pub fn comparator<'a>() -> LParser<'a, Comparator> {
  box MatchParser{
    matcher: box |&: input: &Token| match *input {
      Cmp(c) => Ok(c),
      ref other => Err(format!("Expected comparator, got {}", other))
    }
  }
}

fn variable<'a>() -> LParser<'a, Expr> {
  box MatchParser{
    matcher: box |&: input: &Token| match input {
      &Ident(ref str) => Ok(Variable(str.clone())),
      other => Err(format!("Expected variable, got {}", other))
    }
  }
}

type EParser<'a> = LParser<'a, Expr>;

fn expr<'a>() -> EParser<'a> {

  fn number<'a>() -> EParser<'a> {
    box MatchParser{matcher: box |&: input: &Token| -> Result<Expr, String> match *input {
      Number(num) => Ok(Num(num)),
      ref other => Err(format!("wrong type, expected number, got {}", other))
    }}
  }

  fn term<'a>() -> EParser<'a> {
    or!(paren_expr(), variable(), number())
  }

  fn mult<'a>() -> EParser<'a> {
    map!(
      seq!(
        term(),
        rep!(seq!(or!(literal(MultSign), literal(DivideSign)), term()))
      ),
      |&: (first, rest): (Expr, Vec<(Token, Expr)>)| {
        if rest.len() == 0 {
          first
        } else {
          let mut f = Vec::new();
          f.push(MultTerm(Multiply, first));
          for &(ref sign, ref value) in rest.iter() {
            let s = match *sign {
              MultSign => Multiply,
              DivideSign => Divide,
              _ => panic!("not allowed")
            };
            f.push(MultTerm(s, value.clone())); //maybe box the value instead
          }
          MultDiv(f)
        }
      }
    )
  }

  fn simple_expr<'a>() -> EParser<'a> {
    or!(mult(), term())
  }

  fn plus<'a>() -> EParser<'a> {
    map!(
      seq!(
        simple_expr(),
        rep!(seq!(or!(literal(PlusSign), literal(MinusSign)), simple_expr()))
      ),
      |&: (first, rest): (Expr, Vec<(Token, Expr)>)| {
        if rest.len() == 0 {
          first
        } else {
          let mut f = Vec::new();
          f.push(AddTerm(Add, first));
          for &(ref sign, ref value) in rest.iter() {
            let s = match *sign {
              PlusSign => Add,
              MinusSign => Subtract,
              _ => panic!("not allowed")
            };
            f.push(AddTerm(s, value.clone()));
          }
          AddSub(f)
        }
      }
    )
  }

  fn paren_expr<'a>() -> EParser<'a> {
    map!(
      seq!(literal(OpenParen), expr(), literal(CloseParen)),
      |&: (_, (expr, _))| expr
    )
  }

  plus()
}

fn test_parser<'a, I, O: PartialEq + Show>(input: I, parser: &Parser<'a, I, O>, expected: O) {
  match parser.parse(input) {
    Ok((output, rest)) => {
      assert_eq!(output, expected);
    },
    Err(err) => panic!(err)
  }
}

#[test]
fn test_term() {
  let parser = expr();
  let input = [Number(5)];
  let expected = Num(5);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_plus_sequence() {
  let parser = expr();
  let input = [Number(5), PlusSign, Number(3)];
  let expected = AddSub(vec![AddTerm(Add, Num(5)), AddTerm(Add, Num(3))]);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_plus_mult_sequence() {
  let parser = expr();
  let input = [Number(3), PlusSign, Number(4), MultSign, Number(5)];
  let expected = AddSub(vec![AddTerm(Add, Num(3)), AddTerm(Add, MultDiv(vec![MultTerm(Multiply, Num(4)), MultTerm(Multiply, Num(5))]))]);
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_simple_assign() {
  let parser = assign();
  let input = [Ident(from_str("x").unwrap()), Equals, Number(7)];
  let expected = Assign(from_str("x").unwrap(), Num(7));
  test_parser(input.as_slice(), &*parser, expected);
}

#[test]
fn test_simple_output() {
  let parser = output();
  let input = [OutputCmd, Number(4)];
  let expected = Output(Num(4));
  test_parser(input.as_slice(), &*parser, expected);
}
<|endoftext|>


<commit_before>use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::{DelimToken, IdentStyle};

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr_panic();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id, IdentStyle::Plain) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}
<commit_msg>Update parse_expr() calls as per changes in libsyntax.<commit_after>use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::{DelimToken, IdentStyle};

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id, IdentStyle::Plain) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}
<|endoftext|>


<commit_before>extern crate timely_communication;

fn main() {

    // extract the configuration from user-supplied arguments, initialize the computation.
    let config = timely_communication::Configuration::from_args(std::env::args()).unwrap();
    let guards = timely_communication::initialize(config, |mut allocator| {

        println!("worker {} of {} started", allocator.index(), allocator.peers());

        // allocates pair of senders list and one receiver.
        let (mut senders, mut receiver) = allocator.allocate();

        // send typed data along each channel
        for i in 0 .. allocator.peers() {
            senders[i].send(format!("hello, {}", i));
        }

        // no support for termination notification,
        // we have to count down ourselves.
        let mut received = 0;
        while received < allocator.peers() {
            if let Some(message) = receiver.recv() {
                println!("worker {}: received: <{}>", allocator.index(), message);
                received += 1;
            }
        }

        allocator.index()
    });

    // computation runs until guards are joined or dropped.
    if let Ok(guards) = guards {
        for guard in guards.join() {
            println!("result: {:?}", guard);
        }
    }
    else { println!("error in computation"); }
}
<commit_msg>flush channel, even if not needed<commit_after>extern crate timely_communication;

fn main() {

    // extract the configuration from user-supplied arguments, initialize the computation.
    let config = timely_communication::Configuration::from_args(std::env::args()).unwrap();
    let guards = timely_communication::initialize(config, |mut allocator| {

        println!("worker {} of {} started", allocator.index(), allocator.peers());

        // allocates pair of senders list and one receiver.
        let (mut senders, mut receiver) = allocator.allocate();

        // send typed data along each channel
        for i in 0 .. allocator.peers() {
            senders[i].send(format!("hello, {}", i));
            senders[i].done();
        }

        // no support for termination notification,
        // we have to count down ourselves.
        let mut received = 0;
        while received < allocator.peers() {
            if let Some(message) = receiver.recv() {
                println!("worker {}: received: <{}>", allocator.index(), message);
                received += 1;
            }
        }

        allocator.index()
    });

    // computation runs until guards are joined or dropped.
    if let Ok(guards) = guards {
        for guard in guards.join() {
            println!("result: {:?}", guard);
        }
    }
    else { println!("error in computation"); }
}
<|endoftext|>


<commit_before>//! Structs for reading a ZIP archive

use crc32::Crc32Reader;
use types::ZipFile;
use compression::CompressionMethod;
use spec;
use reader_spec;
use result::{ZipResult, ZipError};
use std::io;
use std::io::prelude::*;
use std::cell::{RefCell, BorrowState};
use std::collections::HashMap;
use flate2::FlateReadExt;
use bzip2::reader::BzDecompressor;

/// Wrapper for reading the contents of a ZIP file.
///
/// ```
/// fn doit() -> zip::result::ZipResult<()>
/// {
///     use std::io::prelude::*;
///
///     // For demonstration purposes we read from an empty buffer.
///     // Normally a File object would be used.
///     let buf: &[u8] = &[0u8; 128];
///     let mut reader = std::io::Cursor::new(buf);
///
///     let zip = try!(zip::ZipReader::new(reader));
///
///     for file in zip.files()
///     {
///         println!("Filename: {}", file.file_name);
///         let mut file_reader = try!(zip.read_file(file));
///         let first_byte = try!(file_reader.bytes().next().unwrap());
///         println!("{}", first_byte);
///     }
///     Ok(())
/// }
///
/// println!("Result: {:?}", doit());
/// ```
pub struct ZipReader<T>
{
    inner: RefCell<T>,
    files: Vec<ZipFile>,
    names_map: HashMap<String, usize>,
}

/// Iterator over the files contained in a zip archive
pub struct ZipFileIterator<'a>
{
    inner: ::std::slice::Iter<'a, ZipFile>,
}

impl<'a> Iterator for ZipFileIterator<'a> {
    type Item = &'a ZipFile;
    fn next(&mut self) -> Option<&'a ZipFile> {
        self.inner.next()
    }
}

fn unsupported_zip_error<T>(detail: &'static str) -> ZipResult<T>
{
    Err(ZipError::UnsupportedZipFile(detail))
}

impl<T: Read+io::Seek> ZipReader<T>
{
    /// Opens a ZIP file and parses the content headers.
    pub fn new(mut reader: T) -> ZipResult<ZipReader<T>>
    {
        let footer = try!(spec::CentralDirectoryEnd::find_and_parse(&mut reader));

        if footer.disk_number != footer.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented") }

        let directory_start = footer.central_directory_offset as u64;
        let number_of_files = footer.number_of_files_on_this_disk as usize;

        let mut files = Vec::with_capacity(number_of_files);
        let mut names_map = HashMap::new();

        try!(reader.seek(io::SeekFrom::Start(directory_start)));
        for _ in (0 .. number_of_files)
        {
            let file = try!(reader_spec::central_header_to_zip_file(&mut reader));
            names_map.insert(file.file_name.clone(), files.len());
            files.push(file);
        }

        Ok(ZipReader { inner: RefCell::new(reader), files: files, names_map: names_map })
    }

    /// An iterator over the information of all contained files.
    pub fn files(&self) -> ZipFileIterator
    {
        ZipFileIterator { inner: (&*self.files).iter(), }
    }

    /// Search for a file entry by name
    pub fn get(&self, name: &str) -> Option<&ZipFile>
    {
        self.names_map.get(name).map(|index| &self.files[*index])
    }

    /// Gets a reader for a contained zipfile.
    ///
    /// May return `ReaderUnavailable` if there is another reader borrowed.
    pub fn read_file<'a>(&'a self, file: &ZipFile) -> ZipResult<Box<Read+'a>>
    {
        let mut inner_reader = match self.inner.borrow_state()
        {
            BorrowState::Unused => self.inner.borrow_mut(),
            _ => return Err(ZipError::ReaderUnavailable),
        };
        let pos = file.data_start as u64;

        if file.encrypted
        {
            return unsupported_zip_error("Encrypted files are not supported")
        }

        try!(inner_reader.seek(io::SeekFrom::Start(pos)));
        let refmut_reader = ::util::RefMutReader::new(inner_reader);
        let limit_reader = refmut_reader.take(file.compressed_size as u64);

        let reader = match file.compression_method
        {
            CompressionMethod::Stored =>
            {
                Box::new(
                    Crc32Reader::new(
                        limit_reader,
                        file.crc32))
                    as Box<Read>
            },
            CompressionMethod::Deflated =>
            {
                let deflate_reader = limit_reader.deflate_decode();
                Box::new(
                    Crc32Reader::new(
                        deflate_reader,
                        file.crc32))
                    as Box<Read>
            },
            CompressionMethod::Bzip2 =>
            {
                let bzip2_reader = BzDecompressor::new(limit_reader);
                Box::new(
                    Crc32Reader::new(
                        bzip2_reader,
                        file.crc32))
                    as Box<Read>
            },
            _ => return unsupported_zip_error("Compression method not supported"),
        };
        Ok(reader)
    }

    /// Unwrap and return the inner reader object
    ///
    /// The position of the reader is undefined.
    pub fn into_inner(self) -> T
    {
        self.inner.into_inner()
    }
}
<commit_msg>Implement size_hint on our iterator wrapper<commit_after>//! Structs for reading a ZIP archive

use crc32::Crc32Reader;
use types::ZipFile;
use compression::CompressionMethod;
use spec;
use reader_spec;
use result::{ZipResult, ZipError};
use std::io;
use std::io::prelude::*;
use std::cell::{RefCell, BorrowState};
use std::collections::HashMap;
use flate2::FlateReadExt;
use bzip2::reader::BzDecompressor;

/// Wrapper for reading the contents of a ZIP file.
///
/// ```
/// fn doit() -> zip::result::ZipResult<()>
/// {
///     use std::io::prelude::*;
///
///     // For demonstration purposes we read from an empty buffer.
///     // Normally a File object would be used.
///     let buf: &[u8] = &[0u8; 128];
///     let mut reader = std::io::Cursor::new(buf);
///
///     let zip = try!(zip::ZipReader::new(reader));
///
///     for file in zip.files()
///     {
///         println!("Filename: {}", file.file_name);
///         let mut file_reader = try!(zip.read_file(file));
///         let first_byte = try!(file_reader.bytes().next().unwrap());
///         println!("{}", first_byte);
///     }
///     Ok(())
/// }
///
/// println!("Result: {:?}", doit());
/// ```
pub struct ZipReader<T>
{
    inner: RefCell<T>,
    files: Vec<ZipFile>,
    names_map: HashMap<String, usize>,
}

/// Iterator over the files contained in a zip archive
pub struct ZipFileIterator<'a>
{
    inner: ::std::slice::Iter<'a, ZipFile>,
}

impl<'a> Iterator for ZipFileIterator<'a> {
    type Item = &'a ZipFile;
    fn next(&mut self) -> Option<&'a ZipFile> {
        self.inner.next()
    }
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.inner.size_hint()
    }
}

fn unsupported_zip_error<T>(detail: &'static str) -> ZipResult<T>
{
    Err(ZipError::UnsupportedZipFile(detail))
}

impl<T: Read+io::Seek> ZipReader<T>
{
    /// Opens a ZIP file and parses the content headers.
    pub fn new(mut reader: T) -> ZipResult<ZipReader<T>>
    {
        let footer = try!(spec::CentralDirectoryEnd::find_and_parse(&mut reader));

        if footer.disk_number != footer.disk_with_central_directory { return unsupported_zip_error("Support for multi-disk files is not implemented") }

        let directory_start = footer.central_directory_offset as u64;
        let number_of_files = footer.number_of_files_on_this_disk as usize;

        let mut files = Vec::with_capacity(number_of_files);
        let mut names_map = HashMap::new();

        try!(reader.seek(io::SeekFrom::Start(directory_start)));
        for _ in (0 .. number_of_files)
        {
            let file = try!(reader_spec::central_header_to_zip_file(&mut reader));
            names_map.insert(file.file_name.clone(), files.len());
            files.push(file);
        }

        Ok(ZipReader { inner: RefCell::new(reader), files: files, names_map: names_map })
    }

    /// An iterator over the information of all contained files.
    pub fn files(&self) -> ZipFileIterator
    {
        ZipFileIterator { inner: (&*self.files).iter(), }
    }

    /// Search for a file entry by name
    pub fn get(&self, name: &str) -> Option<&ZipFile>
    {
        self.names_map.get(name).map(|index| &self.files[*index])
    }

    /// Gets a reader for a contained zipfile.
    ///
    /// May return `ReaderUnavailable` if there is another reader borrowed.
    pub fn read_file<'a>(&'a self, file: &ZipFile) -> ZipResult<Box<Read+'a>>
    {
        let mut inner_reader = match self.inner.borrow_state()
        {
            BorrowState::Unused => self.inner.borrow_mut(),
            _ => return Err(ZipError::ReaderUnavailable),
        };
        let pos = file.data_start as u64;

        if file.encrypted
        {
            return unsupported_zip_error("Encrypted files are not supported")
        }

        try!(inner_reader.seek(io::SeekFrom::Start(pos)));
        let refmut_reader = ::util::RefMutReader::new(inner_reader);
        let limit_reader = refmut_reader.take(file.compressed_size as u64);

        let reader = match file.compression_method
        {
            CompressionMethod::Stored =>
            {
                Box::new(
                    Crc32Reader::new(
                        limit_reader,
                        file.crc32))
                    as Box<Read>
            },
            CompressionMethod::Deflated =>
            {
                let deflate_reader = limit_reader.deflate_decode();
                Box::new(
                    Crc32Reader::new(
                        deflate_reader,
                        file.crc32))
                    as Box<Read>
            },
            CompressionMethod::Bzip2 =>
            {
                let bzip2_reader = BzDecompressor::new(limit_reader);
                Box::new(
                    Crc32Reader::new(
                        bzip2_reader,
                        file.crc32))
                    as Box<Read>
            },
            _ => return unsupported_zip_error("Compression method not supported"),
        };
        Ok(reader)
    }

    /// Unwrap and return the inner reader object
    ///
    /// The position of the reader is undefined.
    pub fn into_inner(self) -> T
    {
        self.inner.into_inner()
    }
}
<|endoftext|>