<commit_before>use nom::{alpha,alphanumeric,space};
use std;

pub enum Command {
    Exit,
    Help,
    Load( String ),
    Query( Vec<String>, Vec<Filter> ),
}

pub struct Filter {
    pub field_name: String,
    pub filter_type: String,
    pub value: String,
}

named!(
    pub cmd<Command>,
    alt!(
        exit
      | help
      | load
      | query
    )
);

named!(
    pub exit<Command>,
    chain!(
        tag!("EXIT"),
        || { Command::Exit }
    )
);

named!(
    pub help<Command>,
    chain!(
        tag!("HELP"),
        || { Command::Help }
    )
);

named!(
    pub field_names<Vec<String> >,
    alt!(
        tag!("*") => { |_| Vec::new() }
        | chain! (
            field_name: alpha ~
            field_names: many0!(
                chain!(
                    opt!(space) ~
                    tag!(",") ~
                    opt!(space) ~
                    field_name: alpha,
                    || field_name
                )
            ),
            || {
                let mut rtn_field_names = vec!(std::str::from_utf8(field_name).unwrap().to_string());
                for field_name in field_names {
                    rtn_field_names.push(std::str::from_utf8(field_name).unwrap().to_string());
                }

                rtn_field_names
            }
        )
    )
);

named!(
    pub filename<String>,
    chain!(
        chars: many1!(
            map_res!(
                alt!(
                    tag!("-") | tag!("_") | tag!("/") | tag!(".") | alphanumeric
                ),
                std::str::from_utf8
            )
        ),
        || {
            chars.into_iter().fold(
                "".to_string(),
                |mut f, c| {
                   f.push_str(c);
                   f
                }
            )
        }
    )
);

named!(
    pub filter<Filter>,
    chain!(
        field_name: id ~
        space ~
        tag!("~") ~
        filter_type: id ~
        space ~
        value: id,
        || Filter { field_name: field_name, filter_type: filter_type, value: value }
    )
);

named!(
    pub id<String>,
    chain!(
        chars: many1!(
            map_res!(
                alt!(
                    tag!("-") | tag!("_") | tag!(".") | alphanumeric
                ),
                std::str::from_utf8
            )
        ),
        || {
            chars.into_iter().fold(
                "".to_string(),
                |mut f, c| {
                   f.push_str(c);
                   f
                }
            )
        }
    )
);

named!(
    pub load<Command>,
    chain!(
        tag!("LOAD") ~
        space ~
        f: filename,
        || { Command::Load(f) }
    )
);

named!(
    pub query<Command>,
    chain!(
        tag!("SELECT") ~
        space ~
        field_names: field_names ~
        space ~
        tag!("WHERE") ~
        space ~
        f: filter ~
        filters: many0!(
            chain!(
                space ~
                tag!("AND") ~
                space ~
                f: filter,
                || { f }
            )
        ),
        || {
            let mut rtn_filters = vec!(f);
            for filter in filters {
                rtn_filters.push(filter);
            }

            Command::Query(
                field_names,
                rtn_filters,
            )
        }
    )
);
<commit_msg>Can return fields with non-alpha characters from SELECT statement - bug fix.<commit_after>use nom::{alphanumeric,space};
use std;

pub enum Command {
    Exit,
    Help,
    Load( String ),
    Query( Vec<String>, Vec<Filter> ),
}

pub struct Filter {
    pub field_name: String,
    pub filter_type: String,
    pub value: String,
}

named!(
    pub cmd<Command>,
    alt!(
        exit
      | help
      | load
      | query
    )
);

named!(
    pub exit<Command>,
    chain!(
        tag!("EXIT"),
        || { Command::Exit }
    )
);

named!(
    pub help<Command>,
    chain!(
        tag!("HELP"),
        || { Command::Help }
    )
);

named!(
    pub field_names<Vec<String> >,
    alt!(
        tag!("*") => { |_| Vec::new() }
        | chain! (
            field_name: id ~
            field_names: many0!(
                chain!(
                    opt!(space) ~
                    tag!(",") ~
                    opt!(space) ~
                    field_name: id,
                    || field_name
                )
            ),
            || {
                let mut rtn_field_names = vec!(field_name);
                for field_name in field_names {
                    rtn_field_names.push(field_name);
                }

                rtn_field_names
            }
        )
    )
);

named!(
    pub filename<String>,
    chain!(
        chars: many1!(
            map_res!(
                alt!(
                    tag!("-") | tag!("_") | tag!("/") | tag!(".") | alphanumeric
                ),
                std::str::from_utf8
            )
        ),
        || {
            chars.into_iter().fold(
                "".to_string(),
                |mut f, c| {
                   f.push_str(c);
                   f
                }
            )
        }
    )
);

named!(
    pub filter<Filter>,
    chain!(
        field_name: id ~
        space ~
        tag!("~") ~
        filter_type: id ~
        space ~
        value: id,
        || Filter { field_name: field_name, filter_type: filter_type, value: value }
    )
);

named!(
    pub id<String>,
    chain!(
        chars: many1!(
            map_res!(
                alt!(
                    tag!("-") | tag!("_") | tag!(".") | alphanumeric
                ),
                std::str::from_utf8
            )
        ),
        || {
            chars.into_iter().fold(
                "".to_string(),
                |mut f, c| {
                   f.push_str(c);
                   f
                }
            )
        }
    )
);

named!(
    pub load<Command>,
    chain!(
        tag!("LOAD") ~
        space ~
        f: filename,
        || { Command::Load(f) }
    )
);

named!(
    pub query<Command>,
    chain!(
        tag!("SELECT") ~
        space ~
        field_names: field_names ~
        space ~
        tag!("WHERE") ~
        space ~
        f: filter ~
        filters: many0!(
            chain!(
                space ~
                tag!("AND") ~
                space ~
                f: filter,
                || { f }
            )
        ),
        || {
            let mut rtn_filters = vec!(f);
            for filter in filters {
                rtn_filters.push(filter);
            }

            Command::Query(
                field_names,
                rtn_filters,
            )
        }
    )
);
<|endoftext|>


<commit_before>use ast::{Token, Expr};
use ast::{BlockExpr, ThrowExpr, ReturnExpr, CallExpr, ConstructExpr, IfExpr, WhileLoopExpr, SwitchExpr, FunctionDeclExpr, LocalExpr, ArrayDeclExpr, ObjectDeclExpr, GetConstFieldExpr, GetFieldExpr, NumOpExpr, BitOpExpr, ConstExpr};
use ast::{CBool, CNull, CUndefined, CString, CNum};
use ast::{TIdent, TNumber, TString, TSemicolon, TColon, TDot, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TNumOp, TBitOp};
use collections::treemap::TreeMap;
use std::fmt;
use std::vec::Vec;
#[deriving(Clone)]
#[deriving(Eq)]
/// An error encountered during parsing an expression
pub enum ParseError {
	/// When it expected a certain kind of token, but got another
	Expected(Vec<Token>, Token),
	/// When it expected a certain expression, but got another
	ExpectedExpr(~str, Expr)
}
impl fmt::Show for ParseError {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		return match *self {
			Expected(ref wanted, ref got) if wanted.len() == 0 => write!(f.buf, "Expected expression, got {}", got),
			Expected(ref wanted, ref got) => {
				try!(write!(f.buf, "Expected "));
				let last = wanted.last().unwrap();
				for wanted_token in wanted.iter() {
					match write!(f.buf, "'{}'{}", wanted_token, if wanted_token == last {""} else {", "}) {
						Err(err) => return Err(err),
						Ok(_) => ()
					}
				}
				write!(f.buf, " but got {}", got)
			},
			ExpectedExpr(ref wanted, ref got) => {
				write!(f.buf, "Expected {}, but got {}", wanted, got)
			}
		}
	}
}
/// A result which gives a pretty-printed error if it fails
pub trait VerboseResult<T> {
	/// Unwrap the value if it's Ok and if not pretty-print the error
	fn v_unwrap(&self) -> T;
}
impl<T : Clone, E : fmt::Show> VerboseResult<T> for Result<T, E> {
	fn v_unwrap(&self) -> T {
		match *self {
			Ok(ref val) => val.clone(),
			Err(ref err) => fail!("{}", err)
		}
	}
}
pub type ParseResult = Result<~Expr, ParseError>;
pub type ParseStructResult = Result<Option<~Expr>, ParseError>;
/// A Javascript parser
pub struct Parser {
	/// The tokens being input
	tokens: Vec<Token>,
	/// The current position within the tokens
	pos: uint
}
impl Parser {
	/// Creates a new parser, using [tokens] as input
	pub fn new(tokens: Vec<Token>) -> ~Parser {
		return ~Parser {tokens: tokens, pos: 0};
	}
	/// Parse all expressions in the token array
	pub fn parse_all(&mut self) -> ParseResult {
		let mut exprs = Vec::new();
		while self.pos < self.tokens.len() {
			let result = try!(self.parse());
			exprs.push(result);
		}
		return Ok(~BlockExpr(exprs));
	}
	fn parse_struct(&mut self, name:&str) -> ParseStructResult {
		match name {
			"true" => Ok(Some(~ConstExpr(CBool(true)))),
			"false" => Ok(Some(~ConstExpr(CBool(false)))),
			"null" => Ok(Some(~ConstExpr(CNull))),
			"undefined" => Ok(Some(~ConstExpr(CUndefined))),
			"throw" => {
				let thrown = try!(self.parse());
				Ok(Some(~ThrowExpr(thrown)))
			},
			"return" => Ok(Some(~ReturnExpr(Some(try!(self.parse()))))),
			"new" => {
				let call = try!(self.parse());
				print!("new {}", call);
				match *call {
					CallExpr(ref func, ref args) => Ok(Some(~ConstructExpr(func.clone(), args.clone()))),
					_ => Err(ExpectedExpr(~"constructor", *call))
				}
			}
			"if" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				let next = self.tokens.get(self.pos + 1).clone();
				Ok(Some(~IfExpr(cond, expr, if next == TIdent(~"else") {
					self.pos += 2;
					Some(try!(self.parse()))
				} else {
					None
				})))
			},
			"while" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				Ok(Some(~WhileLoopExpr(cond, expr)))
			},
			"switch" => {
				try!(self.expect(TOpenParen));
				let value = self.parse();
				try!(self.expect(TCloseParen));
				try!(self.expect(TOpenBlock));
				let mut cases = Vec::new();
				let mut default = None;
				while self.pos + 1 < self.tokens.len() {
					let tok = self.tokens.get(self.pos).clone();
					self.pos += 1;
					match tok {
						TIdent(ref id) if *id == ~"case" => {
							let cond = self.parse();
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							cases.push((cond.unwrap(), block));
						},
						TIdent(ref id) if *id == ~"default" => {
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							default = Some(~BlockExpr(block));
						},
						TCloseBlock => break,
						_ => fail!("Expected 'case', 'default' or '{}', not '{}'", '}', tok)
					}
				}
				try!(self.expect(TCloseBlock));
				Ok(Some(~SwitchExpr(value.unwrap(), cases, default)))
			},
			"function" => {
				let tk = self.tokens.get(self.pos).clone();
				let name : Option<~str> = match tk {
					TIdent(ref name) => {
						self.pos += 1;
						Some(name.clone())
					},
					TOpenParen => None,
					_ => return Err(Expected(vec!(TIdent(~"identifier")), tk.clone()))
				};
				try!(self.expect(TOpenParen));
				let mut args:Vec<~str> = Vec::new();
				let mut tk = self.tokens.get(self.pos).clone();
				while tk != TCloseParen {
					match tk {
						TIdent(ref id) => args.push(id.to_owned()),
						_ => return Err(Expected(vec!(TIdent(~"identifier")), tk))
					}
					self.pos += 1;
					if *self.tokens.get(self.pos) == TComma {
						self.pos += 1;
					}
					tk = self.tokens.get(self.pos).clone();
				}
				self.pos += 1;
				let block = try!(self.parse());
				Ok(Some(~FunctionDeclExpr(name, args, block)))
			},
			_ => Ok(None)
		}
	}
	/// Parse a single expression
	pub fn parse(&mut self) -> ParseResult {
		let token = self.tokens.get(self.pos).clone();
		self.pos += 1;
		let expr : ~Expr = match token {
			TSemicolon => try!(self.parse()),
			TIdent(ref s) => {
				let structure = try!(self.parse_struct(s.clone()));
				match structure {
					Some(v) => v,
					None => ~LocalExpr(s.to_owned())
				}
			},
			TString(ref s) => ~ConstExpr(CString(s.to_owned())),
			TOpenParen => {
				let nexte = try!(self.parse());
				try!(self.expect(TCloseParen));
				nexte
			},
			TOpenArray => {
				let mut array = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos) == TCloseArray;
				loop {
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseArray && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if token == TComma && !expect_comma_or_end {
						array.push(~ConstExpr(CNull));
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseArray), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						array.push(parsed);
						expect_comma_or_end = true;
					}
					self.pos += 1;
				}
				~ArrayDeclExpr(array)
			},
			TOpenBlock if self.tokens.get(self.pos) == &TCloseBlock => {
				self.pos += 1;
				~ObjectDeclExpr(~TreeMap::new())
			},
			TOpenBlock if self.tokens.get(self.pos + 1) == &TColon => {
				let mut map = ~TreeMap::new();
				while self.tokens.get(self.pos - 1) == &TComma || map.len() == 0 {
					let tk = self.tokens.get(self.pos).clone();
					let name = match tk {
						TIdent(ref id) => id,
						TString(ref str) => str,
						_ => return Err(Expected(vec!(TIdent(~"identifier"), TString(~"string")), tk))
					};
					self.pos += 1;
					try!(self.expect(TColon));
					let value = try!(self.parse());
					map.insert(name.to_owned(), value);
					self.pos += 1;
				}
				~ObjectDeclExpr(map)
			},
			TOpenBlock => {
				let mut exprs = Vec::new();
				loop {
					if *self.tokens.get(self.pos) == TCloseBlock {
						break;
					} else {
						exprs.push(try!(self.parse()));
					}
				}
				self.pos += 1;
				~BlockExpr(exprs)
			},
			TNumber(num) => ~ConstExpr(CNum(num)),
			_ => return Err(Expected(Vec::new(), token.clone()))
		};
		return if self.pos >= self.tokens.len() { Ok(expr) } else {self.parse_next(expr)};
	}
	fn parse_next(&mut self, expr:~Expr) -> ParseResult {
		let next = self.tokens.get(self.pos).clone();
		let mut carry_on = true;
		let mut result = expr.clone();
		match next {
			TDot => {
				self.pos += 1;
				let tk = self.tokens.get(self.pos).clone();
				match tk {
					TIdent(ref s) => result = ~GetConstFieldExpr(expr, s.to_owned()),
					_ => return Err(Expected(vec!(TIdent(~"identifier")), next))
				}
				self.pos += 1;
			},
			TOpenParen => {
				let mut args = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos + 1) == TCloseParen;
				loop {
					self.pos += 1;
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseParen && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseParen), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						args.push(parsed);
						expect_comma_or_end = true;
					}
				}
				result = ~CallExpr(expr, args);
			},
			TQuestion => {
				self.pos += 1;
				let if_e = try!(self.parse());
				try!(self.expect(TColon));
				let else_e = try!(self.parse());
				result = ~IfExpr(expr, if_e, Some(else_e));
			},
			TOpenArray => {
				self.pos += 1;
				let index = try!(self.parse());
				try!(self.expect(TCloseArray));
				result = ~GetFieldExpr(expr, index);
			},
			TSemicolon => {
				self.pos += 1;
			},
			TNumOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~NumOpExpr(op, expr, next);
			},
			TBitOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~BitOpExpr(op, expr, next);
			},
			_ => carry_on = false
		};
		return if carry_on && self.pos < self.tokens.len() { self.parse_next(result) } else {Ok(result)};
	}
	/// Returns an error if the next symbol is not [tk]
	fn expect(&mut self, tk:Token) -> Result<(), ParseError> {
		self.pos += 1;
		let curr_tk = self.tokens.get(self.pos - 1).clone();
		return if curr_tk != tk {
			Err(Expected(vec!(tk), curr_tk))
		} else {
			Ok(())
		};
	}
}<commit_msg>Parse = as assign expr<commit_after>use ast::{Token, Expr};
use ast::{BlockExpr, ThrowExpr, ReturnExpr, CallExpr, ConstructExpr, IfExpr, WhileLoopExpr, SwitchExpr, FunctionDeclExpr, LocalExpr, ArrayDeclExpr, ObjectDeclExpr, GetConstFieldExpr, GetFieldExpr, NumOpExpr, BitOpExpr, ConstExpr, AssignExpr};
use ast::{CBool, CNull, CUndefined, CString, CNum};
use ast::{TIdent, TNumber, TString, TSemicolon, TColon, TDot, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TNumOp, TBitOp, TEqual};
use collections::treemap::TreeMap;
use std::fmt;
use std::vec::Vec;
#[deriving(Clone)]
#[deriving(Eq)]
/// An error encountered during parsing an expression
pub enum ParseError {
	/// When it expected a certain kind of token, but got another
	Expected(Vec<Token>, Token),
	/// When it expected a certain expression, but got another
	ExpectedExpr(~str, Expr)
}
impl fmt::Show for ParseError {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		return match *self {
			Expected(ref wanted, ref got) if wanted.len() == 0 => write!(f.buf, "Expected expression, got {}", got),
			Expected(ref wanted, ref got) => {
				try!(write!(f.buf, "Expected "));
				let last = wanted.last().unwrap();
				for wanted_token in wanted.iter() {
					match write!(f.buf, "'{}'{}", wanted_token, if wanted_token == last {""} else {", "}) {
						Err(err) => return Err(err),
						Ok(_) => ()
					}
				}
				write!(f.buf, " but got {}", got)
			},
			ExpectedExpr(ref wanted, ref got) => {
				write!(f.buf, "Expected {}, but got {}", wanted, got)
			}
		}
	}
}
/// A result which gives a pretty-printed error if it fails
pub trait VerboseResult<T> {
	/// Unwrap the value if it's Ok and if not pretty-print the error
	fn v_unwrap(&self) -> T;
}
impl<T : Clone, E : fmt::Show> VerboseResult<T> for Result<T, E> {
	fn v_unwrap(&self) -> T {
		match *self {
			Ok(ref val) => val.clone(),
			Err(ref err) => fail!("{}", err)
		}
	}
}
pub type ParseResult = Result<~Expr, ParseError>;
pub type ParseStructResult = Result<Option<~Expr>, ParseError>;
/// A Javascript parser
pub struct Parser {
	/// The tokens being input
	tokens: Vec<Token>,
	/// The current position within the tokens
	pos: uint
}
impl Parser {
	/// Creates a new parser, using [tokens] as input
	pub fn new(tokens: Vec<Token>) -> ~Parser {
		return ~Parser {tokens: tokens, pos: 0};
	}
	/// Parse all expressions in the token array
	pub fn parse_all(&mut self) -> ParseResult {
		let mut exprs = Vec::new();
		while self.pos < self.tokens.len() {
			let result = try!(self.parse());
			exprs.push(result);
		}
		return Ok(~BlockExpr(exprs));
	}
	fn parse_struct(&mut self, name:&str) -> ParseStructResult {
		match name {
			"true" => Ok(Some(~ConstExpr(CBool(true)))),
			"false" => Ok(Some(~ConstExpr(CBool(false)))),
			"null" => Ok(Some(~ConstExpr(CNull))),
			"undefined" => Ok(Some(~ConstExpr(CUndefined))),
			"throw" => {
				let thrown = try!(self.parse());
				Ok(Some(~ThrowExpr(thrown)))
			},
			"return" => Ok(Some(~ReturnExpr(Some(try!(self.parse()))))),
			"new" => {
				let call = try!(self.parse());
				print!("new {}", call);
				match *call {
					CallExpr(ref func, ref args) => Ok(Some(~ConstructExpr(func.clone(), args.clone()))),
					_ => Err(ExpectedExpr(~"constructor", *call))
				}
			}
			"if" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				let next = self.tokens.get(self.pos + 1).clone();
				Ok(Some(~IfExpr(cond, expr, if next == TIdent(~"else") {
					self.pos += 2;
					Some(try!(self.parse()))
				} else {
					None
				})))
			},
			"while" => {
				try!(self.expect(TOpenParen));
				let cond = try!(self.parse());
				try!(self.expect(TCloseParen));
				let expr = try!(self.parse());
				Ok(Some(~WhileLoopExpr(cond, expr)))
			},
			"switch" => {
				try!(self.expect(TOpenParen));
				let value = self.parse();
				try!(self.expect(TCloseParen));
				try!(self.expect(TOpenBlock));
				let mut cases = Vec::new();
				let mut default = None;
				while self.pos + 1 < self.tokens.len() {
					let tok = self.tokens.get(self.pos).clone();
					self.pos += 1;
					match tok {
						TIdent(ref id) if *id == ~"case" => {
							let cond = self.parse();
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							cases.push((cond.unwrap(), block));
						},
						TIdent(ref id) if *id == ~"default" => {
							let mut block = Vec::new();
							try!(self.expect(TColon));
							loop {
								match self.tokens.get(self.pos).clone() {
									TIdent(ref id) if *id == ~"case" || *id == ~"default" => break,
									TCloseBlock => break,
									_ => block.push(try!(self.parse()))
								}
							}
							default = Some(~BlockExpr(block));
						},
						TCloseBlock => break,
						_ => fail!("Expected 'case', 'default' or '{}', not '{}'", '}', tok)
					}
				}
				try!(self.expect(TCloseBlock));
				Ok(Some(~SwitchExpr(value.unwrap(), cases, default)))
			},
			"function" => {
				let tk = self.tokens.get(self.pos).clone();
				let name : Option<~str> = match tk {
					TIdent(ref name) => {
						self.pos += 1;
						Some(name.clone())
					},
					TOpenParen => None,
					_ => return Err(Expected(vec!(TIdent(~"identifier")), tk.clone()))
				};
				try!(self.expect(TOpenParen));
				let mut args:Vec<~str> = Vec::new();
				let mut tk = self.tokens.get(self.pos).clone();
				while tk != TCloseParen {
					match tk {
						TIdent(ref id) => args.push(id.to_owned()),
						_ => return Err(Expected(vec!(TIdent(~"identifier")), tk))
					}
					self.pos += 1;
					if *self.tokens.get(self.pos) == TComma {
						self.pos += 1;
					}
					tk = self.tokens.get(self.pos).clone();
				}
				self.pos += 1;
				let block = try!(self.parse());
				Ok(Some(~FunctionDeclExpr(name, args, block)))
			},
			_ => Ok(None)
		}
	}
	/// Parse a single expression
	pub fn parse(&mut self) -> ParseResult {
		let token = self.tokens.get(self.pos).clone();
		self.pos += 1;
		let expr : ~Expr = match token {
			TSemicolon => try!(self.parse()),
			TIdent(ref s) => {
				let structure = try!(self.parse_struct(s.clone()));
				match structure {
					Some(v) => v,
					None => ~LocalExpr(s.to_owned())
				}
			},
			TString(ref s) => ~ConstExpr(CString(s.to_owned())),
			TOpenParen => {
				let nexte = try!(self.parse());
				try!(self.expect(TCloseParen));
				nexte
			},
			TOpenArray => {
				let mut array = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos) == TCloseArray;
				loop {
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseArray && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if token == TComma && !expect_comma_or_end {
						array.push(~ConstExpr(CNull));
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseArray), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						array.push(parsed);
						expect_comma_or_end = true;
					}
					self.pos += 1;
				}
				~ArrayDeclExpr(array)
			},
			TOpenBlock if self.tokens.get(self.pos) == &TCloseBlock => {
				self.pos += 1;
				~ObjectDeclExpr(~TreeMap::new())
			},
			TOpenBlock if self.tokens.get(self.pos + 1) == &TColon => {
				let mut map = ~TreeMap::new();
				while self.tokens.get(self.pos - 1) == &TComma || map.len() == 0 {
					let tk = self.tokens.get(self.pos).clone();
					let name = match tk {
						TIdent(ref id) => id,
						TString(ref str) => str,
						_ => return Err(Expected(vec!(TIdent(~"identifier"), TString(~"string")), tk))
					};
					self.pos += 1;
					try!(self.expect(TColon));
					let value = try!(self.parse());
					map.insert(name.to_owned(), value);
					self.pos += 1;
				}
				~ObjectDeclExpr(map)
			},
			TOpenBlock => {
				let mut exprs = Vec::new();
				loop {
					if *self.tokens.get(self.pos) == TCloseBlock {
						break;
					} else {
						exprs.push(try!(self.parse()));
					}
				}
				self.pos += 1;
				~BlockExpr(exprs)
			},
			TNumber(num) => ~ConstExpr(CNum(num)),
			_ => return Err(Expected(Vec::new(), token.clone()))
		};
		return if self.pos >= self.tokens.len() { Ok(expr) } else {self.parse_next(expr)};
	}
	fn parse_next(&mut self, expr:~Expr) -> ParseResult {
		let next = self.tokens.get(self.pos).clone();
		let mut carry_on = true;
		let mut result = expr.clone();
		match next {
			TDot => {
				self.pos += 1;
				let tk = self.tokens.get(self.pos).clone();
				match tk {
					TIdent(ref s) => result = ~GetConstFieldExpr(expr, s.to_owned()),
					_ => return Err(Expected(vec!(TIdent(~"identifier")), next))
				}
				self.pos += 1;
			},
			TOpenParen => {
				let mut args = Vec::new();
				let mut expect_comma_or_end = *self.tokens.get(self.pos + 1) == TCloseParen;
				loop {
					self.pos += 1;
					let token = self.tokens.get(self.pos).clone();
					if token == TCloseParen && expect_comma_or_end {
						self.pos += 1;
						break;
					} else if token == TComma && expect_comma_or_end {
						expect_comma_or_end = false;
					} else if expect_comma_or_end {
						return Err(Expected(vec!(TComma, TCloseParen), token));
					} else {
						let parsed = try!(self.parse());
						self.pos -= 1;
						args.push(parsed);
						expect_comma_or_end = true;
					}
				}
				result = ~CallExpr(expr, args);
			},
			TQuestion => {
				self.pos += 1;
				let if_e = try!(self.parse());
				try!(self.expect(TColon));
				let else_e = try!(self.parse());
				result = ~IfExpr(expr, if_e, Some(else_e));
			},
			TOpenArray => {
				self.pos += 1;
				let index = try!(self.parse());
				try!(self.expect(TCloseArray));
				result = ~GetFieldExpr(expr, index);
			},
			TSemicolon => {
				self.pos += 1;
			},
			TNumOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~NumOpExpr(op, expr, next);
			},
			TBitOp(op) => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~BitOpExpr(op, expr, next);
			},
			TEqual => {
				self.pos += 1;
				let next = try!(self.parse());
				result = ~AssignExpr(expr, next);
			},
			_ => carry_on = false
		};
		return if carry_on && self.pos < self.tokens.len() { self.parse_next(result) } else {Ok(result)};
	}
	/// Returns an error if the next symbol is not [tk]
	fn expect(&mut self, tk:Token) -> Result<(), ParseError> {
		self.pos += 1;
		let curr_tk = self.tokens.get(self.pos - 1).clone();
		return if curr_tk != tk {
			Err(Expected(vec!(tk), curr_tk))
		} else {
			Ok(())
		};
	}
}<|endoftext|>


<commit_before>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a [u8]
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8)                  ~
            length: expr_opt!({($size as usize).checked_sub(s.len())}) ~
            take!(length),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit_consume(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      let mut append = true;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if append {
              if $stop(&o) {
                append = false;
              } else {
                res.push(o);
              }
            }
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorKind::TakeUntil,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit_consume!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> TypeFlag {
    char_to_type_flag(i[0] as char)
}

named!(parse_type_flag<&[u8], TypeFlag>, map!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

named!(parse_one_sparse<&[u8], Sparse>, map!(pair!(parse_octal12, parse_octal12), |(offset, numbytes)| Sparse { offset: offset, numbytes: numbytes }));

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit_consume!(i, parse_one_sparse, |s: &Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec<'a, 'b>(sparses: &'a mut Vec<Sparse>, extra: &'b mut Vec<Sparse>) -> &'a mut Vec<Sparse> {
    while sparses.len() != 0 {
      extra.pop().map(|s| sparses.push(s));
    }
    sparses
}

fn parse_extra_sparses<'a, 'b>(i: &'a [u8], isextended: bool, sparses: &'b mut Vec<Sparse>) -> IResult<&'a [u8], &'b mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            mut sps: apply!(parse_sparses_with_limit, 21) ~
            extended:      parse_bool                     ~
            take!(7) /* padding to 512 */                 ~
            extra_sparses: apply!(parse_extra_sparses, extended, add_to_vec(sparses, &mut sps)),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

fn parse_pax_extra_sparses<'a, 'b>(i: &'a [u8], h: &'b mut PaxHeader) -> IResult<&'a [u8], &'b mut Vec<Sparse>> {
    parse_extra_sparses(i, h.isextended, &mut h.sparses)
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> bool {
    i[0] != 0
}

named!(parse_bool<&[u8], bool>, map!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

named!(parse_ustar00_extra_posix<&[u8], UStarExtraHeader>, map!(pair!(parse_str155, take!(12)), |(prefix, _)| UStarExtraHeader::PosixUStar(PosixUStarHeader { prefix: prefix })));

fn parse_ustar00_extra<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], UStarExtraHeader<'a>> {
    match *flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                mut header: parse_ustar00_extra_pax ~
                apply!(parse_pax_extra_sparses, &mut header),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], ExtraHeader<'a>> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], ExtraHeader<'a>> {
    map!(i, pair!(tag!("ustar\0"), apply!(parse_ustar00, flag)), |(_, ustar)| ustar)
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    map!(i, take!(255), |_| ExtraHeader::Padding) /* padding to 512 */
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, &typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &[u8]> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Vec<TarEntry> {
    /* Filter out empty entries */
    entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>()
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<commit_msg>simplify parse_contents<commit_after>use std::str::from_utf8;
use std::result::Result;
use nom::*;

/*
 * Core structs
 */

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: &'a [u8]
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     &'a str,
    pub mode:     &'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   &'a str,
    pub typeflag: TypeFlag,
    pub linkname: &'a str,
    pub ustar:    ExtraHeader<'a>
}

/* TODO: support vendor specific */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    PaxInterexchangeFormat,
    PaxExtendedAttributes,
    VendorSpecific
}

#[derive(Debug,PartialEq,Eq)]
pub enum ExtraHeader<'a> {
    UStar(UStarHeader<'a>),
    Padding
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    &'a str,
    pub version:  &'a str,
    pub uname:    &'a str,
    pub gname:    &'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub extra:    UStarExtraHeader<'a>
}

#[derive(Debug,PartialEq,Eq)]
pub enum UStarExtraHeader<'a> {
    PosixUStar(PosixUStarHeader<'a>),
    Pax(PaxHeader<'a>)
}

#[derive(Debug,PartialEq,Eq)]
pub struct PosixUStarHeader<'a> {
    pub prefix: &'a str
}

#[derive(Debug,PartialEq,Eq)]
pub struct PaxHeader<'a> {
    pub atime:      u64,
    pub ctime:      u64,
    pub offset:     u64,
    pub longnames:  &'a str,
    pub sparses:    Vec<Sparse>,
    pub isextended: bool,
    pub realsize:   u64,
}

#[derive(Debug,PartialEq,Eq)]
pub struct Sparse {
    pub offset:   u64,
    pub numbytes: u64
}

#[derive(Debug,PartialEq,Eq)]
pub struct Padding;

/*
 * Useful macros
 */

macro_rules! take_str_eat_garbage (
    ( $i:expr, $size:expr ) => (
        chain!($i,
            s: map_res!(take_until!("\0"), from_utf8)                  ~
            length: expr_opt!({($size as usize).checked_sub(s.len())}) ~
            take!(length),
            ||{
                s
            }
        )
    );
);

named!(parse_str4<&[u8], &str>, take_str_eat_garbage!(4));
named!(parse_str8<&[u8], &str>, take_str_eat_garbage!(8));
named!(parse_str32<&[u8], &str>, take_str_eat_garbage!(32));
named!(parse_str100<&[u8], &str>, take_str_eat_garbage!(100));
named!(parse_str155<&[u8], &str>, take_str_eat_garbage!(155));

macro_rules! take_until_expr_with_limit_consume(
  ($i:expr, $submac:ident!( $($args:tt)* ), $stop: expr, $limit: expr) => (
    {
      let mut begin = 0;
      let mut remaining = $i.len();
      let mut res = Vec::new();
      let mut cnt = 0;
      let mut err = false;
      let mut append = true;
      loop {
        match $submac!(&$i[begin..], $($args)*) {
          IResult::Done(i,o) => {
            if append {
              if $stop(&o) {
                append = false;
              } else {
                res.push(o);
              }
            }
            begin += remaining - i.len();
            remaining = i.len();
            cnt = cnt + 1;
            if cnt == $limit {
              break
            }
          },
          IResult::Error(_)  => {
            err = true;
            break;
          },
          IResult::Incomplete(_) => {
            break;
          }
        }
      }
      if err {
        IResult::Error(Err::Position(ErrorKind::TakeUntil,$i))
      } else if cnt == $limit {
        IResult::Done(&$i[begin..], res)
      } else {
        IResult::Incomplete(Needed::Unknown)
      }
    }
  );
  ($i:expr, $f:expr, $stop: expr, $limit: expr) => (
    take_until_expr_with_limit_consume!($i, call!($f), $stop, $limit);
  );
);

/*
 * Octal string parsing
 */

pub fn octal_to_u64(s: &str) -> Result<u64, &'static str> {
    let mut u = 0;

    for c in s.chars() {
        if c < '0' || c > '7' {
            return Err("invalid octal string received");
        }
        u *= 8;
        u += (c as u64) - ('0' as u64);
    }

    Ok(u)
}

fn parse_octal(i: &[u8], n: usize) -> IResult<&[u8], u64> {
    map_res!(i, take_str_eat_garbage!(n), octal_to_u64)
}

named!(parse_octal8<&[u8], u64>, apply!(parse_octal, 8));
named!(parse_octal12<&[u8], u64>, apply!(parse_octal, 12));

/*
 * TypeFlag parsing
 */

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::PaxInterexchangeFormat,
        'x' => TypeFlag::PaxExtendedAttributes,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::NormalFile
    }
}

fn bytes_to_type_flag(i: &[u8]) -> TypeFlag {
    char_to_type_flag(i[0] as char)
}

named!(parse_type_flag<&[u8], TypeFlag>, map!(take!(1), bytes_to_type_flag));

/*
 * Sparse parsing
 */

named!(parse_one_sparse<&[u8], Sparse>, map!(pair!(parse_octal12, parse_octal12), |(offset, numbytes)| Sparse { offset: offset, numbytes: numbytes }));

fn parse_sparses_with_limit(i: &[u8], limit: usize) -> IResult<&[u8], Vec<Sparse>> {
    take_until_expr_with_limit_consume!(i, parse_one_sparse, |s: &Sparse| s.offset == 0 && s.numbytes == 0, limit)
}

fn add_to_vec<'a, 'b>(sparses: &'a mut Vec<Sparse>, extra: &'b mut Vec<Sparse>) -> &'a mut Vec<Sparse> {
    while sparses.len() != 0 {
      extra.pop().map(|s| sparses.push(s));
    }
    sparses
}

fn parse_extra_sparses<'a, 'b>(i: &'a [u8], isextended: bool, sparses: &'b mut Vec<Sparse>) -> IResult<&'a [u8], &'b mut Vec<Sparse>> {
    if isextended {
        chain!(i,
            mut sps:       apply!(parse_sparses_with_limit, 21) ~
            extended:      parse_bool                     ~
            take!(7) /* padding to 512 */                 ~
            extra_sparses: apply!(parse_extra_sparses, extended, add_to_vec(sparses, &mut sps)),
            ||{
                extra_sparses
            }
        )
    } else {
        IResult::Done(i, sparses)
    }
}

fn parse_pax_extra_sparses<'a, 'b>(i: &'a [u8], h: &'b mut PaxHeader) -> IResult<&'a [u8], &'b mut Vec<Sparse>> {
    parse_extra_sparses(i, h.isextended, &mut h.sparses)
}

/*
 * Boolean parsing
 */

fn to_bool(i: &[u8]) -> bool {
    i[0] != 0
}

named!(parse_bool<&[u8], bool>, map!(take!(1), to_bool));

/*
 * UStar PAX extended parsing
 */

fn parse_ustar00_extra_pax(i: &[u8]) -> IResult<&[u8], PaxHeader> {
    chain!(i,
        atime:       parse_octal12                       ~
        ctime:       parse_octal12                       ~
        offset:      parse_octal12                       ~
        longnames:   parse_str4                          ~
        take!(1)                                         ~
        sparses:     apply!(parse_sparses_with_limit, 4) ~
        isextended:  parse_bool                          ~
        realsize:    parse_octal12                       ~
        take!(17), /* padding to 512 */
        ||{
            PaxHeader {
                atime:         atime,
                ctime:         ctime,
                offset:        offset,
                longnames:     longnames,
                sparses:       sparses,
                isextended:    isextended,
                realsize:      realsize,
            }
        }
    )
}

/*
 * UStar Posix parsing
 */

named!(parse_ustar00_extra_posix<&[u8], UStarExtraHeader>, map!(pair!(parse_str155, take!(12)), |(prefix, _)| UStarExtraHeader::PosixUStar(PosixUStarHeader { prefix: prefix })));

fn parse_ustar00_extra<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], UStarExtraHeader<'a>> {
    match *flag {
        TypeFlag::PaxInterexchangeFormat => {
            chain!(i,
                mut header: parse_ustar00_extra_pax ~
                apply!(parse_pax_extra_sparses, &mut header),
                ||{
                    UStarExtraHeader::Pax(header)
                }
            )
        },
        _ => parse_ustar00_extra_posix(i)
    }
}

fn parse_ustar00<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], ExtraHeader<'a>> {
    chain!(i,
        tag!("00")             ~
        uname:    parse_str32  ~
        gname:    parse_str32  ~
        devmajor: parse_octal8 ~
        devminor: parse_octal8 ~
        extra:    apply!(parse_ustar00_extra, flag),
        ||{
            ExtraHeader::UStar(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: devmajor,
                devminor: devminor,
                extra:    extra
            })
        }
    )
}

fn parse_ustar<'a, 'b>(i: &'a [u8], flag: &'b TypeFlag) -> IResult<&'a [u8], ExtraHeader<'a>> {
    map!(i, pair!(tag!("ustar\0"), apply!(parse_ustar00, flag)), |(_, ustar)| ustar)
}

/*
 * Posix tar archive header parsing
 */

fn parse_posix(i: &[u8]) -> IResult<&[u8], ExtraHeader> {
    map!(i, take!(255), |_| ExtraHeader::Padding) /* padding to 512 */
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     parse_str100    ~
        mode:     parse_str8      ~
        uid:      parse_octal8    ~
        gid:      parse_octal8    ~
        size:     parse_octal12   ~
        mtime:    parse_octal12   ~
        chksum:   parse_str8      ~
        typeflag: parse_type_flag ~
        linkname: parse_str100    ~
        ustar:    alt!(apply!(parse_ustar, &typeflag) | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      uid,
                gid:      gid,
                size:     size,
                mtime:    mtime,
                chksum:   chksum,
                typeflag: typeflag,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

/*
 * Contents parsing
 */

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &[u8]> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    map!(i, pair!(take!(size as usize), take!(padding as usize)), |(contents, _)| contents)
}

/*
 * Tar entry header + contents parsing
 */

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

/*
 * Tar archive parsing
 */

fn filter_entries(entries: Vec<TarEntry>) -> Vec<TarEntry> {
    /* Filter out empty entries */
    entries.into_iter().filter(|e| e.header.name != "").collect::<Vec<TarEntry>>()
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    chain!(i,
        entries: map!(many0!(parse_entry), filter_entries) ~
        eof,
        ||{
            entries
        }
    )
}

/*
 * Tests
 */

#[cfg(test)]
mod tests {
    use super::*;
    use std::str::from_utf8;
    use nom::IResult;

    #[test]
    fn octal_to_u64_ok_test() {
        assert_eq!(octal_to_u64("756"), Ok(494));
        assert_eq!(octal_to_u64(""), Ok(0));
    }

    #[test]
    fn octal_to_u64_error_test() {
        assert_eq!(octal_to_u64("1238"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("a"), Err("invalid octal string received"));
        assert_eq!(octal_to_u64("A"), Err("invalid octal string received"));
    }

    #[test]
    fn take_str_eat_garbage_test() {
        let s = b"foobar\0\0\0\0baz";
        let baz = b"baz";
        assert_eq!(take_str_eat_garbage!(&s[..], 10), IResult::Done(&baz[..], "foobar"));
    }
}
<|endoftext|>


<commit_before>use syntax_definition::*;
use scope::*;
use onig::{self, Region};
use std::usize;
use std::i32;

#[derive(Debug, Clone)]
pub struct ParseState {
    stack: Vec<StateLevel>,
    pub scope_stack: ScopeStack,
    first_line: bool,
}

#[derive(Debug, Clone)]
struct StateLevel {
    context: ContextPtr,
    prototype: Option<ContextPtr>,
    scopes_pushed: usize,
}

#[derive(Debug)]
struct RegexMatch {
    regions: Region,
    context: ContextPtr,
    pat_index: usize,
}

impl ParseState {
    pub fn new(syntax: &SyntaxDefinition) -> ParseState {
        let start_state = StateLevel {
            context: syntax.contexts["main"].clone(),
            prototype: None,
            scopes_pushed: 1,
        };
        let mut scope_stack = ScopeStack::new();
        scope_stack.push(syntax.scope);
        ParseState {
            stack: vec![start_state],
            scope_stack: scope_stack,
            first_line: true,
        }
    }

    pub fn parse_line(&mut self, line: &str) -> Vec<(usize, ScopeStackOp)> {
        assert!(self.stack.len() > 0,
                "Somehow main context was popped from the stack");
        let mut match_start = 0;
        let mut res = Vec::new();
        while self.parse_next_token(line, &mut match_start, &mut res) {
        }
        // apply operations to our scope to keep up
        // TODO do we even need to keep a scope stack in the parser state?
        for &(_, ref op) in res.iter() {
            self.scope_stack.apply(op);
        }
        return res;
    }

    fn parse_next_token(&mut self,
                        line: &str,
                        start: &mut usize,
                        ops: &mut Vec<(usize, ScopeStackOp)>)
                        -> bool {
        let cur_match = {
            let cur_level = &self.stack[self.stack.len() - 1];
            let mut min_start = usize::MAX;
            let mut cur_match: Option<RegexMatch> = None;
            for (pat_context_ptr, pat_index) in context_iter(cur_level.context.clone()) {
                let pat_context = pat_context_ptr.borrow();
                let match_pat = pat_context.match_at(pat_index);

                // println!("{:?}", match_pat.regex_str);
                let regex = match_pat.regex.as_ref().unwrap(); // TODO handle backrefs
                let mut regions = Region::new();
                // TODO caching
                let matched = regex.search_with_options(line,
                                                        *start,
                                                        line.len(),
                                                        onig::SEARCH_OPTION_NONE,
                                                        Some(&mut regions));
                if let Some(match_start) = matched {
                    if match_start < min_start {
                        min_start = match_start;
                        cur_match = Some(RegexMatch {
                            regions: regions,
                            context: pat_context_ptr.clone(),
                            pat_index: pat_index,
                        });
                    }
                }
            }
            cur_match
        };

        if let Some(reg_match) = cur_match {
            let (_, match_end) = reg_match.regions.pos(0).unwrap();
            *start = match_end;
            self.exec_pattern(line, reg_match, ops);
            true
        } else {
            false
        }
    }

    fn exec_pattern(&mut self,
                    line: &str,
                    reg_match: RegexMatch,
                    ops: &mut Vec<(usize, ScopeStackOp)>) {
        let (match_start, match_end) = reg_match.regions.pos(0).unwrap();
        let context = reg_match.context.borrow();
        let pat = context.match_at(reg_match.pat_index);

        self.push_meta_ops(true, match_start, &*context, &pat.operation, ops);
        for s in pat.scope.iter() {
            ops.push((match_start, ScopeStackOp::Push(s.clone())));
        }
        if let Some(ref capture_map) = pat.captures {
            // captures could appear in an arbitrary order, have to produce ops in right order
            // ex: ((bob)|(hi))* could match hibob in wrong order, and outer has to push first
            // we don't have to handle a capture matching multiple times, Sublime doesn't
            let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new();
            for (cap_index, scopes) in capture_map.iter() {
                if let Some((cap_start, cap_end)) = reg_match.regions.pos(*cap_index) {
                    for scope in scopes.iter() {
                        map.push(((cap_start, -((cap_end - cap_start) as i32)),
                                  ScopeStackOp::Push(scope.clone())));
                    }
                    map.push(((cap_end, i32::MIN), ScopeStackOp::Pop(scopes.len())));
                }
            }
            map.sort_by(|a, b| a.0.cmp(&b.0));
            for ((index, _), op) in map.into_iter() {
                ops.push((index, op));
            }
        }
        if !pat.scope.is_empty() {
            ops.push((match_end, ScopeStackOp::Pop(pat.scope.len())));
        }
        self.push_meta_ops(false, match_end, &*context, &pat.operation, ops);
        // TODO perform operation (with prototype)
    }

    fn push_meta_ops(&self, initial: bool, index: usize, cur_context: &Context, match_op: &MatchOperation, ops: &mut Vec<(usize, ScopeStackOp)>) {
        match match_op {
            &MatchOperation::Push(ref context_refs) | &MatchOperation::Set(ref context_refs) => {
                for r in context_refs {
                    let ctx_ptr = r.resolve();
                    let ctx = ctx_ptr.borrow();
                    let v = if initial {
                        &ctx.meta_scope
                    } else {
                        &ctx.meta_content_scope
                    };
                    for scope in v.iter() {
                        ops.push((index, ScopeStackOp::Push(scope.clone())));
                    }
                }
            },
            &MatchOperation::Pop => {
                let v = if initial {
                    &cur_context.meta_content_scope
                } else {
                    &cur_context.meta_scope
                };
                if !v.is_empty() {
                    ops.push((index, ScopeStackOp::Pop(v.len())));
                }
            },
            &MatchOperation::None => (),
        }
    }
}

#[cfg(test)]
mod tests {
    use package_set::PackageSet;
    use parser::*;
    use scope::*;
    fn debug_print_ops(line: &str,
                       scope_repo: &ScopeRepository,
                       ops: &Vec<(usize, ScopeStackOp)>) {
        println!("{}", line);
        for &(i, ref op) in ops.iter() {
            print!("{: <1$}", "", i);
            match op {
                &ScopeStackOp::Push(s) => {
                    println!("^ +{}", scope_repo.to_string(s));
                }
                &ScopeStackOp::Pop(count) => {
                    println!("^ pop {}", count);
                }
            }
        }
    }

    #[test]
    fn can_parse() {
        let mut ps = PackageSet::load_from_folder("testdata/Packages").unwrap();
        let mut state = {
            let syntax = ps.find_syntax_by_name("Ruby on Rails").unwrap();
            ParseState::new(syntax)
        };

        let line = "module Bob::Wow::Troll::Five; 5; end";
        let ops = state.parse_line(line);
        debug_print_ops(line, &ps.scope_repo, &ops);

        let test_ops = vec![
            (0, ScopeStackOp::Push(ps.scope_repo.build("meta.module.ruby"))),
            (0, ScopeStackOp::Push(ps.scope_repo.build("keyword.control.module.ruby"))),
            (6, ScopeStackOp::Pop(1)),
            (7, ScopeStackOp::Push(ps.scope_repo.build("entity.name.type.module.ruby"))),
            (7, ScopeStackOp::Push(ps.scope_repo.build("entity.other.inherited-class.module.first.ruby"))),
            (10, ScopeStackOp::Push(ps.scope_repo.build("punctuation.separator.inheritance.ruby"))),
            (12, ScopeStackOp::Pop(1)),
            (12, ScopeStackOp::Pop(1)),
        ];
        assert_eq!(&ops[0..test_ops.len()], &test_ops[..]);

        let line2 = "__END__\nhilol";
        let ops2 = state.parse_line(line2);
        debug_print_ops(line2, &ps.scope_repo, &ops2);
        let test_ops2 = vec![
            (0, ScopeStackOp::Push(ps.scope_repo.build("string.unquoted.program-block.ruby"))),
            (8, ScopeStackOp::Pop(1)),
            (8, ScopeStackOp::Push(ps.scope_repo.build("text.plain"))),
        ];
        assert_eq!(&ops2[0..test_ops2.len()], &test_ops2[..]);

        // assert!(false);
    }
}
<commit_msg>Basic operation execution without prototypes<commit_after>use syntax_definition::*;
use scope::*;
use onig::{self, Region};
use std::usize;
use std::i32;

#[derive(Debug, Clone)]
pub struct ParseState {
    stack: Vec<StateLevel>,
    pub scope_stack: ScopeStack,
    first_line: bool,
}

#[derive(Debug, Clone)]
struct StateLevel {
    context: ContextPtr,
    prototype: Option<ContextPtr>,
}

#[derive(Debug)]
struct RegexMatch {
    regions: Region,
    context: ContextPtr,
    pat_index: usize,
}

impl ParseState {
    pub fn new(syntax: &SyntaxDefinition) -> ParseState {
        let start_state = StateLevel {
            context: syntax.contexts["main"].clone(),
            prototype: None,
        };
        let mut scope_stack = ScopeStack::new();
        scope_stack.push(syntax.scope);
        ParseState {
            stack: vec![start_state],
            scope_stack: scope_stack,
            first_line: true,
        }
    }

    pub fn parse_line(&mut self, line: &str) -> Vec<(usize, ScopeStackOp)> {
        assert!(self.stack.len() > 0,
                "Somehow main context was popped from the stack");
        let mut match_start = 0;
        let mut res = Vec::new();
        while self.parse_next_token(line, &mut match_start, &mut res) {
        }
        // apply operations to our scope to keep up
        // TODO do we even need to keep a scope stack in the parser state?
        for &(_, ref op) in res.iter() {
            self.scope_stack.apply(op);
        }
        return res;
    }

    fn parse_next_token(&mut self,
                        line: &str,
                        start: &mut usize,
                        ops: &mut Vec<(usize, ScopeStackOp)>)
                        -> bool {
        let cur_match = {
            let cur_level = &self.stack[self.stack.len() - 1];
            let mut min_start = usize::MAX;
            let mut cur_match: Option<RegexMatch> = None;
            // TODO iterate through prototypes first
            for (pat_context_ptr, pat_index) in context_iter(cur_level.context.clone()) {
                let pat_context = pat_context_ptr.borrow();
                let match_pat = pat_context.match_at(pat_index);

                // println!("{:?}", match_pat.regex_str);
                let regex = match_pat.regex.as_ref().unwrap(); // TODO handle backrefs
                let mut regions = Region::new();
                // TODO caching
                let matched = regex.search_with_options(line,
                                                        *start,
                                                        line.len(),
                                                        onig::SEARCH_OPTION_NONE,
                                                        Some(&mut regions));
                if let Some(match_start) = matched {
                    if match_start < min_start {
                        min_start = match_start;
                        cur_match = Some(RegexMatch {
                            regions: regions,
                            context: pat_context_ptr.clone(),
                            pat_index: pat_index,
                        });
                    }
                }
            }
            cur_match
        };

        if let Some(reg_match) = cur_match {
            let (_, match_end) = reg_match.regions.pos(0).unwrap();
            *start = match_end;
            self.exec_pattern(line, reg_match, ops);
            true
        } else {
            false
        }
    }

    fn exec_pattern(&mut self,
                    line: &str,
                    reg_match: RegexMatch,
                    ops: &mut Vec<(usize, ScopeStackOp)>) {
        let (match_start, match_end) = reg_match.regions.pos(0).unwrap();
        let context = reg_match.context.borrow();
        let pat = context.match_at(reg_match.pat_index);

        self.push_meta_ops(true, match_start, &*context, &pat.operation, ops);
        for s in pat.scope.iter() {
            ops.push((match_start, ScopeStackOp::Push(s.clone())));
        }
        if let Some(ref capture_map) = pat.captures {
            // captures could appear in an arbitrary order, have to produce ops in right order
            // ex: ((bob)|(hi))* could match hibob in wrong order, and outer has to push first
            // we don't have to handle a capture matching multiple times, Sublime doesn't
            let mut map: Vec<((usize, i32), ScopeStackOp)> = Vec::new();
            for (cap_index, scopes) in capture_map.iter() {
                if let Some((cap_start, cap_end)) = reg_match.regions.pos(*cap_index) {
                    for scope in scopes.iter() {
                        map.push(((cap_start, -((cap_end - cap_start) as i32)),
                                  ScopeStackOp::Push(scope.clone())));
                    }
                    map.push(((cap_end, i32::MIN), ScopeStackOp::Pop(scopes.len())));
                }
            }
            map.sort_by(|a, b| a.0.cmp(&b.0));
            for ((index, _), op) in map.into_iter() {
                ops.push((index, op));
            }
        }
        if !pat.scope.is_empty() {
            ops.push((match_end, ScopeStackOp::Pop(pat.scope.len())));
        }
        self.push_meta_ops(false, match_end, &*context, &pat.operation, ops);

        self.perform_op(&pat.operation);
    }

    fn push_meta_ops(&self,
                     initial: bool,
                     index: usize,
                     cur_context: &Context,
                     match_op: &MatchOperation,
                     ops: &mut Vec<(usize, ScopeStackOp)>) {
        match match_op {
            &MatchOperation::Push(ref context_refs) |
            &MatchOperation::Set(ref context_refs) => {
                for r in context_refs {
                    let ctx_ptr = r.resolve();
                    let ctx = ctx_ptr.borrow();
                    let v = if initial {
                        &ctx.meta_scope
                    } else {
                        &ctx.meta_content_scope
                    };
                    for scope in v.iter() {
                        ops.push((index, ScopeStackOp::Push(scope.clone())));
                    }
                }
            }
            &MatchOperation::Pop => {
                let v = if initial {
                    &cur_context.meta_content_scope
                } else {
                    &cur_context.meta_scope
                };
                if !v.is_empty() {
                    ops.push((index, ScopeStackOp::Pop(v.len())));
                }
            }
            &MatchOperation::None => (),
        }
    }

    fn perform_op(&mut self, match_op: &MatchOperation) {
        let ctx_refs = match match_op {
            &MatchOperation::Push(ref ctx_refs) => ctx_refs,
            &MatchOperation::Set(ref ctx_refs) => {
                self.stack.pop();
                ctx_refs
            }
            &MatchOperation::Pop => {
                self.stack.pop();
                return;
            }
            &MatchOperation::None => return,
        };
        for r in ctx_refs {
            let ctx_ptr = r.resolve();
            self.stack.push(StateLevel {
                context: ctx_ptr,
                prototype: None, // TODO prototype
            });
        }
    }
}

#[cfg(test)]
mod tests {
    use package_set::PackageSet;
    use parser::*;
    use scope::*;
    fn debug_print_ops(line: &str,
                       scope_repo: &ScopeRepository,
                       ops: &Vec<(usize, ScopeStackOp)>) {
        for &(i, ref op) in ops.iter() {
            println!("{}", line);
            print!("{: <1$}", "", i);
            match op {
                &ScopeStackOp::Push(s) => {
                    println!("^ +{}", scope_repo.to_string(s));
                }
                &ScopeStackOp::Pop(count) => {
                    println!("^ pop {}", count);
                }
            }
        }
    }

    #[test]
    fn can_parse() {
        use scope::ScopeStackOp::{Push, Pop};
        let mut ps = PackageSet::load_from_folder("testdata/Packages").unwrap();
        let mut state = {
            let syntax = ps.find_syntax_by_name("Ruby on Rails").unwrap();
            ParseState::new(syntax)
        };

        let line = "module Bob::Wow::Troll::Five; 5; end";
        let ops = state.parse_line(line);
        debug_print_ops(line, &ps.scope_repo, &ops);

        let test_ops = vec![
            (0, Push(ps.scope_repo.build("meta.module.ruby"))),
            (0, Push(ps.scope_repo.build("keyword.control.module.ruby"))),
            (6, Pop(1)),
            (7, Push(ps.scope_repo.build("entity.name.type.module.ruby"))),
            (7, Push(ps.scope_repo.build("entity.other.inherited-class.module.first.ruby"))),
            (10, Push(ps.scope_repo.build("punctuation.separator.inheritance.ruby"))),
            (12, Pop(1)),
            (12, Pop(1)),
        ];
        assert_eq!(&ops[0..test_ops.len()], &test_ops[..]);

        let line2 = "def lol(wow = 5)";
        let ops2 = state.parse_line(line2);
        debug_print_ops(line2, &ps.scope_repo, &ops2);
        let test_ops2 =
            vec![(0, Push(ps.scope_repo.build("meta.function.method.with-arguments.ruby"))),
                 (0, Push(ps.scope_repo.build("keyword.control.def.ruby"))),
                 (3, Pop(1)),
                 (4, Push(ps.scope_repo.build("entity.name.function.ruby"))),
                 (7, Pop(1)),
                 (7, Push(ps.scope_repo.build("punctuation.definition.parameters.ruby"))),
                 (8, Pop(1)),
                 (8, Push(ps.scope_repo.build("variable.parameter.function.ruby"))),
                 (12, Push(ps.scope_repo.build("keyword.operator.assignment.ruby"))),
                 (13, Pop(1)),
                 (14, Push(ps.scope_repo.build("constant.numeric.ruby"))),
                 (15, Pop(1)),
                 (15, Pop(1)),
                 (15, Push(ps.scope_repo.build("punctuation.definition.parameters.ruby"))),
                 (16, Pop(1)),
                 (16, Pop(1))];
        assert_eq!(ops2, test_ops2);

        // assert!(false);
    }
}
<|endoftext|>


<commit_before>pub enum Entry {
  Entity {id: String, value: String}
}

pub enum Value {
  Str(String)
}

pub struct Parser<'a> {
  source: std::str::Chars<'a>,
  ch: Option<char>,
  pos: u8
}

impl<'a> Parser<'a> {
  pub fn new(source: &'a str) -> Parser<'a> {
    Parser {
      source: source.chars(),
      ch: None,
      pos: 0
    }
  }

  fn bump(&mut self) {
    self.ch = self.source.next();

    self.pos += 1;
  }

  fn ch_is(&self, ch: char) -> bool {
    self.ch == Some(ch)
  }

  fn parse_whitespace(&mut self) {
    while self.ch_is(' ') ||
          self.ch_is('\n') ||
          self.ch_is('\t') ||
          self.ch_is('\r') { self.bump(); }
  }

  pub fn parse(&mut self) -> Vec<Entry> {
    let mut entries: Vec<Entry> = Vec::new();
    self.bump();

    loop {
      self.parse_whitespace();
      let ch = match self.ch {
        Some(ch) => ch,
        None => break
      };

      if ch == '<' {
        entries.push(self.parse_entry());
      }
    }
    entries
  }

  fn parse_entry(&mut self) -> Entry {
    let id = self.parse_identifier();
    let val = match self.ch {
      Some(_) => self.parse_entity(id),
      None => panic!()
    };

    self.parse_whitespace();
    if !self.ch_is('>') {
      panic!();
    }
    self.bump();
    val
  }

  fn parse_entity(&mut self, id: String) -> Entry {
    if !self.ch_is(' ') {
      panic!();
    }
    self.parse_whitespace();
    let value = self.parse_value();
    Entry::Entity{id: id, value:value}
  }

  fn parse_identifier(&mut self) -> String {
    let mut id = String::new();

    loop {
      self.bump();
      let ch = match self.ch {
        Some(c) => c,
        None => break,
      };

      match ch {
        'a'...'z' | 'A'...'Z' | '0'...'9' | '_' => id.push(ch),
        _ => break,
      }
    }

    id
  }

  fn parse_value(&mut self) -> String {
    match self.ch {
      Some('"') | Some('\'') => self.parse_string(),
      _ => panic!()
    }
  }

  fn parse_string(&mut self) -> String {
    let mut s = String::new();
    let quote = self.ch.unwrap();

    loop {
      self.bump();
      match self.ch {
        Some(c) if c == quote => { self.bump(); break },
        Some(c) => s.push(c),
        None => panic!()
      }
    }
    s
  }
}

fn read_file() -> String {
  let s = "<entity1 \"foo\"> <entity2 \"foo2\">".to_string();
  return s
}

fn main() {
  let source = read_file();
  let mut parser = Parser::new(source.trim());
  let mut entries = parser.parse();

  loop {
    if entries.is_empty() {
      break;
    }
    let entry1 = Some(entries.remove(0));
    let (id, value) = match entry1 {
      Some(Entry::Entity{id, value}) => (id.clone(), value.clone()),
      None => break
    };

    println!("The result id is {}", id);
    println!("The result value is {}", value);
  }
}
<commit_msg>Partially support hashes<commit_after>use std::collections::HashMap;

pub type ParseResult<T> = Result<T, ParseError>;

pub struct ParseError {
  pub pos: u8,
}

pub enum Value {
  Str(String),
  Hash(HashMap<String, Value>, Option<String>)
}

pub enum Entry {
  Entity {id: String, value: Value}
}

pub struct Parser<'a> {
  source: std::str::Chars<'a>,
  ch: Option<char>,
  pos: u8
}

impl<'a> Parser<'a> {
  pub fn new(source: &'a str) -> Parser<'a> {
    Parser {
      source: source.chars(),
      ch: None,
      pos: 0
    }
  }

  fn bump(&mut self) {
    self.ch = self.source.next();

    self.pos += 1;
  }

  fn ch_is(&self, ch: char) -> bool {
    self.ch == Some(ch)
  }

  fn parse_whitespace(&mut self) {
    while self.ch_is(' ') ||
          self.ch_is('\n') ||
          self.ch_is('\t') ||
          self.ch_is('\r') { self.bump(); }
  }

  pub fn parse(&mut self) -> Vec<Entry> {
    let mut entries: Vec<Entry> = Vec::new();
    self.bump();

    loop {
      self.parse_whitespace();
      let ch = match self.ch {
        Some(ch) => ch,
        None => break
      };

      if ch == '<' {
        entries.push(self.parse_entry());
      }
    }
    entries
  }

  fn parse_entry(&mut self) -> Entry {
    let id = self.parse_identifier();
    let val = match self.ch {
      Some(_) => self.parse_entity(id),
      None => panic!()
    };

    self.parse_whitespace();
    if !self.ch_is('>') {
      panic!();
    }
    self.bump();
    val
  }

  fn parse_entity(&mut self, id: String) -> Entry {
    if !self.ch_is(' ') {
      panic!();
    }
    self.parse_whitespace();
    let value = self.parse_value();
    Entry::Entity{id: id, value:value}
  }

  fn parse_identifier(&mut self) -> String {
    let mut id = String::new();

    loop {
      self.bump();
      let ch = match self.ch {
        Some(c) => c,
        None => break,
      };

      match ch {
        'a'...'z' | 'A'...'Z' | '0'...'9' | '_' => id.push(ch),
        _ => break,
      }
    }

    id
  }

  fn parse_value(&mut self) -> Value {
    match self.ch {
      Some('"') | Some('\'') => self.parse_string(),
      Some('{') => self.parse_hash(),
      _ => panic!()
    }
  }

  fn parse_string(&mut self) -> Value {
    let mut s = String::new();
    let quote = self.ch.unwrap();

    loop {
      self.bump();
      match self.ch {
        Some(c) if c == quote => { self.bump(); break },
        Some(c) => s.push(c),
        None => panic!()
      }
    }
    Value::Str(s)
  }

  fn parse_hash(&mut self) -> Value {
    self.bump();
    self.parse_whitespace();

    let mut map = HashMap::new();
    let mut default = None;

    loop {
      let id = self.parse_identifier();
      self.parse_whitespace();
      self.bump();
      self.parse_whitespace();
      let value = self.parse_value();
      self.bump();
      map.insert(id, value);
      break;
    }
    Value::Hash(map, default)
  }
}

fn read_file() -> String {
  let s = "<entity1 {one: 'foo'}> <entity2 'foo'>".to_string();
  return s
}

fn main() {
  let source = read_file();
  let mut parser = Parser::new(source.trim());
  let mut entries = parser.parse();

  loop {
    if entries.is_empty() {
      break;
    }
    let entry1 = Some(entries.remove(0));
    let id = match entry1 {
      Some(Entry::Entity{id, value}) => id.clone(),
      None => break
    };

    println!("ID: {}", id);
  }
}
<|endoftext|>


<commit_before>//! The parser functionality

use c_signatures::*;
use global::*;
use tree::*;

use std::ffi::CString;
use std::fmt;

enum XmlParserOption {
    XmlParseRecover = 1, // Relaxed parsing
    // XML_PARSE_NODEFDTD = 4, // do not default a doctype if not found
    XmlParseNoerror = 32, // suppress error reports
    XmlParseNowarning = 64, // suppress warning reports
    // XML_PARSE_PEDANTIC = 128, // pedantic error reporting
    // XML_PARSE_NOBLANKS = 256, // remove blank nodes
    // XML_PARSE_NONET = 2048, // Forbid network access
    // XML_PARSE_NOIMPLIED = 8192, // Do not add implied Xml/body... elements
    // XML_PARSE_COMPACT = 65536, // compact small text nodes
    // XML_PARSE_IGNORE_ENC = 2097152, // ignore internal document encoding hint
}

enum HtmlParserOption {
    HtmlParseRecover = 1, // Relaxed parsing
    // HTML_PARSE_NODEFDTD = 4, // do not default a doctype if not found
    HtmlParseNoerror = 32, // suppress error reports
    HtmlParseNowarning = 64, // suppress warning reports
    // HTML_PARSE_PEDANTIC = 128, // pedantic error reporting
    // HTML_PARSE_NOBLANKS = 256, // remove blank nodes
    // HTML_PARSE_NONET = 2048, // Forbid network access
    // HTML_PARSE_NOIMPLIED = 8192, // Do not add implied html/body... elements
    // HTML_PARSE_COMPACT = 65536, // compact small text nodes
    // HTML_PARSE_IGNORE_ENC = 2097152, // ignore internal document encoding hint
}

///Parser Errors
pub enum XmlParseError {
    ///Parsing returned a null pointer as document pointer
    GotNullPointer,
}

impl fmt::Debug for XmlParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            XmlParseError::GotNullPointer => write!(f, "Got a Null pointer")
        }
    }
}
#[derive(PartialEq)]
pub enum ParseFormat {
  XML,
  HTML
}
pub struct Parser {
  pub format : ParseFormat
}
impl Default for Parser {
  fn default() -> Self {
    _libxml_global_init();
    Parser { format : ParseFormat::XML}
  }
}
impl Parser {
  pub fn default_html() -> Self {
    _libxml_global_init();
    Parser { format : ParseFormat::HTML}
  }
  ///Parses the XML/HTML file `filename` to generate a new `Document`
  pub fn parse_file(&self, filename : &str) -> Result<Document, XmlParseError> {
    let c_filename = CString::new(filename).unwrap().as_ptr();
    let c_utf8 = CString::new("utf-8").unwrap().as_ptr();
    let options : u32 = XmlParserOption::XmlParseRecover as u32 +
                        XmlParserOption::XmlParseNoerror as u32 +
                        XmlParserOption::XmlParseNowarning as u32;
    match self.format {
      ParseFormat::XML => { unsafe {
        xmlKeepBlanksDefault(1);
        let docptr = xmlReadFile(c_filename, c_utf8, options);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } }
      },
      ParseFormat::HTML => {
        // TODO: Allow user-specified options later on
        unsafe {
          let options : u32 = HtmlParserOption::HtmlParseRecover as u32 +
                              HtmlParserOption::HtmlParseNoerror as u32 +
                              HtmlParserOption::HtmlParseNowarning as u32;
          xmlKeepBlanksDefault(1);
          let docptr = htmlReadFile(c_filename, c_utf8, options);
          match docptr.is_null() {
            true => Err(XmlParseError::GotNullPointer),
            false => Ok(Document::new_ptr(docptr))
          }
        }
      }
    }
  }

  ///Parses the XML string (0-terminated) contained in 'xml_string' to generate a new 'XmlDoc'
  pub fn parse_string(&self, input_string: &str) -> Result<Document, XmlParseError> {
    let c_string = CString::new(input_string).unwrap().as_ptr();
    let c_utf8 = CString::new("utf-8").unwrap().as_ptr();
    match self.format {
      ParseFormat::XML => { unsafe {
        let docptr = xmlParseDoc(c_string);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } } },
      ParseFormat::HTML => { unsafe {
        let docptr = htmlParseDoc(c_string, c_utf8);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } } },
    }
  }
}

impl Drop for Parser {
  fn drop(&mut self) {
    // unsafe {
    //   xmlCleanupParser();
    // }
    _libxml_global_drop();
  }
}<commit_msg>doc update<commit_after>//! The parser functionality

use c_signatures::*;
use global::*;
use tree::*;

use std::ffi::CString;
use std::fmt;

enum XmlParserOption {
    XmlParseRecover = 1, // Relaxed parsing
    // XML_PARSE_NODEFDTD = 4, // do not default a doctype if not found
    XmlParseNoerror = 32, // suppress error reports
    XmlParseNowarning = 64, // suppress warning reports
    // XML_PARSE_PEDANTIC = 128, // pedantic error reporting
    // XML_PARSE_NOBLANKS = 256, // remove blank nodes
    // XML_PARSE_NONET = 2048, // Forbid network access
    // XML_PARSE_NOIMPLIED = 8192, // Do not add implied Xml/body... elements
    // XML_PARSE_COMPACT = 65536, // compact small text nodes
    // XML_PARSE_IGNORE_ENC = 2097152, // ignore internal document encoding hint
}

enum HtmlParserOption {
    HtmlParseRecover = 1, // Relaxed parsing
    // HTML_PARSE_NODEFDTD = 4, // do not default a doctype if not found
    HtmlParseNoerror = 32, // suppress error reports
    HtmlParseNowarning = 64, // suppress warning reports
    // HTML_PARSE_PEDANTIC = 128, // pedantic error reporting
    // HTML_PARSE_NOBLANKS = 256, // remove blank nodes
    // HTML_PARSE_NONET = 2048, // Forbid network access
    // HTML_PARSE_NOIMPLIED = 8192, // Do not add implied html/body... elements
    // HTML_PARSE_COMPACT = 65536, // compact small text nodes
    // HTML_PARSE_IGNORE_ENC = 2097152, // ignore internal document encoding hint
}

///Parser Errors
pub enum XmlParseError {
    ///Parsing returned a null pointer as document pointer
    GotNullPointer,
}

impl fmt::Debug for XmlParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            XmlParseError::GotNullPointer => write!(f, "Got a Null pointer")
        }
    }
}
#[derive(PartialEq)]
pub enum ParseFormat {
  XML,
  HTML
}
pub struct Parser {
  pub format : ParseFormat
}
impl Default for Parser {
  fn default() -> Self {
    _libxml_global_init();
    Parser { format : ParseFormat::XML}
  }
}
impl Parser {
  pub fn default_html() -> Self {
    _libxml_global_init();
    Parser { format : ParseFormat::HTML}
  }
  ///Parses the XML/HTML file `filename` to generate a new `Document`
  pub fn parse_file(&self, filename : &str) -> Result<Document, XmlParseError> {
    let c_filename = CString::new(filename).unwrap().as_ptr();
    let c_utf8 = CString::new("utf-8").unwrap().as_ptr();
    let options : u32 = XmlParserOption::XmlParseRecover as u32 +
                        XmlParserOption::XmlParseNoerror as u32 +
                        XmlParserOption::XmlParseNowarning as u32;
    match self.format {
      ParseFormat::XML => { unsafe {
        xmlKeepBlanksDefault(1);
        let docptr = xmlReadFile(c_filename, c_utf8, options);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } }
      },
      ParseFormat::HTML => {
        // TODO: Allow user-specified options later on
        unsafe {
          let options : u32 = HtmlParserOption::HtmlParseRecover as u32 +
                              HtmlParserOption::HtmlParseNoerror as u32 +
                              HtmlParserOption::HtmlParseNowarning as u32;
          xmlKeepBlanksDefault(1);
          let docptr = htmlReadFile(c_filename, c_utf8, options);
          match docptr.is_null() {
            true => Err(XmlParseError::GotNullPointer),
            false => Ok(Document::new_ptr(docptr))
          }
        }
      }
    }
  }

  ///Parses the XML/HTML string `input_string` to generate a new `Document`
  pub fn parse_string(&self, input_string: &str) -> Result<Document, XmlParseError> {
    let c_string = CString::new(input_string).unwrap().as_ptr();
    let c_utf8 = CString::new("utf-8").unwrap().as_ptr();
    match self.format {
      ParseFormat::XML => { unsafe {
        let docptr = xmlParseDoc(c_string);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } } },
      ParseFormat::HTML => { unsafe {
        let docptr = htmlParseDoc(c_string, c_utf8);
        match docptr.is_null() {
          true => Err(XmlParseError::GotNullPointer),
          false => Ok(Document::new_ptr(docptr))
        } } },
    }
  }
}

impl Drop for Parser {
  fn drop(&mut self) {
    // unsafe {
    //   xmlCleanupParser();
    // }
    _libxml_global_drop();
  }
}<|endoftext|>