<commit_before>use ast::{TIdent, TNumber, TString, TSemicolon, TComment, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TBinOp, TAssignOp, TUnaryOp, TArrow};
use ast::{OpAdd, OpSub, OpMul, OpDiv, OpMod};
use ast::{BitAnd, BitOr, BitXor, BitShl, BitShr};
use ast::{CompEqual, CompStrictEqual, CompNotEqual, CompStrictNotEqual, CompLessThan, CompGreaterThan, CompLessThanOrEqual, CompGreaterThanOrEqual};
use ast::{LogAnd, LogOr};
use ast::{UnaryNot};
use ast::{BinNum, BinBit, BinComp, BinLog};
use ast::{Token, TokenData};
use std::io::{BufReader, BufferedReader, Buffer, IoResult, EndOfFile};
use std::strbuf::StrBuf;
use std::char::from_u32;
use std::num::from_str_radix;
#[deriving(Clone, Eq)]
/// The type of string used
pub enum StringType {
	/// `"` - Double-quoted
	DoubleQuote,
	/// `'` - Single-quoted
	SingleQuote
}
#[deriving(Clone, Eq)]
/// The type of comment used
pub enum CommentType {
	/// `/*...*/` - A multi-line comment
	MultiLineComment,
	/// `//...` - A single-line comment
	SingleLineComment
}
#[deriving(Clone, Eq)]
/// The type of number used
pub enum NumberType {
	/// A decimal number, such as `3.1415`
	DecimalNumber,
	/// A hexadecimal number, such as `0xFF00FF`
	HexadecimalNumber,
	/// An octal number, such as `011`
	OctalNumber
}
/// A Javascript lexer
pub struct Lexer<B> {
	/// The list of tokens generated so far
	pub tokens : Vec<Token>,
	/// The current line number in the script
	line_number : uint,
	/// The current column number in the script
	column_number : uint,
	/// The reader
	buffer: B,
	/// The peeked character buffer
	peek_buffer: StrBuf
}
impl<B:Buffer> Lexer<B> {
	/// Creates a new lexer with empty buffers
	pub fn new(buffer: B) -> Lexer<B> {
		return Lexer {
			tokens: Vec::new(),
			line_number: 1,
			column_number: 0,
			buffer: buffer,
			peek_buffer: StrBuf::new()
		};
	}
	fn push_token(&mut self, tk:TokenData) {
		self.tokens.push(Token::new(tk, self.line_number, self.column_number))
	}
	/// Processes an input stream from a string into an array of tokens
	pub fn lex_str(script:&str) -> Vec<Token> {
		let script_bytes:&[u8] = script.as_bytes();
		let reader = BufReader::new(script_bytes);
		let buf_reader = BufferedReader::new(reader);
		let mut lexer = Lexer::new(buf_reader);
		lexer.lex().unwrap();
		lexer.tokens
	}
	fn next(&mut self) -> IoResult<char> {
		if self.peek_buffer.len() == 0 {
			self.buffer.read_char()
		} else {
			Ok(self.peek_buffer.pop_char().unwrap())
		}
	}
	fn peek_for(&mut self, peek:char) -> bool {
		if self.peek_buffer.len() > 0 {
			self.peek_buffer.pop_char() == Some(peek)
		} else {
			match self.buffer.read_char() {
				Ok(ch) if ch == peek => {
					self.column_number += 1;
					true
				},
				Ok(ch) if ch != peek => {
					self.peek_buffer.push_char(ch);
					false
				},
				_ => false
			}
		}
	}
	/// Processes an input stream from the `buffer` into a vector of tokens
	pub fn lex(&mut self) -> IoResult<()> {
		loop {
			let ch = match self.next() {
				Ok(ch) => ch,
				Err(ref err) if err.kind == EndOfFile => break,
				Err(err) => return Err(err)
			};
			self.column_number += 1;
			match ch {
				'"' | '\'' => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\'' if ch == '\'' => {
								break;
							},
							'"' if ch == '"' => {
								break;
							},
							'\\' => {
								let escape = try!(self.next());
								if escape != '\n' {
									let escaped_ch = match escape {
										'n' => '\n',
										'r' => '\r',
										't' => '\t',
										'b' => '\x08',
										'f' => '\x0c',
										'0' => '\0',
										'x' => {
											let mut nums = StrBuf::with_capacity(2);
											for _ in range(0, 2) {
												nums.push_char(try!(self.next()));
											}
											self.column_number += 2;
											let as_num = match from_str_radix(nums.as_slice(), 16) {
												Some(v) => v,
												None => 0
											};
											match from_u32(as_num) {
												Some(v) => v,
												None => fail!("{}:{}: {} is not a valid unicode scalar value", self.line_number, self.column_number, as_num)
											}
										},
										'u' => {
											let mut nums = StrBuf::new();
											for _ in range(0, 4) {
												nums.push_char(try!(self.next()));
											}
											self.column_number += 4;
											let as_num = match from_str_radix(nums.as_slice(), 16) {
												Some(v) => v,
												None => 0
											};
											match from_u32(as_num) {
												Some(v) => v,
												None => fail!("{}:{}: {} is not a valid unicode scalar value", self.line_number, self.column_number, as_num)
											}
										},
										'\'' | '"' => escape,
										_ => fail!("{}:{}: Invalid escape `{}`", self.line_number, self.column_number, ch)
									};
									buf.push_char(escaped_ch);
								}
							},
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TString(buf))
				},
				'0' if self.peek_for('x') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							ch if ch.is_digit_radix(16) => buf.push_char(ch),
							ch => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(from_str_radix(buf.as_slice(), 16).unwrap()));
				},
				'0' => {
					let mut buf = StrBuf::new();
					let mut gone_decimal = false;
					loop {
						let ch = try!(self.next());
						match ch {
							_ if ch.is_digit_radix(8) => buf.push_char(ch),
							'8' | '9' | '.' => {
								gone_decimal = true;
								buf.push_char(ch);
							},
							ch => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(if gone_decimal {
						from_str(buf.as_slice())
					} else {
						from_str_radix(buf.as_slice(), 8)
					}.unwrap()));
				},
				_ if ch.is_digit() => {
					let mut buf = StrBuf::new();
					buf.push_char(ch);
					loop {
						let ch = try!(self.next());
						match ch {
							'.' => buf.push_char(ch),
							_ if ch.is_digit() => buf.push_char(ch),
							_ => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(from_str(buf.as_slice()).unwrap()));
				},
				_ if ch.is_alphabetic() || ch == '$' || ch == '_' => {
					let mut buf = StrBuf::new();
					buf.push_char(ch);
					loop {
						let ch = try!(self.next());
						match ch {
							_ if ch.is_alphabetic() || ch.is_digit() => buf.push_char(ch),
							_ => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TIdent(buf));
				},
				';' => {
					self.push_token(TSemicolon);
				},
				':' => {
					self.push_token(TColon);
				},
				'.' => {
					self.push_token(TDot);
				},
				'(' => {
					self.push_token(TOpenParen);
				},
				')' => {
					self.push_token(TCloseParen);
				},
				',' => {
					self.push_token(TComma);
				},
				'{' => {
					self.push_token(TOpenBlock);
				},
				'}' => {
					self.push_token(TCloseBlock);
				},
				'[' => {
					self.push_token(TOpenArray);
				},
				']' => {
					self.push_token(TCloseArray);
				},
				'?' => {
					self.push_token(TQuestion);
				},
				'/' if self.peek_for('/') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\n' => break,
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TComment(buf));
				},
				'/' if self.peek_for('*') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\n' => break,
							'*' if self.peek_for('/') => break,
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TComment(buf));
				},
				'/' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpDiv)));
				},
				'/' => {
					self.push_token(TBinOp(BinNum(OpDiv)));
				},
				'*' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpMul)));
				},
				'*' => {
					self.push_token(TBinOp(BinNum(OpMul)));
				},
				'+' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpAdd)));
				},
				'+' => {
					self.push_token(TBinOp(BinNum(OpAdd)));
				},
				'-' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpSub)));
				},
				'-' => {
					self.push_token(TBinOp(BinNum(OpSub)));
				},
				'%' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpMod)));
				},
				'%' => {
					self.push_token(TBinOp(BinNum(OpMod)));
				},
				'|' if self.peek_for('|') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinLog(LogOr)));
					} else {
						self.push_token(TBinOp(BinLog(LogOr)));
					}
				},
				'|' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitOr)));
				},
				'|' => {
					self.push_token(TBinOp(BinBit(BitOr)));
				},
				'&' if self.peek_for('&') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinLog(LogAnd)));
					} else {
						self.push_token(TBinOp(BinLog(LogAnd)));
					}
				},
				'&' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitAnd)));
				},
				'&' => {
					self.push_token(TBinOp(BinBit(BitAnd)));
				},
				'^' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitXor)));
				},
				'^' => {
					self.push_token(TBinOp(BinBit(BitXor)));
				},
				'=' if self.peek_for('>') => {
					self.push_token(TArrow);
				},
				'=' if self.peek_for('=') => {
					if self.peek_for('=') {
						self.push_token(TBinOp(BinComp(CompStrictEqual)));
					} else {
						self.push_token(TBinOp(BinComp(CompEqual)));
					}
				},
				'=' => {
					self.push_token(TEqual);
				},
				'<' if self.peek_for('=') => {
					self.push_token(TBinOp(BinComp(CompLessThanOrEqual)));
				},
				'<' if self.peek_for('<') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinBit(BitShl)));
					} else {
						self.push_token(TBinOp(BinBit(BitShl)));
					}
				},
				'<' => {
					self.push_token(TBinOp(BinComp(CompLessThan)));
				},
				'>' if self.peek_for('=') => {
					self.push_token(TBinOp(BinComp(CompGreaterThanOrEqual)));
				},
				'>' if self.peek_for('>') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinBit(BitShr)));
					} else {
						self.push_token(TBinOp(BinBit(BitShr)));
					}
				},
				'>' => {
					self.push_token(TBinOp(BinComp(CompGreaterThan)));
				},
				'!' if self.peek_for('=') => {
					if self.peek_for('=') {
						self.push_token(TBinOp(BinComp(CompStrictNotEqual)));
					} else {
						self.push_token(TBinOp(BinComp(CompNotEqual)));
					}
				},
				'!' => {
					self.push_token(TUnaryOp(UnaryNot));
				},
				'\n' => {
					self.line_number += 1;
					self.column_number = 0;
				},
				'\r' => {
					self.column_number = 0;
				},
				ch => fail!("{}:{}: Unexpected '{}'", self.line_number, self.column_number, ch)
			};
		}
		Ok(())
	}
}<commit_msg>Expand unwrap in Lexer.next<commit_after>use ast::{TIdent, TNumber, TString, TSemicolon, TComment, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TBinOp, TAssignOp, TUnaryOp, TArrow};
use ast::{OpAdd, OpSub, OpMul, OpDiv, OpMod};
use ast::{BitAnd, BitOr, BitXor, BitShl, BitShr};
use ast::{CompEqual, CompStrictEqual, CompNotEqual, CompStrictNotEqual, CompLessThan, CompGreaterThan, CompLessThanOrEqual, CompGreaterThanOrEqual};
use ast::{LogAnd, LogOr};
use ast::{UnaryNot};
use ast::{BinNum, BinBit, BinComp, BinLog};
use ast::{Token, TokenData};
use std::io::{BufReader, BufferedReader, Buffer, IoResult, EndOfFile};
use std::strbuf::StrBuf;
use std::char::from_u32;
use std::num::from_str_radix;
#[deriving(Clone, Eq)]
/// The type of string used
pub enum StringType {
	/// `"` - Double-quoted
	DoubleQuote,
	/// `'` - Single-quoted
	SingleQuote
}
#[deriving(Clone, Eq)]
/// The type of comment used
pub enum CommentType {
	/// `/*...*/` - A multi-line comment
	MultiLineComment,
	/// `//...` - A single-line comment
	SingleLineComment
}
#[deriving(Clone, Eq)]
/// The type of number used
pub enum NumberType {
	/// A decimal number, such as `3.1415`
	DecimalNumber,
	/// A hexadecimal number, such as `0xFF00FF`
	HexadecimalNumber,
	/// An octal number, such as `011`
	OctalNumber
}
/// A Javascript lexer
pub struct Lexer<B> {
	/// The list of tokens generated so far
	pub tokens : Vec<Token>,
	/// The current line number in the script
	line_number : uint,
	/// The current column number in the script
	column_number : uint,
	/// The reader
	buffer: B,
	/// The peeked character buffer
	peek_buffer: StrBuf
}
impl<B:Buffer> Lexer<B> {
	/// Creates a new lexer with empty buffers
	pub fn new(buffer: B) -> Lexer<B> {
		return Lexer {
			tokens: Vec::new(),
			line_number: 1,
			column_number: 0,
			buffer: buffer,
			peek_buffer: StrBuf::new()
		};
	}
	fn push_token(&mut self, tk:TokenData) {
		self.tokens.push(Token::new(tk, self.line_number, self.column_number))
	}
	/// Processes an input stream from a string into an array of tokens
	pub fn lex_str(script:&str) -> Vec<Token> {
		let script_bytes:&[u8] = script.as_bytes();
		let reader = BufReader::new(script_bytes);
		let buf_reader = BufferedReader::new(reader);
		let mut lexer = Lexer::new(buf_reader);
		lexer.lex().unwrap();
		lexer.tokens
	}
	fn next(&mut self) -> IoResult<char> {
		if self.peek_buffer.len() == 0 {
			self.buffer.read_char()
		} else {
			Ok(match self.peek_buffer.pop_char() {
				Some(v) => v,
				None => unreachable!()
			})
		}
	}
	fn peek_for(&mut self, peek:char) -> bool {
		if self.peek_buffer.len() > 0 {
			self.peek_buffer.pop_char() == Some(peek)
		} else {
			match self.buffer.read_char() {
				Ok(ch) if ch == peek => {
					self.column_number += 1;
					true
				},
				Ok(ch) if ch != peek => {
					self.peek_buffer.push_char(ch);
					false
				},
				_ => false
			}
		}
	}
	/// Processes an input stream from the `buffer` into a vector of tokens
	pub fn lex(&mut self) -> IoResult<()> {
		loop {
			let ch = match self.next() {
				Ok(ch) => ch,
				Err(ref err) if err.kind == EndOfFile => break,
				Err(err) => return Err(err)
			};
			self.column_number += 1;
			match ch {
				'"' | '\'' => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\'' if ch == '\'' => {
								break;
							},
							'"' if ch == '"' => {
								break;
							},
							'\\' => {
								let escape = try!(self.next());
								if escape != '\n' {
									let escaped_ch = match escape {
										'n' => '\n',
										'r' => '\r',
										't' => '\t',
										'b' => '\x08',
										'f' => '\x0c',
										'0' => '\0',
										'x' => {
											let mut nums = StrBuf::with_capacity(2);
											for _ in range(0, 2) {
												nums.push_char(try!(self.next()));
											}
											self.column_number += 2;
											let as_num = match from_str_radix(nums.as_slice(), 16) {
												Some(v) => v,
												None => 0
											};
											match from_u32(as_num) {
												Some(v) => v,
												None => fail!("{}:{}: {} is not a valid unicode scalar value", self.line_number, self.column_number, as_num)
											}
										},
										'u' => {
											let mut nums = StrBuf::new();
											for _ in range(0, 4) {
												nums.push_char(try!(self.next()));
											}
											self.column_number += 4;
											let as_num = match from_str_radix(nums.as_slice(), 16) {
												Some(v) => v,
												None => 0
											};
											match from_u32(as_num) {
												Some(v) => v,
												None => fail!("{}:{}: {} is not a valid unicode scalar value", self.line_number, self.column_number, as_num)
											}
										},
										'\'' | '"' => escape,
										_ => fail!("{}:{}: Invalid escape `{}`", self.line_number, self.column_number, ch)
									};
									buf.push_char(escaped_ch);
								}
							},
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TString(buf))
				},
				'0' if self.peek_for('x') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							ch if ch.is_digit_radix(16) => buf.push_char(ch),
							ch => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(from_str_radix(buf.as_slice(), 16).unwrap()));
				},
				'0' => {
					let mut buf = StrBuf::new();
					let mut gone_decimal = false;
					loop {
						let ch = try!(self.next());
						match ch {
							_ if ch.is_digit_radix(8) => buf.push_char(ch),
							'8' | '9' | '.' => {
								gone_decimal = true;
								buf.push_char(ch);
							},
							ch => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(if gone_decimal {
						from_str(buf.as_slice())
					} else {
						from_str_radix(buf.as_slice(), 8)
					}.unwrap()));
				},
				_ if ch.is_digit() => {
					let mut buf = StrBuf::new();
					buf.push_char(ch);
					loop {
						let ch = try!(self.next());
						match ch {
							'.' => buf.push_char(ch),
							_ if ch.is_digit() => buf.push_char(ch),
							_ => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TNumber(from_str(buf.as_slice()).unwrap()));
				},
				_ if ch.is_alphabetic() || ch == '$' || ch == '_' => {
					let mut buf = StrBuf::new();
					buf.push_char(ch);
					loop {
						let ch = try!(self.next());
						match ch {
							_ if ch.is_alphabetic() || ch.is_digit() => buf.push_char(ch),
							_ => {
								self.peek_buffer.push_char(ch);
								break;
							}
						}
					}
					self.push_token(TIdent(buf));
				},
				';' => {
					self.push_token(TSemicolon);
				},
				':' => {
					self.push_token(TColon);
				},
				'.' => {
					self.push_token(TDot);
				},
				'(' => {
					self.push_token(TOpenParen);
				},
				')' => {
					self.push_token(TCloseParen);
				},
				',' => {
					self.push_token(TComma);
				},
				'{' => {
					self.push_token(TOpenBlock);
				},
				'}' => {
					self.push_token(TCloseBlock);
				},
				'[' => {
					self.push_token(TOpenArray);
				},
				']' => {
					self.push_token(TCloseArray);
				},
				'?' => {
					self.push_token(TQuestion);
				},
				'/' if self.peek_for('/') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\n' => break,
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TComment(buf));
				},
				'/' if self.peek_for('*') => {
					let mut buf = StrBuf::new();
					loop {
						match try!(self.next()) {
							'\n' => break,
							'*' if self.peek_for('/') => break,
							ch => buf.push_char(ch)
						}
					}
					self.push_token(TComment(buf));
				},
				'/' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpDiv)));
				},
				'/' => {
					self.push_token(TBinOp(BinNum(OpDiv)));
				},
				'*' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpMul)));
				},
				'*' => {
					self.push_token(TBinOp(BinNum(OpMul)));
				},
				'+' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpAdd)));
				},
				'+' => {
					self.push_token(TBinOp(BinNum(OpAdd)));
				},
				'-' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpSub)));
				},
				'-' => {
					self.push_token(TBinOp(BinNum(OpSub)));
				},
				'%' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinNum(OpMod)));
				},
				'%' => {
					self.push_token(TBinOp(BinNum(OpMod)));
				},
				'|' if self.peek_for('|') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinLog(LogOr)));
					} else {
						self.push_token(TBinOp(BinLog(LogOr)));
					}
				},
				'|' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitOr)));
				},
				'|' => {
					self.push_token(TBinOp(BinBit(BitOr)));
				},
				'&' if self.peek_for('&') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinLog(LogAnd)));
					} else {
						self.push_token(TBinOp(BinLog(LogAnd)));
					}
				},
				'&' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitAnd)));
				},
				'&' => {
					self.push_token(TBinOp(BinBit(BitAnd)));
				},
				'^' if self.peek_for('=') => {
					self.push_token(TAssignOp(BinBit(BitXor)));
				},
				'^' => {
					self.push_token(TBinOp(BinBit(BitXor)));
				},
				'=' if self.peek_for('>') => {
					self.push_token(TArrow);
				},
				'=' if self.peek_for('=') => {
					if self.peek_for('=') {
						self.push_token(TBinOp(BinComp(CompStrictEqual)));
					} else {
						self.push_token(TBinOp(BinComp(CompEqual)));
					}
				},
				'=' => {
					self.push_token(TEqual);
				},
				'<' if self.peek_for('=') => {
					self.push_token(TBinOp(BinComp(CompLessThanOrEqual)));
				},
				'<' if self.peek_for('<') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinBit(BitShl)));
					} else {
						self.push_token(TBinOp(BinBit(BitShl)));
					}
				},
				'<' => {
					self.push_token(TBinOp(BinComp(CompLessThan)));
				},
				'>' if self.peek_for('=') => {
					self.push_token(TBinOp(BinComp(CompGreaterThanOrEqual)));
				},
				'>' if self.peek_for('>') => {
					if self.peek_for('=') {
						self.push_token(TAssignOp(BinBit(BitShr)));
					} else {
						self.push_token(TBinOp(BinBit(BitShr)));
					}
				},
				'>' => {
					self.push_token(TBinOp(BinComp(CompGreaterThan)));
				},
				'!' if self.peek_for('=') => {
					if self.peek_for('=') {
						self.push_token(TBinOp(BinComp(CompStrictNotEqual)));
					} else {
						self.push_token(TBinOp(BinComp(CompNotEqual)));
					}
				},
				'!' => {
					self.push_token(TUnaryOp(UnaryNot));
				},
				'\n' => {
					self.line_number += 1;
					self.column_number = 0;
				},
				'\r' => {
					self.column_number = 0;
				},
				ch => fail!("{}:{}: Unexpected '{}'", self.line_number, self.column_number, ch)
			};
		}
		Ok(())
	}
}<|endoftext|>


<commit_before>//! Generate update information from dependency information.

use std::rc::Rc;
use std::collections::HashMap;

use semver::Version;

use dependencies;

/// Stores old and new version.
pub struct Bump {
    /// The old version.
    pub old: Version,
    /// The new version.
    pub new: Version,
}

/// Stores update info for dependency.
pub struct Dependency {
    /// The library name.
    pub name: Rc<String>,
    /// Bump info.
    pub bump: Bump,
}

/// Stores information about a package.
pub struct Package {
    /// The package name.
    pub name: Rc<String>,
    /// The order to update library.
    pub order: u32,
    /// Update information.
    pub bump: Bump,
    /// The dependencies.
    pub dependencies: Vec<Dependency>,
}

/// Generates update info.
pub fn generate_update_info_from(dependency_info: &str) -> Result<String, String> {
    use piston_meta::*;

    fn depth_of(
        package: &dependencies::Package,
        depth: &mut HashMap<Rc<String>, u32>,
        dependencies_data: &[dependencies::Package]
    ) -> u32 {
        let d = depth.get(&package.name).map(|d| *d);
        match d {
            None => {
                // The depth of a package equals maximum depth of the dependencies + 1.
                let new_depth: u32 = package.dependencies.iter().map(|dep| {
                        // Get depth of dependency.
                        // If none of the dependencies are listed, treat it as zero.
                        dependencies_data.iter().filter(|p| {
                                p.name == dep.name
                            }).next().map(|p| {
                                depth_of(p, depth, dependencies_data)
                            }).unwrap_or(0)
                    }).max().unwrap_or(0) + 1;
                depth.insert(package.name.clone(), new_depth);
                new_depth
            }
            Some(x) => x
        }
    }

    // Parse and convert to dependencies data.
    let dependencies_meta_syntax = include_str!("../assets/dependencies/syntax.txt");
    let dependencies_meta_rules = stderr_unwrap(dependencies_meta_syntax,
        syntax(dependencies_meta_syntax));
    let dependency_info = stderr_unwrap(dependency_info,
        parse(&dependencies_meta_rules, dependency_info));
    let mut ignored = vec![];
    let dependencies_data = try!(dependencies::convert(&dependency_info, &mut ignored)
        .map_err(|_| String::from("Could not convert dependency info")));

    // Store the depths of libraries.
    let mut depths: HashMap<Rc<String>, u32> = HashMap::new();
    for package in &dependencies_data {
        let _depth = depth_of(package, &mut depths, &dependencies_data);
    }

    let mut new_versions: HashMap<Rc<String>, Version> = HashMap::new();
    for package in &dependencies_data {
        // Get latest version used by any dependency.
        for dep in &package.dependencies {
            let version = try!(Version::parse(&dep.version)
                .map_err(|_| format!("Could not parse version `{}` for `{}`",
                    &dep.version, &dep.name)));
            let v = new_versions.get(&dep.name).map(|v| v.clone());
            match v {
                None => {
                    new_versions.insert(dep.name.clone(), version);
                }
                Some(v) => {
                    if v < version {
                        new_versions.insert(dep.name.clone(), version);
                    }
                }
            }
        }
    }

    // Overwrite the versions used by packages.
    for package in &dependencies_data {
        let version = try!(Version::parse(&package.version)
            .map_err(|_| format!("Could not parse version `{}` for `{}`",
                &package.version, &package.name)));
        new_versions.insert(package.name.clone(), version);
    }

    Ok(String::from(""))
}
<commit_msg>Use package index as key in `depths`<commit_after>//! Generate update information from dependency information.

use std::rc::Rc;
use std::collections::HashMap;

use semver::Version;

use dependencies;

/// Stores old and new version.
pub struct Bump {
    /// The old version.
    pub old: Version,
    /// The new version.
    pub new: Version,
}

/// Stores update info for dependency.
pub struct Dependency {
    /// The library name.
    pub name: Rc<String>,
    /// Bump info.
    pub bump: Bump,
}

/// Stores information about a package.
pub struct Package {
    /// The package name.
    pub name: Rc<String>,
    /// The order to update library.
    pub order: u32,
    /// Update information.
    pub bump: Bump,
    /// The dependencies.
    pub dependencies: Vec<Dependency>,
}

/// Generates update info.
pub fn generate_update_info_from(dependency_info: &str) -> Result<String, String> {
    use piston_meta::*;

    type PackageIndex = usize;
    type Depth = u32;

    fn depth_of(
        package_index: PackageIndex,
        depths: &mut HashMap<PackageIndex, Depth>,
        dependencies_data: &[dependencies::Package]
    ) -> Depth {
        let package = &dependencies_data[package_index];
        let d = depths.get(&package_index).map(|d| *d);
        match d {
            None => {
                // The depth of a package equals maximum depth of the dependencies + 1.
                let new_depth: Depth = package.dependencies.iter().map(|dep| {
                        let mut depth = 0;
                        for (i, p) in dependencies_data.iter().enumerate() {
                            if p.name == dep.name {
                                depth = depth_of(i, depths, dependencies_data);
                            }
                        }
                        depth
                    }).max().unwrap_or(0) + 1;
                depths.insert(package_index, new_depth);
                new_depth
            }
            Some(x) => x
        }
    }

    // Parse and convert to dependencies data.
    let dependencies_meta_syntax = include_str!("../assets/dependencies/syntax.txt");
    let dependencies_meta_rules = stderr_unwrap(dependencies_meta_syntax,
        syntax(dependencies_meta_syntax));
    let dependency_info = stderr_unwrap(dependency_info,
        parse(&dependencies_meta_rules, dependency_info));
    let mut ignored = vec![];
    let dependencies_data = try!(dependencies::convert(&dependency_info, &mut ignored)
        .map_err(|_| String::from("Could not convert dependency info")));

    // Store the depths of libraries.
    let mut depths: HashMap<PackageIndex, Depth> = HashMap::new();
    for i in 0 .. dependencies_data.len() {
        let _depth = depth_of(i, &mut depths, &dependencies_data);
    }

    let mut new_versions: HashMap<Rc<String>, Version> = HashMap::new();
    for package in &dependencies_data {
        // Get latest version used by any dependency.
        for dep in &package.dependencies {
            let version = try!(Version::parse(&dep.version)
                .map_err(|_| format!("Could not parse version `{}` for `{}`",
                    &dep.version, &dep.name)));
            let v = new_versions.get(&dep.name).map(|v| v.clone());
            match v {
                None => {
                    new_versions.insert(dep.name.clone(), version);
                }
                Some(v) => {
                    if v < version {
                        new_versions.insert(dep.name.clone(), version);
                    }
                }
            }
        }
    }

    // Overwrite the versions used by packages.
    for package in &dependencies_data {
        let version = try!(Version::parse(&package.version)
            .map_err(|_| format!("Could not parse version `{}` for `{}`",
                &package.version, &package.name)));
        new_versions.insert(package.name.clone(), version);
    }

    Ok(String::from(""))
}
<|endoftext|>


<commit_before>use common::*;
use std::io::{Read, Write};

mod interrupts;
mod memory;
mod modes;
mod opcodes;
mod registers;

use self::memory::*;

pub struct VM<'a, R: Read, W: Write> {
    input: R,
    output: W,

    registers: Registers,
    memory: Memory<'a>,
}

impl<'a, R: Read, W: Write> VM<'a, R, W> {
    pub fn new(input: R, output: W) -> Self {
        unimplemented!()
    }

    pub fn run(&mut self, executable: Data) {
        unimplemented!()
    }
}

#[cfg(test)]
mod tests {
    use self::interrupts::*;
    use self::modes::*;
    use self::opcodes::*;
    use self::registers::*;
    use std::io::{BufReader, BufWriter};
    use super::*;

    #[cfg_attr(rustfmt, rustfmt_skip)]
    #[test]
    fn simple() {
        assert!(run(&[], vec![]).0.is_empty());

        {
            let mut executable = vec![0, 0,
                MOV, REG, A, VALUE, 123, 0,
                ADD, REG, B, REG, A, REG, A];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(123, registers[A as usize]);
            assert_eq!(246, registers[B as usize]);
            assert!(output.is_empty());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                // stack allocation for 2 words
                SUB, REG, SP,  REG, SP,  VALUE, 4, 0,

                PUSH, VALUE, 5, 0,
                PUSH, VALUE, 6, 0,
                ADD,  REG, A,  PTR, REG, SP,  PTR_WITH_OFFSET, REG, SP, 2, 0,
                // add a, [sp], [sp+2]
                POP, B,
                POP, C,
                ADD, REG, SP,  REG, SP,  4, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory)  = run(&[], executable);

            assert_eq!(11, registers[A as usize]);
            assert_eq!(6, registers[B as usize]);
            assert_eq!(5, registers[C as usize]);
            assert!(output.is_empty());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV, REG, A, VALUE, 10, 0,
                DEC, REG, A,
                INT, OUTPUT,
                JNZ, 2, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[10, 0, 9, 0, 8, 0, 7, 0, 6, 0, 5, 0,
                         4, 0, 3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV, REG, B, VALUE, 24, 0,       // data address
                MOV, REG, A, PTR, REG, B,        // dereference B
                INT, OUTPUT,
                INC, REG, B,
                INC, REG, B,
                JNZ, 2, 0,

                3, 0,                            // data with address 24
                2, 0,                            // data with address 26
                1, 0,
                0, 0];
            let data_length = 8;
            let code_length = executable.len() as u8 - data_length;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(8, memory.data.len());
            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                INT, INPUT,
                INT, OUTPUT,
                JNZ, 2, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[3, 0, 2, 0, 1, 0, 0, 0], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                                                      // do
                INT, INPUT,                           // a = read
                SUB, REG, SP,  REG, SP,  VALUE, 2, 0, //   (stack allocation)
                PUSH, REG, A,
                CALL, VALUE, 35, 0,                   //   a = f(a)
                INT, OUTPUT,                          //   print a
                ADD, REG, SP,  REG, SP,  VALUE, 2, 0, //   (stack deallocation)
                JNZ, 2, 0,                            // while a != 0 jmp addr 0
                JMP, 63, 0,                           // exit

                // label f
                PUSH, REG, BP,
                MOV, REG, BP, REG, SP,

                POP, REG, A,
                MUL, REG, A, REG, A, VALUE, 2, 0,    // a = a * 2

                MOV, REG, SP, REG, BP,
                POP, REG, BP,
                RET,

                NOP                                  // optional
            ];

            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[3, 0, 2, 0, 1, 0, 0, 0],
                                                  executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[6, 0, 4, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                DIV, REG, A, VALUE, 1, VALUE, 0,
                MOV, REG, A, 123,
            ];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(b"Unknown Error", output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV,  PTR, VALUE,  255, 255,  VALUE, 1,
                MOV, A, 123,
            ];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(b"Segfault", output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }
    }

    fn run<'a>(input: &[u8], executable: Data) -> (Data, Registers, Memory<'a>) {
        let input = BufReader::new(input);

        let output: Data = vec![];
        let output = BufWriter::new(output);

        let mut vm = VM::new(input, output);
        vm.run(executable);

        let output = vm.output
            .get_mut()
            .by_ref()
            .iter()
            .map(|x| *x)
            .collect::<Data>();

        (output, vm.registers, vm.memory)
    }
}
<commit_msg>bugfix<commit_after>use common::*;
use std::io::{Read, Write};

mod interrupts;
mod memory;
mod modes;
mod opcodes;
mod registers;

use self::memory::*;

pub struct VM<'a, R: Read, W: Write> {
    input: R,
    output: W,

    registers: Registers,
    memory: Memory<'a>,
}

impl<'a, R: Read, W: Write> VM<'a, R, W> {
    pub fn new(input: R, output: W) -> Self {
        unimplemented!()
    }

    pub fn run(&mut self, executable: Data) {
        unimplemented!()
    }
}

#[cfg(test)]
mod tests {
    use self::interrupts::*;
    use self::modes::*;
    use self::opcodes::*;
    use self::registers::*;
    use std::io::{BufReader, BufWriter};
    use super::*;

    #[cfg_attr(rustfmt, rustfmt_skip)]
    #[test]
    fn simple() {
        assert!(run(&[], vec![]).0.is_empty());

        {
            let mut executable = vec![0, 0,
                MOV, REG, A, VALUE, 123, 0,
                ADD, REG, B, REG, A, REG, A];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(123, registers[A as usize]);
            assert_eq!(246, registers[B as usize]);
            assert!(output.is_empty());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                // stack allocation for 2 words
                SUB, REG, SP,  REG, SP,  VALUE, 4, 0,

                PUSH, VALUE, 5, 0,
                PUSH, VALUE, 6, 0,
                ADD,  REG, A,  PTR, REG, SP,  PTR_WITH_OFFSET, REG, SP, 2, 0,
                // add a, [sp], [sp+2]
                POP, B,
                POP, C,
                ADD, REG, SP,  REG, SP,  4, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory)  = run(&[], executable);

            assert_eq!(11, registers[A as usize]);
            assert_eq!(6, registers[B as usize]);
            assert_eq!(5, registers[C as usize]);
            assert!(output.is_empty());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV, REG, A, VALUE, 10, 0,
                DEC, REG, A,
                INT, OUTPUT,
                JNZ, 2, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[10, 0, 9, 0, 8, 0, 7, 0, 6, 0, 5, 0,
                         4, 0, 3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV, REG, B, VALUE, 25, 0,       // data address
                MOV, REG, A, PTR, REG, B,        // dereference B
                INT, OUTPUT,
                INC, REG, B,
                INC, REG, B,
                JNZ, 2, 0,

                3, 0,                            // data with address 24
                2, 0,                            // data with address 26
                1, 0,
                0, 0];
            let data_length = 8;
            let code_length = executable.len() as u8 - data_length;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(8, memory.data.len());
            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                INT, INPUT,
                INT, OUTPUT,
                JNZ, 2, 0];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[3, 0, 2, 0, 1, 0, 0, 0], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[3, 0, 2, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                                                      // do
                INT, INPUT,                           // a = read
                SUB, REG, SP,  REG, SP,  VALUE, 2, 0, //   (stack allocation)
                PUSH, REG, A,
                CALL, VALUE, 35, 0,                   //   a = f(a)
                INT, OUTPUT,                          //   print a
                ADD, REG, SP,  REG, SP,  VALUE, 2, 0, //   (stack deallocation)
                JNZ, 2, 0,                            // while a != 0 jmp addr 0
                JMP, 63, 0,                           // exit

                // label f
                PUSH, REG, BP,
                MOV, REG, BP, REG, SP,

                POP, REG, A,
                MUL, REG, A, REG, A, VALUE, 2, 0,    // a = a * 2

                MOV, REG, SP, REG, BP,
                POP, REG, BP,
                RET,

                NOP                                  // optional
            ];

            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[3, 0, 2, 0, 1, 0, 0, 0],
                                                  executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(&[6, 0, 4, 0, 1, 0, 0, 0], output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                DIV, REG, A, VALUE, 1, VALUE, 0,
                MOV, REG, A, 123,
            ];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(b"Unknown Error", output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }

        {
            let mut executable = vec![0, 0,
                MOV,  PTR, VALUE,  255, 255,  VALUE, 1,
                MOV, A, 123,
            ];
            let code_length = executable.len() as u8;
            executable[0] = code_length;

            let (output, registers, memory) = run(&[], executable);

            assert_eq!(0, registers[A as usize]);
            assert_eq!(b"Segfault", output.as_slice());
            assert!(memory.stack_is_empty(registers[SP as usize]));
        }
    }

    fn run<'a>(input: &[u8], executable: Data) -> (Data, Registers, Memory<'a>) {
        let input = BufReader::new(input);

        let output: Data = vec![];
        let output = BufWriter::new(output);

        let mut vm = VM::new(input, output);
        vm.run(executable);

        let output = vm.output
            .get_mut()
            .by_ref()
            .iter()
            .map(|x| *x)
            .collect::<Data>();

        (output, vm.registers, vm.memory)
    }
}
<|endoftext|>


<commit_before>use std::io::Write;

use errors::Result;
use message::MessageWrite;

use byteorder::WriteBytesExt;
use byteorder::LittleEndian as LE;

/// A struct to write protobuf messages
pub struct Writer<W> {
    inner: W,
}

impl<W: Write> Writer<W> {

    /// Creates a new `ProtobufWriter`
    pub fn new(w: W) -> Writer<W> {
        Writer { inner: w }
    }

    pub fn write_varint(&mut self, mut v: u64) -> Result<()> {
        while v > 0x7F {
            self.inner.write_u8(((v as u8) & 0x7F) | 0x80)?;
            v >>= 7;
        }
        self.inner.write_u8(v as u8)?;
        Ok(())
    }

    pub fn write_tag(&mut self, tag: u32) -> Result<()> {
        self.write_varint(tag as u64)?;
        Ok(())
    }

    pub fn write_int32(&mut self, v: i32) -> Result<()> {
        self.write_varint(v as u64)
    }

    pub fn write_int64(&mut self, v: i64) -> Result<()> {
        self.write_varint(v as u64)
    }

    pub fn write_uint32(&mut self, v: u32) -> Result<()> {
        self.write_varint(v as u64)
    }

    pub fn write_uint64(&mut self, v: u64) -> Result<()> {
        self.write_varint(v)
    }

    pub fn write_sint32(&mut self, v: i32) -> Result<()> {
        self.write_varint(((v << 1) ^ (v >> 31)) as u64)
    }

    pub fn write_sint64(&mut self, v: i64) -> Result<()> {
        self.write_varint(((v << 1) ^ (v >> 63)) as u64)
    }

    pub fn write_fixed64(&mut self, v: u64) -> Result<()> {
        self.inner.write_u64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_fixed32(&mut self, v: u32) -> Result<()> {
        self.inner.write_u32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_sfixed64(&mut self, v: i64) -> Result<()> {
        self.inner.write_i64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_sfixed32(&mut self, v: i32) -> Result<()> {
        self.inner.write_i32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_float(&mut self, v: f32) -> Result<()> {
        self.inner.write_f32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_double(&mut self, v: f64) -> Result<()> {
        self.inner.write_f64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_bool(&mut self, v: bool) -> Result<()> {
        self.write_varint(if v { 1 } else { 0 })
    }

    pub fn write_enum(&mut self, v: i32) -> Result<()> {
        self.write_int32(v)
    }

    pub fn write_bytes(&mut self, bytes: &[u8]) -> Result<()> {
        self.write_varint(bytes.len() as u64)?;
        self.inner.write_all(bytes).map_err(|e| e.into())
    }

    pub fn write_string(&mut self, s: &str) -> Result<()> {
        self.write_bytes(s.as_bytes())
    }

    pub fn write_packed_repeated_field<M, F, S>(&mut self, v: &[M], mut write: F, size: &S) -> Result<()>
        where F: FnMut(&mut Self, &M) -> Result<()>,
              S: Fn(&M) -> usize,
    {
        let len: usize = v.iter().map(|m| size(m)).sum();
        self.write_varint(len as u64)?;
        for m in v {
            write(self, m)?;
        }
        Ok(())
    }

    pub fn write_packed_fixed_size<M>(&mut self, v: &[M], item_size: usize) -> Result<()> {
        let len = v.len() * item_size;
        let bytes = unsafe { ::std::slice::from_raw_parts(v as *const [M] as *const M as *const u8, len) };
        self.write_bytes(bytes)
    }

    pub fn write_message<M: MessageWrite>(&mut self, m: &M) -> Result<()> {
        let len = m.get_size();
        self.write_varint(len as u64)?;
        m.write_message(self)
    }

    pub fn write_int32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    pub fn write_int64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    pub fn write_uint32_with_tag(&mut self, tag: u32, v: u32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    pub fn write_uint64_with_tag(&mut self, tag: u32, v: u64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v)
    }

    pub fn write_sint32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_sint32(v)
    }

    pub fn write_sint64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_sint64(v)
    }

    pub fn write_fixed64_with_tag(&mut self, tag: u32, v: u64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_u64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_fixed32_with_tag(&mut self, tag: u32, v: u32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_u32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_sfixed64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_i64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_sfixed32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_i32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_float_with_tag(&mut self, tag: u32, v: f32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_f32::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_double_with_tag(&mut self, tag: u32, v: f64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_f64::<LE>(v).map_err(|e| e.into())
    }

    pub fn write_bool_with_tag(&mut self, tag: u32, v: bool) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(if v { 1 } else { 0 })
    }

    pub fn write_bytes_with_tag(&mut self, tag: u32, bytes: &[u8]) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(bytes.len() as u64)?;
        self.inner.write_all(bytes).map_err(|e| e.into())
    }

    pub fn write_string_with_tag(&mut self, tag: u32, s: &str) -> Result<()> {
        self.write_tag(tag)?;
        self.write_bytes(s.as_bytes())
    }

    pub fn write_packed_repeated_field_with_tag<M, F, S>(&mut self,
                                                         tag: u32,
                                                         v: &[M],
                                                         mut write: F,
                                                         size: &S) -> Result<()>
        where F: FnMut(&mut Self, &M) -> Result<()>,
              S: Fn(&M) -> usize,
    {
        if v.is_empty() {
            return Ok(());
        }

        self.write_tag(tag)?;
        let len: usize = v.iter().map(|m| size(m)).sum();
        self.write_varint(len as u64)?;
        for m in v {
            write(self, m)?;
        }
        Ok(())
    }

    pub fn write_packed_fixed_size_with_tag<M>(&mut self,
                                               tag: u32,
                                               v: &[M],
                                               item_size: usize) -> Result<()> {
        self.write_tag(tag)?;
        let len = v.len() * item_size;
        let bytes = unsafe { ::std::slice::from_raw_parts(v as *const [M] as *const M as *const u8, len) };
        self.write_bytes(bytes)
    }

    pub fn write_message_with_tag<M: MessageWrite>(&mut self, tag: u32, m: &M) -> Result<()> {
        self.write_tag(tag)?;
        self.write_message(m)
    }

    pub fn write_enum_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_int32(v)
    }
}
<commit_msg>add basic comments on writer<commit_after>//! A module to manage protobuf serialization

use std::io::Write;

use errors::Result;
use message::MessageWrite;

use byteorder::WriteBytesExt;
use byteorder::LittleEndian as LE;

/// A struct to write protobuf messages
pub struct Writer<W> {
    inner: W,
}

impl<W: Write> Writer<W> {

    /// Creates a new `ProtobufWriter`
    pub fn new(w: W) -> Writer<W> {
        Writer { inner: w }
    }

    /// Writes a `varint` (compacted `u64`)
    pub fn write_varint(&mut self, mut v: u64) -> Result<()> {
        while v > 0x7F {
            self.inner.write_u8(((v as u8) & 0x7F) | 0x80)?;
            v >>= 7;
        }
        self.inner.write_u8(v as u8).map_err(|e| e.into())
    }

    /// Writes a tag, which represents both the field number and the wire type
    pub fn write_tag(&mut self, tag: u32) -> Result<()> {
        self.write_varint(tag as u64)
    }

    /// Writes a `int32` which is internally coded as a `varint`
    pub fn write_int32(&mut self, v: i32) -> Result<()> {
        self.write_varint(v as u64)
    }

    /// Writes a `int64` which is internally coded as a `varint`
    pub fn write_int64(&mut self, v: i64) -> Result<()> {
        self.write_varint(v as u64)
    }

    /// Writes a `uint32` which is internally coded as a `varint`
    pub fn write_uint32(&mut self, v: u32) -> Result<()> {
        self.write_varint(v as u64)
    }

    /// Writes a `uint64` which is internally coded as a `varint`
    pub fn write_uint64(&mut self, v: u64) -> Result<()> {
        self.write_varint(v)
    }

    /// Writes a `sint32` which is internally coded as a `varint`
    pub fn write_sint32(&mut self, v: i32) -> Result<()> {
        self.write_varint(((v << 1) ^ (v >> 31)) as u64)
    }

    /// Writes a `sint64` which is internally coded as a `varint`
    pub fn write_sint64(&mut self, v: i64) -> Result<()> {
        self.write_varint(((v << 1) ^ (v >> 63)) as u64)
    }

    /// Writes a `fixed64` which is little endian coded `u64`
    pub fn write_fixed64(&mut self, v: u64) -> Result<()> {
        self.inner.write_u64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `fixed32` which is little endian coded `u32`
    pub fn write_fixed32(&mut self, v: u32) -> Result<()> {
        self.inner.write_u32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `sfixed64` which is little endian coded `i64`
    pub fn write_sfixed64(&mut self, v: i64) -> Result<()> {
        self.inner.write_i64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `sfixed32` which is little endian coded `i32`
    pub fn write_sfixed32(&mut self, v: i32) -> Result<()> {
        self.inner.write_i32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `float`
    pub fn write_float(&mut self, v: f32) -> Result<()> {
        self.inner.write_f32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `double`
    pub fn write_double(&mut self, v: f64) -> Result<()> {
        self.inner.write_f64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes a `bool` 1 = true, 0 = false
    pub fn write_bool(&mut self, v: bool) -> Result<()> {
        self.write_varint(if v { 1 } else { 0 })
    }

    /// Writes an `enum` converting it to a `i32` first
    pub fn write_enum(&mut self, v: i32) -> Result<()> {
        self.write_int32(v)
    }

    /// Writes `bytes`: length first then the chunk of data
    pub fn write_bytes(&mut self, bytes: &[u8]) -> Result<()> {
        self.write_varint(bytes.len() as u64)?;
        self.inner.write_all(bytes).map_err(|e| e.into())
    }

    /// Writes `string`: length first then the chunk of data
    pub fn write_string(&mut self, s: &str) -> Result<()> {
        self.write_bytes(s.as_bytes())
    }

    /// Writes packed repeated field: length first then the chunk of data
    pub fn write_packed_repeated_field<M, F, S>(&mut self, v: &[M], mut write: F, size: &S) -> Result<()>
        where F: FnMut(&mut Self, &M) -> Result<()>,
              S: Fn(&M) -> usize,
    {
        if v.is_empty() {
            return Ok(());
        }
        let len: usize = v.iter().map(|m| size(m)).sum();
        self.write_varint(len as u64)?;
        for m in v {
            write(self, m)?;
        }
        Ok(())
    }

    /// Writes packed repeated field when we know the size of items
    ///
    /// `item_size` is internally used to compute the total length
    /// As the length is fixed (and the same as rust internal representation, we can directly dump
    /// all data at once
    pub fn write_packed_fixed_size<M>(&mut self, v: &[M], item_size: usize) -> Result<()> {
        let len = v.len() * item_size;
        let bytes = unsafe { ::std::slice::from_raw_parts(v as *const [M] as *const M as *const u8, len) };
        self.write_bytes(bytes)
    }

    /// Writes a message which implements `MessageWrite`
    pub fn write_message<M: MessageWrite>(&mut self, m: &M) -> Result<()> {
        let len = m.get_size();
        self.write_varint(len as u64)?;
        m.write_message(self)
    }

    /// Writes tag then `int32`
    pub fn write_int32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    /// Writes tag then `int64`
    pub fn write_int64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    /// Writes tag then `uint32`
    pub fn write_uint32_with_tag(&mut self, tag: u32, v: u32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v as u64)
    }

    /// Writes tag then `uint64`
    pub fn write_uint64_with_tag(&mut self, tag: u32, v: u64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(v)
    }

    /// Writes tag then `sint32`
    pub fn write_sint32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_sint32(v)
    }

    /// Writes tag then `sint64`
    pub fn write_sint64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.write_sint64(v)
    }

    /// Writes tag then `fixed64`
    pub fn write_fixed64_with_tag(&mut self, tag: u32, v: u64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_u64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `fixed32`
    pub fn write_fixed32_with_tag(&mut self, tag: u32, v: u32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_u32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `sfixed64`
    pub fn write_sfixed64_with_tag(&mut self, tag: u32, v: i64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_i64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `sfixed32`
    pub fn write_sfixed32_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_i32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `float`
    pub fn write_float_with_tag(&mut self, tag: u32, v: f32) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_f32::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `double`
    pub fn write_double_with_tag(&mut self, tag: u32, v: f64) -> Result<()> {
        self.write_tag(tag)?;
        self.inner.write_f64::<LE>(v).map_err(|e| e.into())
    }

    /// Writes tag then `bool`
    pub fn write_bool_with_tag(&mut self, tag: u32, v: bool) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(if v { 1 } else { 0 })
    }

    /// Writes tag then `bytes`
    pub fn write_bytes_with_tag(&mut self, tag: u32, bytes: &[u8]) -> Result<()> {
        self.write_tag(tag)?;
        self.write_varint(bytes.len() as u64)?;
        self.inner.write_all(bytes).map_err(|e| e.into())
    }

    /// Writes tag then `string`
    pub fn write_string_with_tag(&mut self, tag: u32, s: &str) -> Result<()> {
        self.write_tag(tag)?;
        self.write_bytes(s.as_bytes())
    }

    /// Writes tag then repeated field
    ///
    /// If array is empty, then do nothing (do not even write the tag)
    pub fn write_packed_repeated_field_with_tag<M, F, S>(&mut self,
                                                         tag: u32,
                                                         v: &[M],
                                                         mut write: F,
                                                         size: &S) -> Result<()>
        where F: FnMut(&mut Self, &M) -> Result<()>,
              S: Fn(&M) -> usize,
    {
        if v.is_empty() {
            return Ok(());
        }

        self.write_tag(tag)?;
        let len: usize = v.iter().map(|m| size(m)).sum();
        self.write_varint(len as u64)?;
        for m in v {
            write(self, m)?;
        }
        Ok(())
    }

    /// Writes tag then repeated field with fixed length item size
    ///
    /// If array is empty, then do nothing (do not even write the tag)
    pub fn write_packed_fixed_size_with_tag<M>(&mut self,
                                               tag: u32,
                                               v: &[M],
                                               item_size: usize) -> Result<()> {
        if v.is_empty() {
            return Ok(());
        }
        self.write_tag(tag)?;
        let len = v.len() * item_size;
        let bytes = unsafe { ::std::slice::from_raw_parts(v as *const [M] as *const M as *const u8, len) };
        self.write_bytes(bytes)
    }

    /// Writes tag then message
    pub fn write_message_with_tag<M: MessageWrite>(&mut self, tag: u32, m: &M) -> Result<()> {
        self.write_tag(tag)?;
        self.write_message(m)
    }

    /// Writes tag then enum
    pub fn write_enum_with_tag(&mut self, tag: u32, v: i32) -> Result<()> {
        self.write_tag(tag)?;
        self.write_int32(v)
    }
}
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Panic handling and stack unwinding

use core::fmt::{Arguments, Write};
use super::io::term;
use super::arch::drivers::vga::Color;

#[lang = "panic_fmt"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn rust_begin_unwind( args: Arguments
                               , file: &'static str
                               , line: usize ) -> !
{
    let console
        = term::CONSOLE.lock()
                       .set_colors(Color::White, Color::Red);
    write!(console
          , "\nKERNEL PANIC in {}:{}\n\t{}"
          , file, line, args
          );
    loop { }
}

/// Required for Rust stack unwinding
#[lang = "eh_personality"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn eh_personality() {
    // TODO: add support for stack unwinding
}

#[lang = "stack_exhausted"]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn __morestack() -> ! {
    loop { }
}

#[allow(non_snake_case)]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn _Unwind_Resume() -> ! {
    loop { }
}

// #[lang = "begin_unwind"]
// #[no_mangle] #[inline(never)] #[cold]
// pub fn begin_unwind<M: Send>(msg: M
//                             , file_line: &(&'static str, usize))  -> !
// { loop { } }
<commit_msg>Fix console lifetime issue in panics.rs<commit_after>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Panic handling and stack unwinding

use core::fmt::{Arguments, Write};
use super::io::term;
use super::arch::drivers::vga::Color;

#[lang = "panic_fmt"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn rust_begin_unwind( args: Arguments
                               , file: &'static str
                               , line: usize ) -> !
{
    write!(term::CONSOLE.lock()
                        .set_colors(Color::White, Color::Red)
          , "\nKERNEL PANIC in {}:{}\n\t{}"
          , file, line, args
          );
    loop { }
}

/// Required for Rust stack unwinding
#[lang = "eh_personality"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn eh_personality() {
    // TODO: add support for stack unwinding
}

#[lang = "stack_exhausted"]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn __morestack() -> ! {
    loop { }
}

#[allow(non_snake_case)]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn _Unwind_Resume() -> ! {
    loop { }
}

// #[lang = "begin_unwind"]
// #[no_mangle] #[inline(never)] #[cold]
// pub fn begin_unwind<M: Send>(msg: M
//                             , file_line: &(&'static str, usize))  -> !
// { loop { } }
<|endoftext|>


<commit_before>use regex::Regex;

use std::collections::VecDeque;
use std::fmt;

use eval::FuncError;
use value::Value;

#[derive(Debug, Clone)]
pub enum ParseError {
    UnclosedList,
    InvalidListDelimitter,
    UnreadableSourceCode,
}

#[derive(Debug, Clone)]
pub struct FilePos {
    line: usize,
    col: usize,
}

impl FilePos {
    fn from_offset(text: &str, mut offset: usize) -> FilePos {
        let mut line_no = 1;

        for (cur_offset, ch) in text.bytes().enumerate() {
            if cur_offset >= offset {
                break;
            }

            if ch == '\n' as u8 {
                line_no += 1;
            }
        }

        FilePos {
            line: line_no,
            col: 0,
        }
    }
}

pub type ParseResult = Result<Value, ParseError>; // result of tokenizing a single token

const LIST_OPEN: &'static str = "(";
const LIST_CLOSE: &'static str = ")";

const QUOTE_OPEN: &'static str = "{";
const QUOTE_CLOSE: &'static str = "}";

pub fn write_list<T>(fmt: &mut fmt::Formatter, list: &Vec<T>, start: &str, sep: &str, end: &str) -> fmt::Result where T: fmt::Debug {
    if list.len() == 0 {
        return write!(fmt, "{}{}", start, end);
    }

    write!(fmt, "{}{:?}", start, list[0]);

    for token in &list[1..] {
        write!(fmt, "{}{:?}", sep, token);
    }

    write!(fmt, "{}", end)
}

fn tokenize(code: &str) -> VecDeque<String> {
    let string_re = r#""[^"]*""#;
    let sym_re = r"[-!?#\w\.]+";
    let num_re = r"\d+\.?\d*e?\d*";
    let list_re = r"[(){}\[\]]";
    let op_re = r"\+|-|\*|/|\^|&|\||=|\\|<|>";
    let quote_re = r"'";

    let regex = format!("{}|{}|{}|{}|{}|{}", string_re, num_re, sym_re, list_re, op_re, quote_re);

    let re = match Regex::new(&regex) {
        Ok(re) => re,
        Err(_) => return VecDeque::new(),
    };

    let spaced_code = code.to_string()
        .replace("(", " ( ")
        .replace(")", " ) ");

    let mut token_strs = VecDeque::new();

    for cap in re.captures_iter(&spaced_code) {
        let (offset, _) = cap.get(0)
            .map(|text| (text.start(), text.end()))
            .unwrap_or((0, 0));

        let match_str = cap.get(0)
            .map(|text| text.as_str())
            .unwrap_or("");

        token_strs.push_back(match_str.to_string());
    }

    return token_strs;
}

pub fn parse_str(code: &str) -> Result<Vec<Value>, ParseError> {
    let mut values = Vec::new();

    let mut tokens = tokenize(code);

    while tokens.len() > 0 {
        values.push(try!(parse(&mut tokens)));
    }

    Ok(values)
}

/// Extracts a single token out of a list of strings, which may contain multiple tokens
fn parse(list: &mut VecDeque<String>) -> ParseResult {
    let head = match list.pop_front() {
        Some(string) => string,
        None => return Err(ParseError::UnreadableSourceCode),
    };

    match &head[..] {
        LIST_OPEN => {
            let tokens = try!(parse_list(list, LIST_CLOSE));
            Ok(Value::List(tokens))
        },
        QUOTE_OPEN => {
            let tokens = try!(parse_list(list, QUOTE_CLOSE));
            Ok(Value::Quote(box Value::List(tokens)))
        },
        LIST_CLOSE | QUOTE_CLOSE => Err(ParseError::InvalidListDelimitter),
        r"'" => {
            let token = try!(parse(list));
            Ok(Value::Quote(box token))
        },
        atom => Ok(parse_atom(atom.to_string())),
    }
}

fn parse_atom(atom: String) -> Value {
    if let Ok(n) = atom.parse() {
        Value::Number(n)
    } else if let Some(lit) = string_lit(&atom[..]) {
        Value::String(lit)
    } else {
        Value::Symbol(atom.to_string())
    }
}

fn parse_list(list: &mut VecDeque<String>, delimit: &str) -> Result<Vec<Value>, ParseError> {
    let mut tokens = Vec::new();

    while let Some(item_str) = list.pop_front() {
        if &item_str[..] == delimit {
            return Ok(tokens);
        }

        // If not the end of the list, push the item back and continue to process the list
        list.push_front(item_str);
        tokens.push(try!(parse(list)));
    }

    Err(ParseError::UnclosedList)
}

fn string_lit(slice: &str) -> Option<String> {
    let len = slice.len();
    let end = len - 1;

    if &slice[0..1] == "\"" && &slice[end .. len] == "\"" {
        let lit = &slice[1..end];

        let lit_str = lit.to_string()
            .replace(r"\n", "\n")
            .replace(r"\t", "\t")
            .replace(r"\r", "\r");

        return Some(lit_str);
    }

    return None;
}
<commit_msg>Add hex support<commit_after>use regex::Regex;

use std::collections::VecDeque;
use std::fmt;

use eval::FuncError;
use value::Value;

#[derive(Debug, Clone)]
pub enum ParseError {
    UnclosedList,
    InvalidListDelimitter,
    UnreadableSourceCode,
    InvalidHexLiteral,
}

#[derive(Debug, Clone)]
pub struct FilePos {
    line: usize,
    col: usize,
}

impl FilePos {
    fn from_offset(text: &str, mut offset: usize) -> FilePos {
        let mut line_no = 1;

        for (cur_offset, ch) in text.bytes().enumerate() {
            if cur_offset >= offset {
                break;
            }

            if ch == '\n' as u8 {
                line_no += 1;
            }
        }

        FilePos {
            line: line_no,
            col: 0,
        }
    }
}

pub type ParseResult = Result<Value, ParseError>; // result of tokenizing a single token

const LIST_OPEN: &'static str = "(";
const LIST_CLOSE: &'static str = ")";

const QUOTE_OPEN: &'static str = "{";
const QUOTE_CLOSE: &'static str = "}";

pub fn write_list<T>(fmt: &mut fmt::Formatter, list: &Vec<T>, start: &str, sep: &str, end: &str) -> fmt::Result where T: fmt::Debug {
    if list.len() == 0 {
        return write!(fmt, "{}{}", start, end);
    }

    write!(fmt, "{}{:?}", start, list[0]);

    for token in &list[1..] {
        write!(fmt, "{}{:?}", sep, token);
    }

    write!(fmt, "{}", end)
}

fn tokenize(code: &str) -> VecDeque<String> {
    let string_re = r#""[^"]*""#;
    let sym_re = r"[-!?#\w\.]+";
    let num_re = r"\d+\.?\d*e?\d*";
    let list_re = r"[(){}\[\]]";
    let op_re = r"\+|-|\*|/|\^|&|\||=|\\|<|>";
    let quote_re = r"'";

    let regex = format!("{}|{}|{}|{}|{}|{}", string_re, num_re, sym_re, list_re, op_re, quote_re);

    let re = match Regex::new(&regex) {
        Ok(re) => re,
        Err(_) => return VecDeque::new(),
    };

    let spaced_code = code.to_string()
        .replace("(", " ( ")
        .replace(")", " ) ");

    let mut token_strs = VecDeque::new();

    for cap in re.captures_iter(&spaced_code) {
        let (offset, _) = cap.get(0)
            .map(|text| (text.start(), text.end()))
            .unwrap_or((0, 0));

        let match_str = cap.get(0)
            .map(|text| text.as_str())
            .unwrap_or("");

        token_strs.push_back(match_str.to_string());
    }

    return token_strs;
}

pub fn parse_str(code: &str) -> Result<Vec<Value>, ParseError> {
    let mut values = Vec::new();

    let mut tokens = tokenize(code);

    while tokens.len() > 0 {
        values.push(try!(parse(&mut tokens)));
    }

    Ok(values)
}

/// Extracts a single token out of a list of strings, which may contain multiple tokens
fn parse(list: &mut VecDeque<String>) -> ParseResult {
    let head = match list.pop_front() {
        Some(string) => string,
        None => return Err(ParseError::UnreadableSourceCode),
    };

    match &head[..] {
        LIST_OPEN => {
            let tokens = try!(parse_list(list, LIST_CLOSE));
            Ok(Value::List(tokens))
        },
        QUOTE_OPEN => {
            let tokens = try!(parse_list(list, QUOTE_CLOSE));
            Ok(Value::Quote(box Value::List(tokens)))
        },
        LIST_CLOSE | QUOTE_CLOSE => Err(ParseError::InvalidListDelimitter),
        r"'" => {
            let token = try!(parse(list));
            Ok(Value::Quote(box token))
        },
        atom => parse_atom(atom.to_string()),
    }
}

fn parse_atom(atom: String) -> ParseResult {
    if atom.starts_with("#") {
        let value = usize::from_str_radix(&atom[1..], 16).map_err(|_| ParseError::InvalidHexLiteral)?;
        Ok(Value::Number(value as f32))
    } else if let Ok(n) = atom.parse() {
        Ok(Value::Number(n))
    } else if let Some(lit) = string_lit(&atom[..]) {
        Ok(Value::String(lit))
    } else {
        Ok(Value::Symbol(atom.to_string()))
    }
}

fn parse_list(list: &mut VecDeque<String>, delimit: &str) -> Result<Vec<Value>, ParseError> {
    let mut tokens = Vec::new();

    while let Some(item_str) = list.pop_front() {
        if &item_str[..] == delimit {
            return Ok(tokens);
        }

        // If not the end of the list, push the item back and continue to process the list
        list.push_front(item_str);
        tokens.push(try!(parse(list)));
    }

    Err(ParseError::UnclosedList)
}

fn string_lit(slice: &str) -> Option<String> {
    let len = slice.len();
    let end = len - 1;

    if &slice[0..1] == "\"" && &slice[end .. len] == "\"" {
        let lit = &slice[1..end];

        let lit_str = lit.to_string()
            .replace(r"\n", "\n")
            .replace(r"\t", "\t")
            .replace(r"\r", "\r");

        return Some(lit_str);
    }

    return None;
}
<|endoftext|>


<commit_before>use std::collections::{HashMap,BTreeSet};

use source::Src;
use var::Var;
use eval::Eval;

#[derive(Debug,PartialEq)]
pub struct SrcBlock {
    pub name: String,
    pub src: Vec<Src>,
    pub await_idx: usize,
    pub visited: bool,
}

#[derive(Debug,PartialEq)]
pub struct DefBlock {
    pub name: String,
    pub def: HashMap<String,Var>
}

#[derive(Debug,PartialEq)]
pub enum Block {
    Src(SrcBlock),
    Def(DefBlock),
}

/// Intermediate Representation
///
/// This is used during the parsing stage
#[derive(Debug,Clone,PartialEq)]
pub enum IR {
    String(String),
    /// A non-quoted string turns into a symbol/token
    Sym(String),
    /// Key-Value, pre-parsed
    Map(Vec<IR>),
}

impl From<IR> for String {
    fn from(t:IR) -> String {
        match t {
            IR::String(s) => s,
            IR::Sym(s) => s,
            IR::Map(mut v) => {
                let mut s = "{".to_owned();
                for n in v.drain(..) {
                    let n:String = n.into();
                    s.push_str(&n);
                }

                s.push('}');

                s
            }
        }
    }
}

/// Map object for Selects
pub type Map = HashMap<String,Vec<Var>>;

pub struct Parser(Vec<Block>);

use std::ops::Deref;
impl Deref for Parser {
    type Target = Vec<Block>;
    fn deref(&self) -> &Self::Target { &self.0 }
}

impl Parser {
    pub fn parse_blocks (src: &str) -> Parser {
        let mut v = vec!();
        let mut exp = String::new();
        let mut exps: Vec<IR> = vec!();
        let mut map_ir: Vec<IR> = vec!(); //contains pre-parsed map
        let mut block: Option<Block> = None;

        let mut in_string = false;
        let mut in_comment = false;
        let mut in_vec = false;
        let mut in_map = false;
        let mut was_if = false;

        let mut usyms = BTreeSet::new(); //unique set, remove dupes

        for c in src.chars() {
            if !in_comment && !in_string {
                if c == '[' { in_vec = true; continue }
                else if c == ']' { in_vec = false; }

            }

            if c == '#' && !in_string { in_comment = true; continue }
            else if  c == '\n' && in_comment && !in_string { in_comment = false; }

            if c == '\n' && (in_vec || in_map) && !in_comment && !in_string { continue }

            if (c == ']' ||
                c == '}' ||
                c == '#' ||
                c == '\n')
                && !in_string && !in_comment
            {
                for n in exp.split_whitespace() {
                    let sym = IR::Sym(n.trim().to_owned());
                    if in_map { map_ir.push(sym); }
                    else { exps.push(sym); }
                }

                if c == '}' && in_map && !in_comment{
                    in_map = false;
                    exps.push(IR::Map(map_ir));
                    map_ir = vec![];
                }

                exp = String::new();

                if exps.len() < 1 { continue }

                // determine block type
                if block.is_none() {
                    let name = exps.remove(0).into();
                    if name == "def" {
                        let b = DefBlock {
                            name: exps.pop().unwrap().into(),
                            def: HashMap::new(),
                        };

                        block = Some(Block::Def(b));
                    }
                    else {
                        let b = SrcBlock {
                            name: name,
                            src: vec!(),
                            await_idx: 0,
                            visited: false,
                        };

                        block = Some(Block::Src(b));
                    }

                }
                else { // build block type
                    let mut qsyms:Vec<(String,String)> = vec!();
                    let adjust_sym = |qsyms: &mut Vec<(String,String)>, s: &mut String| {
                        if s.chars().next().expect("ERROR: Empty Eager Symbol") == '!' {
                            let mut sym = "not_".to_owned();
                            sym.push_str(s[1..].trim());

                            let osym = s.trim().to_owned();

                            qsyms.push((sym.clone(),osym));
                            *s = sym;
                        }
                    };

                    // this builds symbol refs as a convenience
                    if exps.len() > 2 { //must be if/or/comp blocks
                        for n in exps.iter_mut() {
                            match n {
                                &mut IR::Sym(ref mut s) => {
                                    adjust_sym(&mut qsyms,s);
                                },
                                &mut IR::Map(ref mut v) => {
                                    for n in v.iter_mut() {
                                        match n {
                                            &mut IR::Sym(ref mut s) => {
                                                adjust_sym(&mut qsyms,s);
                                            },
                                            _ => {},
                                        }
                                    }
                                },
                                _ => {},
                            }
                        }
                    }

                    match block {
                        Some(Block::Def(ref mut b)) => {
                            let v = exps.pop().unwrap();
                            b.def.insert(exps.pop().unwrap().into(),
                                         Var::parse(v));
                        },
                        Some(Block::Src(ref mut b)) => {
                            let mut srcs = vec![];

                            for (qsym,sym) in qsyms.drain(..) {
                                if usyms.contains(&qsym) { continue }
                                usyms.insert(qsym.clone());

                                let src = Src::parse(vec![IR::Sym(qsym),
                                                          IR::Sym(sym)]);
                                srcs.push(src);
                            }

                            let src = Src::parse(exps);
                            srcs.push(src);

                            for src in srcs.drain(..) {
                                match &src {
                                    &Src::If(_,_,_) => { was_if = true; },
                                    &Src::Or(_,_) => {
                                        if !was_if {
                                            panic!("ERROR: IF must prepend OR");
                                        }
                                    },
                                    _ => { was_if = false; },
                                }

                                b.src.push(src);
                            }
                        },
                        _ => {}
                    }

                    exps = vec!();
                }
            }
            else if c == '"' && !in_comment {
                in_string = !in_string;
                if in_string { //starting a new quoted string? let's push this sym
                    for n in exp.split_whitespace() {
                        let sym = IR::Sym(n.trim().to_owned());
                        if in_map { map_ir.push(sym); }
                        else { exps.push(sym); }
                    }
                    exp = String::new();
                }
                else if !in_string { //finished the quoted string?
                    let sym = IR::String(exp);
                    if in_map { map_ir.push(sym); }
                    else { exps.push(sym); }
                    exp = String::new();
                }
            }
            else if c == ';' && !in_string && !in_comment {
                //fail otherwise, block should be built!
                v.push(block.expect("ERROR: Parse Block no built!"));
                usyms.clear(); //clear out on new block
                block = None;
            }
            else {
                if c == '{' && !in_comment && !in_string {
                    in_map = true;
                    // push previous symbol
                    let sym = IR::Sym(exp.trim().to_owned());
                    exps.push(sym);
                    exp.clear();
                }
                else if !in_comment {
                    exp.push(c);
                }
            }
        }

        Parser(v)
    }

    pub fn into_env (mut self) -> Env {
        let mut src = HashMap::new();
        let mut def = HashMap::new();

        for b in self.0.drain(..) {
            match b {
                Block::Def(db) => {
                    def.insert(db.name.clone(), db);
                },
                Block::Src(sb) => {
                    src.insert(sb.name.clone(), sb);
                },
            }

        }

        Env { def: def, src: src }
    }

    /// Parses a map from IR
    ///
    /// Currently a map must be identically sized
    /// Future versions will be parsed using commas for variable sized maps
    pub fn parse_map (map_ir: IR) -> Option<Map> {
        let mut map: Map = HashMap::new(); // optionally unbounded val-lengths

        match map_ir {
            IR::Map(mut exps) => {
                let mut key = "".to_owned();
                let mut vals = vec![];

                for n in exps.drain(..) {
                    if key.is_empty() { key = n.into(); continue }

                    match n {
                        IR::Sym(mut s) => {
                            if s.chars().last() == Some(',') {
                                let _ = s.pop();
                                vals.push(Var::parse(IR::Sym(s)));

                                map.insert(key,vals);
                                vals = vec![];
                                key = "".to_owned();

                                continue
                            }

                            vals.push(Var::parse(IR::Sym(s)));
                        },
                        _ => { vals.push(Var::parse(n)); },
                    }
                }

                if !key.is_empty() && vals.len() > 0 {
                    map.insert(key,vals);
                }
                else if !key.is_empty() {
                    panic!("ERROR: Unbalanced MAP at: {:?}",key);
                }

                Some(map)
            },
            _=> {None}
        }
    }
}

/// Def alias used for internal evaluation purposes
pub type Def = HashMap<String, DefBlock>;

impl Env {
    pub fn def_contains(def: &Def, path: Option<Vec<&str>>, lookup: &str) -> bool {
        if let Some(path) = path {
            if let Some(ref def) = def.get(path[0]) {
                return def.def.contains_key(lookup)
            }
        }

        false
    }
}

/// Environment containing all parsed definition and source blocks
pub struct Env {
    pub def: Def,
    pub src: HashMap<String, SrcBlock>
}

impl Eval for Def {
    fn get (&self, path: Option<Vec<&str>>, lookup: &str) -> Option<Var> {
        if let Some(path) = path {
            if let Some(ref def) = self.get(path[0]) {
                if let Some(v) = def.def.get(lookup) {
                    return Some(v.clone())
                }
            }
        }

        None
    }

    #[allow(unused_variables)]
    fn set (&mut self, path: Option<Vec<&str>>, lookup: &str, var: Var) {
        if let Some(path) = path {
            if let Some(ref mut def) = self.get_mut(path[0]) {
                if let Some(v) = def.def.get_mut(lookup) {
                    *v = var;
                }
            }
        }
    }

    #[allow(unused_variables)]
    fn call (&mut self, var: Var, fun: &str, vars: &Vec<Var>) -> Option<Var> {
        None
    }
}
<commit_msg>update doc<commit_after>use std::collections::{HashMap,BTreeSet};

use source::Src;
use var::Var;
use eval::Eval;

#[derive(Debug,PartialEq)]
pub struct SrcBlock {
    pub name: String,
    pub src: Vec<Src>,
    pub await_idx: usize,
    pub visited: bool,
}

#[derive(Debug,PartialEq)]
pub struct DefBlock {
    pub name: String,
    pub def: HashMap<String,Var>
}

#[derive(Debug,PartialEq)]
pub enum Block {
    Src(SrcBlock),
    Def(DefBlock),
}

/// Intermediate Representation
///
/// This is used during the parsing stage
#[derive(Debug,Clone,PartialEq)]
pub enum IR {
    String(String),
    /// A non-quoted string turns into a symbol/token
    Sym(String),
    /// Key-Value, pre-parsed
    Map(Vec<IR>),
}

impl From<IR> for String {
    fn from(t:IR) -> String {
        match t {
            IR::String(s) => s,
            IR::Sym(s) => s,
            IR::Map(mut v) => {
                let mut s = "{".to_owned();
                for n in v.drain(..) {
                    let n:String = n.into();
                    s.push_str(&n);
                }

                s.push('}');

                s
            }
        }
    }
}

/// Map object for Selects
pub type Map = HashMap<String,Vec<Var>>;

pub struct Parser(Vec<Block>);

use std::ops::Deref;
impl Deref for Parser {
    type Target = Vec<Block>;
    fn deref(&self) -> &Self::Target { &self.0 }
}

impl Parser {
    pub fn parse_blocks (src: &str) -> Parser {
        let mut v = vec!();
        let mut exp = String::new();
        let mut exps: Vec<IR> = vec!();
        let mut map_ir: Vec<IR> = vec!(); //contains pre-parsed map
        let mut block: Option<Block> = None;

        let mut in_string = false;
        let mut in_comment = false;
        let mut in_vec = false;
        let mut in_map = false;
        let mut was_if = false;

        let mut usyms = BTreeSet::new(); //unique set, remove dupes

        for c in src.chars() {
            if !in_comment && !in_string {
                if c == '[' { in_vec = true; continue }
                else if c == ']' { in_vec = false; }

            }

            if c == '#' && !in_string { in_comment = true; continue }
            else if  c == '\n' && in_comment && !in_string { in_comment = false; }

            if c == '\n' && (in_vec || in_map) && !in_comment && !in_string { continue }

            if (c == ']' ||
                c == '}' ||
                c == '#' ||
                c == '\n')
                && !in_string && !in_comment
            {
                for n in exp.split_whitespace() {
                    let sym = IR::Sym(n.trim().to_owned());
                    if in_map { map_ir.push(sym); }
                    else { exps.push(sym); }
                }

                if c == '}' && in_map && !in_comment{
                    in_map = false;
                    exps.push(IR::Map(map_ir));
                    map_ir = vec![];
                }

                exp = String::new();

                if exps.len() < 1 { continue }

                // determine block type
                if block.is_none() {
                    let name = exps.remove(0).into();
                    if name == "def" {
                        let b = DefBlock {
                            name: exps.pop().unwrap().into(),
                            def: HashMap::new(),
                        };

                        block = Some(Block::Def(b));
                    }
                    else {
                        let b = SrcBlock {
                            name: name,
                            src: vec!(),
                            await_idx: 0,
                            visited: false,
                        };

                        block = Some(Block::Src(b));
                    }

                }
                else { // build block type
                    let mut qsyms:Vec<(String,String)> = vec!();
                    let adjust_sym = |qsyms: &mut Vec<(String,String)>, s: &mut String| {
                        if s.chars().next().expect("ERROR: Empty Eager Symbol") == '!' {
                            let mut sym = "not_".to_owned();
                            sym.push_str(s[1..].trim());

                            let osym = s.trim().to_owned();

                            qsyms.push((sym.clone(),osym));
                            *s = sym;
                        }
                    };

                    // this builds symbol refs as a convenience
                    if exps.len() > 2 { //must be if/or/comp blocks
                        for n in exps.iter_mut() {
                            match n {
                                &mut IR::Sym(ref mut s) => {
                                    adjust_sym(&mut qsyms,s);
                                },
                                &mut IR::Map(ref mut v) => {
                                    for n in v.iter_mut() {
                                        match n {
                                            &mut IR::Sym(ref mut s) => {
                                                adjust_sym(&mut qsyms,s);
                                            },
                                            _ => {},
                                        }
                                    }
                                },
                                _ => {},
                            }
                        }
                    }

                    match block {
                        Some(Block::Def(ref mut b)) => {
                            let v = exps.pop().unwrap();
                            b.def.insert(exps.pop().unwrap().into(),
                                         Var::parse(v));
                        },
                        Some(Block::Src(ref mut b)) => {
                            let mut srcs = vec![];

                            for (qsym,sym) in qsyms.drain(..) {
                                if usyms.contains(&qsym) { continue }
                                usyms.insert(qsym.clone());

                                let src = Src::parse(vec![IR::Sym(qsym),
                                                          IR::Sym(sym)]);
                                srcs.push(src);
                            }

                            let src = Src::parse(exps);
                            srcs.push(src);

                            for src in srcs.drain(..) {
                                match &src {
                                    &Src::If(_,_,_) => { was_if = true; },
                                    &Src::Or(_,_) => {
                                        if !was_if {
                                            panic!("ERROR: IF must prepend OR");
                                        }
                                    },
                                    _ => { was_if = false; },
                                }

                                b.src.push(src);
                            }
                        },
                        _ => {}
                    }

                    exps = vec!();
                }
            }
            else if c == '"' && !in_comment {
                in_string = !in_string;
                if in_string { //starting a new quoted string? let's push this sym
                    for n in exp.split_whitespace() {
                        let sym = IR::Sym(n.trim().to_owned());
                        if in_map { map_ir.push(sym); }
                        else { exps.push(sym); }
                    }
                    exp = String::new();
                }
                else if !in_string { //finished the quoted string?
                    let sym = IR::String(exp);
                    if in_map { map_ir.push(sym); }
                    else { exps.push(sym); }
                    exp = String::new();
                }
            }
            else if c == ';' && !in_string && !in_comment {
                //fail otherwise, block should be built!
                v.push(block.expect("ERROR: Parse Block no built!"));
                usyms.clear(); //clear out on new block
                block = None;
            }
            else {
                if c == '{' && !in_comment && !in_string {
                    in_map = true;
                    // push previous symbol
                    let sym = IR::Sym(exp.trim().to_owned());
                    exps.push(sym);
                    exp.clear();
                }
                else if !in_comment {
                    exp.push(c);
                }
            }
        }

        Parser(v)
    }

    pub fn into_env (mut self) -> Env {
        let mut src = HashMap::new();
        let mut def = HashMap::new();

        for b in self.0.drain(..) {
            match b {
                Block::Def(db) => {
                    def.insert(db.name.clone(), db);
                },
                Block::Src(sb) => {
                    src.insert(sb.name.clone(), sb);
                },
            }

        }

        Env { def: def, src: src }
    }

    /// Parses a map from IR
    ///
    /// Parsed using commas for variable sized maps
    pub fn parse_map (map_ir: IR) -> Option<Map> {
        let mut map: Map = HashMap::new(); // optionally unbounded val-lengths

        match map_ir {
            IR::Map(mut exps) => {
                let mut key = "".to_owned();
                let mut vals = vec![];

                for n in exps.drain(..) {
                    if key.is_empty() { key = n.into(); continue }

                    match n {
                        IR::Sym(mut s) => {
                            if s.chars().last() == Some(',') {
                                let _ = s.pop();
                                vals.push(Var::parse(IR::Sym(s)));

                                map.insert(key,vals);
                                vals = vec![];
                                key = "".to_owned();

                                continue
                            }

                            vals.push(Var::parse(IR::Sym(s)));
                        },
                        _ => { vals.push(Var::parse(n)); },
                    }
                }

                if !key.is_empty() && vals.len() > 0 {
                    map.insert(key,vals);
                }
                else if !key.is_empty() {
                    panic!("ERROR: Unbalanced MAP at: {:?}",key);
                }

                Some(map)
            },
            _=> {None}
        }
    }
}

/// Def alias used for internal evaluation purposes
pub type Def = HashMap<String, DefBlock>;

impl Env {
    pub fn def_contains(def: &Def, path: Option<Vec<&str>>, lookup: &str) -> bool {
        if let Some(path) = path {
            if let Some(ref def) = def.get(path[0]) {
                return def.def.contains_key(lookup)
            }
        }

        false
    }
}

/// Environment containing all parsed definition and source blocks
pub struct Env {
    pub def: Def,
    pub src: HashMap<String, SrcBlock>
}

impl Eval for Def {
    fn get (&self, path: Option<Vec<&str>>, lookup: &str) -> Option<Var> {
        if let Some(path) = path {
            if let Some(ref def) = self.get(path[0]) {
                if let Some(v) = def.def.get(lookup) {
                    return Some(v.clone())
                }
            }
        }

        None
    }

    #[allow(unused_variables)]
    fn set (&mut self, path: Option<Vec<&str>>, lookup: &str, var: Var) {
        if let Some(path) = path {
            if let Some(ref mut def) = self.get_mut(path[0]) {
                if let Some(v) = def.def.get_mut(lookup) {
                    *v = var;
                }
            }
        }
    }

    #[allow(unused_variables)]
    fn call (&mut self, var: Var, fun: &str, vars: &Vec<Var>) -> Option<Var> {
        None
    }
}
<|endoftext|>