<commit_before>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug)]
pub struct TokenWithContext {
    token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    line: usize,
}

struct Scanner<'a> {
    current: usize,
    current_lexeme: String,
    line: usize,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current: 0,
            current_lexeme: "".into(),
            line: 1, // 1-indexed,
            source: multipeek(source.chars()),
        }
    }

    fn is_at_end(&mut self) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(_) => false,
            None => true,
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn advance(&mut self) -> char {
        self.current += 1;
        // TODO: handle None!
        let c = self.source.next().unwrap();
        self.current_lexeme.push(c);
        if c == '\n' {
            self.line += 1;
        }
        c
    }

    fn is_match(&mut self, expected: char) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => {
                if c == expected {
                    let _ = self.source.next();
                    self.current += 1;
                    true
                } else {
                    false
                }
            }
            None => false,
        }
    }

    fn add_context(&mut self, token: Token) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            line: self.line,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.line;
        self.source.reset_peek();
        while match self.source.peek() {
            Some(&p) => p != '"',
            None => false,
        } {
            self.advance();
        }
        if self.is_at_end() {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        self.advance();
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        while self.peek_check(&is_digit) {
            self.advance();
        }
        // Here we do double lookahead
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => {
                        if p1 == '.' && is_digit(p2) {
                            self.advance(); // Consume the .
                            while self.peek_check(&is_digit) {
                                self.advance();
                            }
                        }
                    }
                    None => (),
                }
            }
            None => (),
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        while self.peek_check(&is_alphanumeric) {
            self.advance();
        }
        // TODO: take a ref in the first place
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let token = match self.advance() {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.is_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.is_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.is_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.is_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.is_match('/') {
                    // Comments go on till the end of the line
                    while self.peek_check(&|c| c != '\n') {
                        self.advance();
                    }
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, pos {}",
                                   c,
                                   self.line,
                                   self.current - 1));
            }
        };
        Ok(self.add_context(token))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    while !scanner.is_at_end() {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            _ => tokens.push(token_with_context),
        }
    }
    tokens.push(TokenWithContext {
        token: Token::Eof,
        lexeme: "".into(),
        line: scanner.line,
    });
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Eof);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[10].token, Token::Eof);
        assert_eq!(tokens[1].line, 1);
        assert_eq!(tokens[9].line, 2);
    }
}
<commit_msg>Double lookahead<commit_after>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug)]
pub struct TokenWithContext {
    token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    line: usize,
}

struct Scanner<'a> {
    current: usize,
    current_lexeme: String,
    line: usize,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current: 0,
            current_lexeme: "".into(),
            line: 1, // 1-indexed,
            source: multipeek(source.chars()),
        }
    }

    fn is_at_end(&mut self) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(_) => false,
            None => true,
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> char {
        self.current += 1;
        // TODO: handle None!
        let c = self.source.next().unwrap();
        self.current_lexeme.push(c);
        if c == '\n' {
            self.line += 1;
        }
        c
    }

    fn is_match(&mut self, expected: char) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => {
                if c == expected {
                    let _ = self.source.next();
                    self.current += 1;
                    true
                } else {
                    false
                }
            }
            None => false,
        }
    }

    fn add_context(&mut self, token: Token) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            line: self.line,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.line;
        self.source.reset_peek();
        while self.peek_check(&|c| c != '"') {
            self.advance();
        }
        if self.is_at_end() {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        self.advance();
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        while self.peek_check(&is_digit) {
            self.advance();
        }
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            while self.peek_check(&is_digit) {
                self.advance();
            }
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        while self.peek_check(&is_alphanumeric) {
            self.advance();
        }
        // TODO: take a ref in the first place
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let token = match self.advance() {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.is_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.is_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.is_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.is_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.is_match('/') {
                    // Comments go on till the end of the line
                    while self.peek_check(&|c| c != '\n') {
                        self.advance();
                    }
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, pos {}",
                                   c,
                                   self.line,
                                   self.current - 1));
            }
        };
        Ok(self.add_context(token))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    while !scanner.is_at_end() {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            _ => tokens.push(token_with_context),
        }
    }
    tokens.push(TokenWithContext {
        token: Token::Eof,
        lexeme: "".into(),
        line: scanner.line,
    });
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Eof);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[10].token, Token::Eof);
        assert_eq!(tokens[1].line, 1);
        assert_eq!(tokens[9].line, 2);
    }
}
<|endoftext|>


<commit_before>use std::from_str::FromStr;

use super::ScanCursor;
use super::{ScanError, OtherScanError};

macro_rules! from_str_scanner {
	($scan_fn:path -> $T:ty as $name:expr) => {
		impl<'a> Scanner<'a, $T> for $T {
			fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<($T, Cur), ScanError> {
				let err = |cur:&Cur| Err(OtherScanError(format!(concat!("expected ",$name,", got `{}`"), cur.tail_str()), cursor.consumed()));

				let end = match $scan_fn(cursor.tail_str()) {
					Some(i) => i,
					None => return err(cursor)
				};

				let s = cursor.str_slice_to(end);
				let cursor = cursor.slice_from(end);

				FromStr::from_str(s)
					.map(ref |i| Ok((i, cursor.clone())))
					.unwrap_or_else(|| err(&cursor))
			}
		}
	};
}

pub trait Scanner<'a, T> {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(Self, Cur), ScanError>;
}

impl<'a> Scanner<'a, bool> for bool {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(bool, Cur), ScanError> {
		cursor.expect_tok("true").map(|c| (true, c))
			.or_else(|_| cursor.expect_tok("false").map(|c| (false, c)))
			.or_else(|_| Err(OtherScanError(format!("expected `true` or `false`, got `{}`", cursor.tail_str()), cursor.consumed())))
	}
}

impl<'a> Scanner<'a, &'a str> for &'a str {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(&'a str, Cur), ScanError> {
		cursor.pop_token().map(|sc| Ok(sc))
			.unwrap_or_else(|| Err(OtherScanError("expected token".into_string(), cursor.consumed())))
	}
}

from_str_scanner! { scan_float -> f32 as "real number" }
from_str_scanner! { scan_float -> f64 as "real number" }
from_str_scanner! { scan_int -> i8 as "8-bit integer" }
from_str_scanner! { scan_int -> i16 as "16-bit integer" }
from_str_scanner! { scan_int -> i32 as "32-bit integer" }
from_str_scanner! { scan_int -> i64 as "64-bit integer" }
from_str_scanner! { scan_int -> int as "integer" }
from_str_scanner! { scan_uint -> u8 as "8-bit unsigned integer" }
from_str_scanner! { scan_uint -> u16 as "16-bit unsigned integer" }
from_str_scanner! { scan_uint -> u32 as "32-bit unsigned integer" }
from_str_scanner! { scan_uint -> u64 as "64-bit unsigned integer" }
from_str_scanner! { scan_uint -> uint as "unsigned integer" }

fn next_char_at(s: &str, at: uint) -> uint {
	let ::std::str::CharRange { ch: _, next } = s.char_range_at(at);
	next
}

fn scan_float(s: &str) -> Option<uint> {
	enum State {
		Start,
		Whole,
		Suffix,
		ExponentStart,
		Exponent,
	}

	let mut state = Start;

	for (i,c) in s.char_indices() {
		match state {
			Start => {
				assert!(i == 0);
				match c {
					'0'..'9' | '-' => state = Whole,
					_ => return None
				}
			},
			Whole => match c {
				'0'..'9' => (),
				'.' => state = Suffix,
				'e' | 'E' => state = ExponentStart,
				_ => return Some(i)
			},
			Suffix => match c {
				'0'..'9' => (),
				'e' | 'E' => state = ExponentStart,
				_ => return Some(i),
			},
			ExponentStart => match c {
				'+' | '-' | '0'..'9' => state = Exponent,
				_ => return Some(i)
			},
			Exponent => match c {
				'0'..'9' => (),
				_ => return Some(i)
			}
		}
	}

	return Some(s.len())
}

fn scan_uint<'a>(s: &'a str) -> Option<uint> {
	s.char_indices()
		.take_while(|&(_,c)| '0' <= c && c <= '9')
		.map(|(i,_)| Some(next_char_at(s, i)))
		.last().unwrap_or(None)
}

fn scan_int<'a>(s: &'a str) -> Option<uint> {
	if s.len() == 0 { return None }

	let (s, off) = if s.char_at(0) == '-' {
		(s.slice_from(1), 1)
	} else {
		(s, 0)
	};

	scan_uint(s).map(|end| end+off)
}

#[cfg(test)]
mod test {
	use Cursor;
	use ScanError;
	use super::Scanner;
	use tokenizer::WordsAndInts;
	use whitespace::Ignore;

	fn cur<'a>(s: &'a str) -> Cursor<'a, WordsAndInts, Ignore> {
		Cursor::new(s, WordsAndInts, Ignore)
	}

	fn scan_a<'a, T: Scanner<'a, T>>(s: &'a str) -> Result<(T, Cursor<'a, WordsAndInts, Ignore>), ScanError> {
		Scanner::scan(&cur(s))
	}

	#[test]
	fn test_bool() {
		assert!(scan_a::<bool>("").err().is_some());
		assert!(scan_a::<bool>("true").unwrap().0 == true);
		assert!(scan_a::<bool>("false").unwrap().0 == false);
		assert!(scan_a::<bool>("yes").err().is_some());
		assert!(scan_a::<bool>("no").err().is_some());
		assert!(scan_a::<bool>("on").err().is_some());
		assert!(scan_a::<bool>("off").err().is_some());
		assert!(scan_a::<bool>("1").err().is_some());
		assert!(scan_a::<bool>("0").err().is_some());
	}

	#[test]
	fn test_floats() {
		use std::from_str::FromStr;
		use std::num::{Float, Zero};

		fn test<'a, F: Float + Scanner<'a, F> + FromStr>() {
			let f = |v:F| v;
			let fs = |s:&str| -> F FromStr::from_str(s).unwrap();

			assert!(scan_a::<F>("").err().is_some());
			assert!(scan_a::<F>("0").ok().unwrap().0 == f(Zero::zero()));
			assert!(scan_a::<F>("0.0").ok().unwrap().0 == f(Zero::zero()));
			assert!(scan_a::<F>("-0").ok().unwrap().0 == -f(Zero::zero()));
			assert!(scan_a::<F>("1.0").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.00").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.0e0").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.0e1").ok().unwrap().0 == fs("10.0"));
		}

		test::<f32>();
		test::<f64>();
	}

	#[test]
	fn test_sized_ints() {
		use std::fmt::Show;
		use std::num::{Bounded, Int, Zero};

		fn test<'a, I: Int + Scanner<'a, I> + Show>(check_past: bool) {
			let zero: I = Zero::zero();
			let min: I = Bounded::min_value();
			let max: I = Bounded::max_value();

			assert!(scan_a::<I>("").err().is_some());
			assert!(scan_a::<I>("0").ok().unwrap().0 == zero);
			assert!(scan_a::<I>(format!("{}", min).as_slice()).ok().unwrap().0 == min);
			assert!(scan_a::<I>(format!("{}", max).as_slice()).ok().unwrap().0 == max);

			if check_past {
				let past_min: i64 = (min.to_i64()).unwrap() - 1;
				let past_max: u64 = (max.to_u64()).unwrap() + 1;
				assert!(scan_a::<I>(format!("{}", past_min).as_slice()).err().is_some());
				assert!(scan_a::<I>(format!("{}", past_max).as_slice()).err().is_some());
			}
		}

		test::<i8>(true);
		test::<i16>(true);
		test::<i32>(true);
		test::<i64>(false);
		test::<u8>(true);
		test::<u16>(true);
		test::<u32>(true);
		test::<u64>(false);
	}

	#[test]
	fn test_int() {
		assert!(scan_a::<int>("").err().is_some());
		assert!(scan_a::<int>("0").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("42").ok().unwrap().0 == 42);
		assert!(scan_a::<int>("1_234").ok().unwrap().0 == 1);
		assert!(scan_a::<int>("x").err().is_some());
		assert!(scan_a::<int>("0x").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("42x").ok().unwrap().0 == 42);
		assert!(scan_a::<int>("-").err().is_some());
		assert!(scan_a::<int>("-0").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("-42").ok().unwrap().0 == -42);
		assert!(scan_a::<int>("-1_234").ok().unwrap().0 == -1);
	}

	#[test]
	fn test_uint() {
		assert!(scan_a::<uint>("").err().is_some());
		assert!(scan_a::<uint>("0").ok().unwrap().0 == 0);
		assert!(scan_a::<uint>("42").ok().unwrap().0 == 42);
		assert!(scan_a::<uint>("1_234").ok().unwrap().0 == 1);
		assert!(scan_a::<uint>("x").err().is_some());
		assert!(scan_a::<uint>("0x").ok().unwrap().0 == 0);
		assert!(scan_a::<uint>("42x").ok().unwrap().0 == 42);
		assert!(scan_a::<uint>("-").err().is_some());
		assert!(scan_a::<uint>("-0").err().is_some());
		assert!(scan_a::<uint>("-42").err().is_some());
		assert!(scan_a::<uint>("-1_234").err().is_some());
	}

	#[test]
	fn test_str() {
		assert!(scan_a::<&str>("").err().is_some());
		assert!(scan_a::<&str>("a").ok().unwrap().0 == "a");
		assert!(scan_a::<&str>("a b").ok().unwrap().0 == "a");
		assert!(scan_a::<&str>("abc").ok().unwrap().0 == "abc");
		assert!(scan_a::<&str>("ab-c").ok().unwrap().0 == "ab");
	}
}
<commit_msg>Added char scanner.<commit_after>use std::from_str::FromStr;

use super::ScanCursor;
use super::{ScanError, OtherScanError};

macro_rules! from_str_scanner {
	($scan_fn:path -> $T:ty as $name:expr) => {
		impl<'a> Scanner<'a, $T> for $T {
			fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<($T, Cur), ScanError> {
				let err = |cur:&Cur| Err(OtherScanError(format!(concat!("expected ",$name,", got `{}`"), cur.tail_str()), cursor.consumed()));

				let end = match $scan_fn(cursor.tail_str()) {
					Some(i) => i,
					None => return err(cursor)
				};

				let s = cursor.str_slice_to(end);
				let cursor = cursor.slice_from(end);

				FromStr::from_str(s)
					.map(ref |i| Ok((i, cursor.clone())))
					.unwrap_or_else(|| err(&cursor))
			}
		}
	};
}

pub trait Scanner<'a, T> {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(Self, Cur), ScanError>;
}

impl<'a> Scanner<'a, bool> for bool {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(bool, Cur), ScanError> {
		cursor.expect_tok("true").map(|c| (true, c))
			.or_else(|_| cursor.expect_tok("false").map(|c| (false, c)))
			.or_else(|_| Err(OtherScanError(format!("expected `true` or `false`, got `{}`", cursor.tail_str()), cursor.consumed())))
	}
}

impl<'a> Scanner<'a, char> for char {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(char, Cur), ScanError> {
		let s = cursor.tail_str();
		if s.len() == 0 {
			Err(OtherScanError("expected character".into_string(), cursor.consumed()))
		} else {
			let ::std::str::CharRange { ch, next } = s.char_range_at(0);
			Ok((ch, cursor.slice_from(next)))
		}
	}
}

impl<'a> Scanner<'a, &'a str> for &'a str {
	fn scan<Cur: ScanCursor<'a>>(cursor: &Cur) -> Result<(&'a str, Cur), ScanError> {
		cursor.pop_token().map(|sc| Ok(sc))
			.unwrap_or_else(|| Err(OtherScanError("expected token".into_string(), cursor.consumed())))
	}
}

from_str_scanner! { scan_float -> f32 as "real number" }
from_str_scanner! { scan_float -> f64 as "real number" }
from_str_scanner! { scan_int -> i8 as "8-bit integer" }
from_str_scanner! { scan_int -> i16 as "16-bit integer" }
from_str_scanner! { scan_int -> i32 as "32-bit integer" }
from_str_scanner! { scan_int -> i64 as "64-bit integer" }
from_str_scanner! { scan_int -> int as "integer" }
from_str_scanner! { scan_uint -> u8 as "8-bit unsigned integer" }
from_str_scanner! { scan_uint -> u16 as "16-bit unsigned integer" }
from_str_scanner! { scan_uint -> u32 as "32-bit unsigned integer" }
from_str_scanner! { scan_uint -> u64 as "64-bit unsigned integer" }
from_str_scanner! { scan_uint -> uint as "unsigned integer" }

fn next_char_at(s: &str, at: uint) -> uint {
	let ::std::str::CharRange { ch: _, next } = s.char_range_at(at);
	next
}

fn scan_float(s: &str) -> Option<uint> {
	enum State {
		Start,
		Whole,
		Suffix,
		ExponentStart,
		Exponent,
	}

	let mut state = Start;

	for (i,c) in s.char_indices() {
		match state {
			Start => {
				assert!(i == 0);
				match c {
					'0'..'9' | '-' => state = Whole,
					_ => return None
				}
			},
			Whole => match c {
				'0'..'9' => (),
				'.' => state = Suffix,
				'e' | 'E' => state = ExponentStart,
				_ => return Some(i)
			},
			Suffix => match c {
				'0'..'9' => (),
				'e' | 'E' => state = ExponentStart,
				_ => return Some(i),
			},
			ExponentStart => match c {
				'+' | '-' | '0'..'9' => state = Exponent,
				_ => return Some(i)
			},
			Exponent => match c {
				'0'..'9' => (),
				_ => return Some(i)
			}
		}
	}

	return Some(s.len())
}

fn scan_uint<'a>(s: &'a str) -> Option<uint> {
	s.char_indices()
		.take_while(|&(_,c)| '0' <= c && c <= '9')
		.map(|(i,_)| Some(next_char_at(s, i)))
		.last().unwrap_or(None)
}

fn scan_int<'a>(s: &'a str) -> Option<uint> {
	if s.len() == 0 { return None }

	let (s, off) = if s.char_at(0) == '-' {
		(s.slice_from(1), 1)
	} else {
		(s, 0)
	};

	scan_uint(s).map(|end| end+off)
}

#[cfg(test)]
mod test {
	use Cursor;
	use ScanError;
	use super::Scanner;
	use tokenizer::WordsAndInts;
	use whitespace::Ignore;

	fn cur<'a>(s: &'a str) -> Cursor<'a, WordsAndInts, Ignore> {
		Cursor::new(s, WordsAndInts, Ignore)
	}

	fn scan_a<'a, T: Scanner<'a, T>>(s: &'a str) -> Result<(T, Cursor<'a, WordsAndInts, Ignore>), ScanError> {
		Scanner::scan(&cur(s))
	}

	#[test]
	fn test_bool() {
		assert!(scan_a::<bool>("").err().is_some());
		assert!(scan_a::<bool>("true").unwrap().0 == true);
		assert!(scan_a::<bool>("false").unwrap().0 == false);
		assert!(scan_a::<bool>("yes").err().is_some());
		assert!(scan_a::<bool>("no").err().is_some());
		assert!(scan_a::<bool>("on").err().is_some());
		assert!(scan_a::<bool>("off").err().is_some());
		assert!(scan_a::<bool>("1").err().is_some());
		assert!(scan_a::<bool>("0").err().is_some());
	}

	#[test]
	fn test_char() {
		assert!(scan_a::<char>("").err().is_some());
		assert!(scan_a::<char>("x").unwrap().0 == 'x');
		assert!(scan_a::<char>("xy").unwrap().0 == 'x');
		assert!(scan_a::<char>("日本語").unwrap().0 == '日');
	}

	#[test]
	fn test_floats() {
		use std::from_str::FromStr;
		use std::num::{Float, Zero};

		fn test<'a, F: Float + Scanner<'a, F> + FromStr>() {
			let f = |v:F| v;
			let fs = |s:&str| -> F FromStr::from_str(s).unwrap();

			assert!(scan_a::<F>("").err().is_some());
			assert!(scan_a::<F>("0").ok().unwrap().0 == f(Zero::zero()));
			assert!(scan_a::<F>("0.0").ok().unwrap().0 == f(Zero::zero()));
			assert!(scan_a::<F>("-0").ok().unwrap().0 == -f(Zero::zero()));
			assert!(scan_a::<F>("1.0").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.00").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.0e0").ok().unwrap().0 == fs("1.0"));
			assert!(scan_a::<F>("1.0e1").ok().unwrap().0 == fs("10.0"));
		}

		test::<f32>();
		test::<f64>();
	}

	#[test]
	fn test_sized_ints() {
		use std::fmt::Show;
		use std::num::{Bounded, Int, Zero};

		fn test<'a, I: Int + Scanner<'a, I> + Show>(check_past: bool) {
			let zero: I = Zero::zero();
			let min: I = Bounded::min_value();
			let max: I = Bounded::max_value();

			assert!(scan_a::<I>("").err().is_some());
			assert!(scan_a::<I>("0").ok().unwrap().0 == zero);
			assert!(scan_a::<I>(format!("{}", min).as_slice()).ok().unwrap().0 == min);
			assert!(scan_a::<I>(format!("{}", max).as_slice()).ok().unwrap().0 == max);

			if check_past {
				let past_min: i64 = (min.to_i64()).unwrap() - 1;
				let past_max: u64 = (max.to_u64()).unwrap() + 1;
				assert!(scan_a::<I>(format!("{}", past_min).as_slice()).err().is_some());
				assert!(scan_a::<I>(format!("{}", past_max).as_slice()).err().is_some());
			}
		}

		test::<i8>(true);
		test::<i16>(true);
		test::<i32>(true);
		test::<i64>(false);
		test::<u8>(true);
		test::<u16>(true);
		test::<u32>(true);
		test::<u64>(false);
	}

	#[test]
	fn test_int() {
		assert!(scan_a::<int>("").err().is_some());
		assert!(scan_a::<int>("0").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("42").ok().unwrap().0 == 42);
		assert!(scan_a::<int>("1_234").ok().unwrap().0 == 1);
		assert!(scan_a::<int>("x").err().is_some());
		assert!(scan_a::<int>("0x").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("42x").ok().unwrap().0 == 42);
		assert!(scan_a::<int>("-").err().is_some());
		assert!(scan_a::<int>("-0").ok().unwrap().0 == 0);
		assert!(scan_a::<int>("-42").ok().unwrap().0 == -42);
		assert!(scan_a::<int>("-1_234").ok().unwrap().0 == -1);
	}

	#[test]
	fn test_uint() {
		assert!(scan_a::<uint>("").err().is_some());
		assert!(scan_a::<uint>("0").ok().unwrap().0 == 0);
		assert!(scan_a::<uint>("42").ok().unwrap().0 == 42);
		assert!(scan_a::<uint>("1_234").ok().unwrap().0 == 1);
		assert!(scan_a::<uint>("x").err().is_some());
		assert!(scan_a::<uint>("0x").ok().unwrap().0 == 0);
		assert!(scan_a::<uint>("42x").ok().unwrap().0 == 42);
		assert!(scan_a::<uint>("-").err().is_some());
		assert!(scan_a::<uint>("-0").err().is_some());
		assert!(scan_a::<uint>("-42").err().is_some());
		assert!(scan_a::<uint>("-1_234").err().is_some());
	}

	#[test]
	fn test_str() {
		assert!(scan_a::<&str>("").err().is_some());
		assert!(scan_a::<&str>("a").ok().unwrap().0 == "a");
		assert!(scan_a::<&str>("a b").ok().unwrap().0 == "a");
		assert!(scan_a::<&str>("abc").ok().unwrap().0 == "abc");
		assert!(scan_a::<&str>("ab-c").ok().unwrap().0 == "ab");
	}
}
<|endoftext|>


<commit_before>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
pub struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

pub type Lexeme = String;

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    pub lexeme: Lexeme,
    /// Position of the first character of the token
    pub position: Position,
}

#[derive(Debug,Clone)]
pub enum ScannerError {
    MissingStringTerminator(Position),
    UnexpectedCharacter(char, Position),
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &str) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        if let Some(c) = next {
            self.current_lexeme.push(c);
            if c == '\n' {
                self.current_position.increment_line();
            } else {
                self.current_position.increment_column();;
            }
        };
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        result
    }

    fn string(&mut self) -> Result<Token, ScannerError> {
        self.advance_while(&|c| c != '"' && c != '\n');
        if !self.advance_if_match('"') {
            return Err(ScannerError::MissingStringTerminator(self.current_position));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Option<Result<TokenWithContext, ScannerError>> {
        let initial_position = self.current_position;
        self.current_lexeme.clear();
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => return None,
        };
        let result = match next_char {
            '(' => Ok(Token::LeftParen),
            ')' => Ok(Token::RightParen),
            '{' => Ok(Token::LeftBrace),
            '}' => Ok(Token::RightBrace),
            ',' => Ok(Token::Comma),
            '.' => Ok(Token::Dot),
            '-' => Ok(Token::Minus),
            '+' => Ok(Token::Plus),
            ';' => Ok(Token::Semicolon),
            '*' => Ok(Token::Star),
            '!' => {
                if self.advance_if_match('=') {
                    Ok(Token::BangEqual)
                } else {
                    Ok(Token::Bang)
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Ok(Token::EqualEqual)
                } else {
                    Ok(Token::Equal)
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Ok(Token::LessEqual)
                } else {
                    Ok(Token::Less)
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Ok(Token::GreaterEqual)
                } else {
                    Ok(Token::Greater)
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Ok(Token::Comment)
                } else {
                    Ok(Token::Slash)
                }
            }
            '"' => self.string(),
            c if is_whitespace(c) => Ok(Token::Whitespace),
            c if is_digit(c) => Ok(self.number()),
            c if is_alpha(c) => Ok(self.identifier()),
            c => Err(ScannerError::UnexpectedCharacter(c, self.current_position)),
        };
        Some(result.map(|token| self.add_context(token, initial_position)))
    }
}

pub fn scan(source: &str) -> (Vec<TokenWithContext>, Vec<ScannerError>) {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = scanner.scan_next() {
        match result {
            Ok(token_with_context) => {
                match token_with_context.token {
                    // Ignoring tokens we don't care about
                    Token::Comment | Token::Whitespace => {}
                    _ => tokens.push(token_with_context),
                };
            }
            Err(error) => errors.push(error),
        }
    }
    (tokens, errors)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let (tokens, _) = scan(&"+");
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let (tokens, _) = scan(&"1+2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn expression_with_whitespaces() {
        let (tokens, _) = scan(&"1 + 2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn assignement_with_comment() {
        let (tokens, _) = scan(&"var a = 1.0; // A comment");
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
    }

    #[test]
    fn multiline_statements() {
        let (tokens, _) = scan(&r#"var a = 1.0;
                              var b = "Hello";"#);
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }

    #[test]
    fn expression_with_bad_character() {
        let (tokens, errors) = scan(&"1 + $2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(errors.len(), 1);
    }

    #[test]
    fn statements_with_unterminated_string() {
        let (tokens, errors) = scan(&r#"var a = "Unterminated;
                              var b = "Hello";"#);
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        // Literal not reported, it wasn't terminated
        // Semicolon got embedded into the string
        assert_eq!(tokens[3].token, Token::Var);
        assert_eq!(tokens[4].token, Token::Identifier("b".into()));
        assert_eq!(tokens[5].token, Token::Equal);
        assert_eq!(tokens[6].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[7].token, Token::Semicolon);
        assert_eq!(errors.len(), 1);
    }

}
<commit_msg>[clippy] identical match expressions<commit_after>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
pub struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

pub type Lexeme = String;

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    pub lexeme: Lexeme,
    /// Position of the first character of the token
    pub position: Position,
}

#[derive(Debug,Clone)]
pub enum ScannerError {
    MissingStringTerminator(Position),
    UnexpectedCharacter(char, Position),
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' | '\r' | '\t' | '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &str) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        if let Some(c) = next {
            self.current_lexeme.push(c);
            if c == '\n' {
                self.current_position.increment_line();
            } else {
                self.current_position.increment_column();;
            }
        };
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        result
    }

    fn string(&mut self) -> Result<Token, ScannerError> {
        self.advance_while(&|c| c != '"' && c != '\n');
        if !self.advance_if_match('"') {
            return Err(ScannerError::MissingStringTerminator(self.current_position));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Option<Result<TokenWithContext, ScannerError>> {
        let initial_position = self.current_position;
        self.current_lexeme.clear();
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => return None,
        };
        let result = match next_char {
            '(' => Ok(Token::LeftParen),
            ')' => Ok(Token::RightParen),
            '{' => Ok(Token::LeftBrace),
            '}' => Ok(Token::RightBrace),
            ',' => Ok(Token::Comma),
            '.' => Ok(Token::Dot),
            '-' => Ok(Token::Minus),
            '+' => Ok(Token::Plus),
            ';' => Ok(Token::Semicolon),
            '*' => Ok(Token::Star),
            '!' => {
                if self.advance_if_match('=') {
                    Ok(Token::BangEqual)
                } else {
                    Ok(Token::Bang)
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Ok(Token::EqualEqual)
                } else {
                    Ok(Token::Equal)
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Ok(Token::LessEqual)
                } else {
                    Ok(Token::Less)
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Ok(Token::GreaterEqual)
                } else {
                    Ok(Token::Greater)
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Ok(Token::Comment)
                } else {
                    Ok(Token::Slash)
                }
            }
            '"' => self.string(),
            c if is_whitespace(c) => Ok(Token::Whitespace),
            c if is_digit(c) => Ok(self.number()),
            c if is_alpha(c) => Ok(self.identifier()),
            c => Err(ScannerError::UnexpectedCharacter(c, self.current_position)),
        };
        Some(result.map(|token| self.add_context(token, initial_position)))
    }
}

pub fn scan(source: &str) -> (Vec<TokenWithContext>, Vec<ScannerError>) {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = scanner.scan_next() {
        match result {
            Ok(token_with_context) => {
                match token_with_context.token {
                    // Ignoring tokens we don't care about
                    Token::Comment | Token::Whitespace => {}
                    _ => tokens.push(token_with_context),
                };
            }
            Err(error) => errors.push(error),
        }
    }
    (tokens, errors)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let (tokens, _) = scan(&"+");
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let (tokens, _) = scan(&"1+2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn expression_with_whitespaces() {
        let (tokens, _) = scan(&"1 + 2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn assignement_with_comment() {
        let (tokens, _) = scan(&"var a = 1.0; // A comment");
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
    }

    #[test]
    fn multiline_statements() {
        let (tokens, _) = scan(&r#"var a = 1.0;
                              var b = "Hello";"#);
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }

    #[test]
    fn expression_with_bad_character() {
        let (tokens, errors) = scan(&"1 + $2");
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(errors.len(), 1);
    }

    #[test]
    fn statements_with_unterminated_string() {
        let (tokens, errors) = scan(&r#"var a = "Unterminated;
                              var b = "Hello";"#);
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        // Literal not reported, it wasn't terminated
        // Semicolon got embedded into the string
        assert_eq!(tokens[3].token, Token::Var);
        assert_eq!(tokens[4].token, Token::Identifier("b".into()));
        assert_eq!(tokens[5].token, Token::Equal);
        assert_eq!(tokens[6].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[7].token, Token::Semicolon);
        assert_eq!(errors.len(), 1);
    }

}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern fn foopy() {}

const f: *u8 = foopy;
const s: S = S { f: foopy };

struct S {
    f: *u8
}

fn main() {
}<commit_msg>Add test case for change in 039bfe<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern fn foopy() {}

const f: *u8 = foopy;
const s: S = S { f: foopy };

struct S {
    f: *u8
}

fn main() {
    assert foopy == f;
    assert f == s.f;
}<|endoftext|>


<commit_before>//! Serialization support for the `application/x-www-form-urlencoded` format.

mod key;
mod pair;
mod value;

use serde::ser;
use std::borrow::Cow;
use std::error;
use std::fmt;
use std::str;
use url::form_urlencoded::Serializer as UrlEncodedSerializer;
use url::form_urlencoded::Target as UrlEncodedTarget;

/// Serializes a value into a `application/x-wwww-url-encoded` `String` buffer.
///
/// ```
/// let meal = &[
///     ("bread", "baguette"),
///     ("cheese", "comté"),
///     ("meat", "ham"),
///     ("fat", "butter"),
/// ];
///
/// assert_eq!(
///     serde_urlencoded::to_string(meal),
///     Ok("bread=baguette&cheese=comt%C3%A9&meat=ham&fat=butter".to_owned()));
/// ```
pub fn to_string<T: ser::Serialize>(input: &T) -> Result<String, Error> {
    let mut output = String::new();
    {
        let mut urlencoder = UrlEncodedSerializer::new(&mut output);
        try!(input.serialize(&mut Serializer::new(&mut urlencoder)));
    }
    Ok(output)
}

/// A serializer for the `application/x-www-form-urlencoded` format.
///
/// * Supported top-level inputs are structs, maps and sequences of pairs,
///   with or without a given length.
///
/// * Supported keys and values are integers, bytes (if convertible to strings),
///   unit structs and unit variants.
///
/// * Newtype structs defer to their inner values.
pub struct Serializer<'output, T: 'output + UrlEncodedTarget> {
    urlencoder: &'output mut UrlEncodedSerializer<T>
}

impl<'output, T: 'output + UrlEncodedTarget> Serializer<'output, T> {
    /// Returns a new `Serializer`.
    pub fn new(urlencoder: &'output mut UrlEncodedSerializer<T>) -> Self {
        Serializer { urlencoder: urlencoder }
    }
}

/// Errors returned during serializing to `application/x-www-form-urlencoded`.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Error {
    Custom(Cow<'static, str>),
    InvalidValue(Cow<'static, str>),
    Utf8(str::Utf8Error),
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            Error::Custom(ref msg) => msg.fmt(f),
            Error::InvalidValue(ref msg) => write!(f, "invalid value: {}", msg),
            Error::Utf8(ref err) => write!(f, "invalid UTF-8: {}", err),
        }
    }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            Error::Custom(ref msg) => msg,
            Error::InvalidValue(ref msg) => msg,
            Error::Utf8(ref err) => error::Error::description(err),
        }
    }

    /// The lower-level cause of this error, in the case of a `Utf8` error.
    fn cause(&self) -> Option<&error::Error> {
        match *self {
            Error::Custom(_) | Error::InvalidValue(_) => None,
            Error::Utf8(ref err) => Some(err),
        }
    }
}

impl ser::Error for Error {
    fn custom<T: Into<String>>(msg: T) -> Self {
        Error::Custom(msg.into().into())
    }

    fn invalid_value(msg: &str) -> Self {
        Error::InvalidValue(String::from(msg).into())
    }
}

/// State used when serializing sequences.
pub struct SeqState {
    _state: (),
}

/// State used when serializing tuples.
pub struct TupleState {
    _state: (),
}

/// State used when serializing tuple structs.
pub struct TupleStructState {
    _state: (),
}

/// State used when serializing tuple variants.
pub struct TupleVariantState {
    _state: (),
}

/// State used when serializing maps.
pub struct MapState {
    key: Option<Cow<'static, str>>
}

/// State used when serializing structs.
pub struct StructState {
    _state: (),
}

/// State used when serializing struct variants.
pub struct StructVariantState {
    _state: (),
}

impl<'output, Target> ser::Serializer for Serializer<'output, Target>
    where Target: 'output + UrlEncodedTarget
{
    type Error = Error;

    /// State used when serializing sequences.
    type SeqState = SeqState;

    /// State used when serializing tuples.
    type TupleState = TupleState;

    /// State used when serializing tuple structs.
    type TupleStructState = TupleStructState;

    /// State used when serializing tuple variants.
    type TupleVariantState = TupleVariantState;

    /// State used when serializing maps.
    type MapState = MapState;

    /// State used when serializing structs.
    type StructState = StructState;

    /// State used when serializing struct variants.
    type StructVariantState = StructVariantState;

    /// Returns an error.
    fn serialize_bool(&mut self, _v: bool) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_isize(&mut self, _v: isize) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i8(&mut self, _v: i8) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i16(&mut self, _v: i16) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i32(&mut self, _v: i32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i64(&mut self, _v: i64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_usize(&mut self, _v: usize) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u8(&mut self, _v: u8) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u16(&mut self, _v: u16) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u32(&mut self, _v: u32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u64(&mut self, _v: u64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_f32(&mut self, _v: f32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_f64(&mut self, _v: f64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_char(&mut self, _v: char) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_str(&mut self, _value: &str) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_bytes(&mut self, _value: &[u8]) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit(&mut self) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit_struct(
            &mut self, _name: &'static str)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Serializes the inner value, ignoring the newtype name.
    fn serialize_newtype_struct<T>(
            &mut self, _name: &'static str, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        value.serialize(self)
    }

    /// Returns an error.
    fn serialize_newtype_variant<T>(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_none(&mut self) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_some<T>(&mut self, _value: T) -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Begins to serialize a sequence, given length (if any) is ignored.
    fn serialize_seq(
            &mut self, _len: Option<usize>)
            -> Result<SeqState, Error> {
        Ok(SeqState { _state: () })
    }

    /// Serializes a sequence element.
    fn serialize_seq_elt<T>(
            &mut self, _state: &mut SeqState, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        value.serialize(&mut pair::PairSerializer::new(self.urlencoder))
    }

    /// Finishes serializing a sequence.
    fn serialize_seq_end(&mut self, _state: SeqState) -> Result<(), Error> {
        Ok(())
    }

    /// Begins to serialize a sequence, given length is ignored.
    fn serialize_seq_fixed_size(
            &mut self, _length: usize)
            -> Result<SeqState, Error> {
        Ok(SeqState { _state: () })
    }

    /// Returns an error.
    fn serialize_tuple(&mut self, _len: usize) -> Result<TupleState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_elt<T>(
            &mut self, _state: &mut TupleState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_end(&mut self, _state: TupleState) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct(
            &mut self, _name: &'static str, _len: usize)
            -> Result<TupleStructState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct_elt<T>(
            &mut self, _state: &mut TupleStructState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct_end(
            &mut self, _state: TupleStructState)
            -> Result<(), Error>
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _len: usize)
            -> Result<TupleVariantState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant_elt<T>(
            &mut self, _state: &mut TupleVariantState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant_end(
            &mut self, _state: TupleVariantState)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Begins to serialize a map, given length (if any) is ignored.
    fn serialize_map(
            &mut self, _len: Option<usize>)
            -> Result<MapState, Error> {
        Ok(MapState { key: None })
    }

    /// Serializes a map key.
    fn serialize_map_key<T>(
            &mut self, state: &mut MapState, key: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        key.serialize(&mut key::MapKeySerializer::new(&mut state.key))
    }

    /// Serializes a map value.
    fn serialize_map_value<T>(
            &mut self, state: &mut MapState, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        let mut value_serializer =
            try!(value::ValueSerializer::new(&mut state.key, self.urlencoder));
        value.serialize(&mut value_serializer)
    }

    /// Finishes serializing a map.
    fn serialize_map_end(&mut self, _state: MapState) -> Result<(), Error> {
        Ok(())
    }

    /// Begins to serialize a struct, given length is ignored.
    fn serialize_struct(
            &mut self, _name: &'static str, _len: usize)
            -> Result<StructState, Error> {
        Ok(StructState { _state: () })
    }

    /// Serializes a struct element.
    fn serialize_struct_elt<T>(
            &mut self,
            _state: &mut StructState,
            key: &'static str,
            value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        let mut key = Some(key.into());
        let mut value_serializer =
            value::ValueSerializer::new(&mut key, self.urlencoder).unwrap();
        value.serialize(&mut value_serializer)
    }

    /// Finishes serializing a struct.
    fn serialize_struct_end(&mut self, _state: StructState)
                            -> Result<(), Error> {
        Ok(())
    }

    /// Returns an error.
    fn serialize_struct_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _len: usize)
            -> Result<StructVariantState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_struct_variant_elt<T>(
            &mut self,
            _state: &mut StructVariantState,
            _key: &'static str,
            _value: T)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_struct_variant_end(
            &mut self, _state: StructVariantState)
            -> Result<(), Error> {
        Err(Error::top_level())
    }
}

impl Error {
    fn top_level() -> Self {
        Error::Custom(
            "top-level serializer supports only maps and structs".into())
    }
}
<commit_msg>Support serializing top-level Option values<commit_after>//! Serialization support for the `application/x-www-form-urlencoded` format.

mod key;
mod pair;
mod value;

use serde::ser;
use std::borrow::Cow;
use std::error;
use std::fmt;
use std::str;
use url::form_urlencoded::Serializer as UrlEncodedSerializer;
use url::form_urlencoded::Target as UrlEncodedTarget;

/// Serializes a value into a `application/x-wwww-url-encoded` `String` buffer.
///
/// ```
/// let meal = &[
///     ("bread", "baguette"),
///     ("cheese", "comté"),
///     ("meat", "ham"),
///     ("fat", "butter"),
/// ];
///
/// assert_eq!(
///     serde_urlencoded::to_string(meal),
///     Ok("bread=baguette&cheese=comt%C3%A9&meat=ham&fat=butter".to_owned()));
/// ```
pub fn to_string<T: ser::Serialize>(input: &T) -> Result<String, Error> {
    let mut output = String::new();
    {
        let mut urlencoder = UrlEncodedSerializer::new(&mut output);
        try!(input.serialize(&mut Serializer::new(&mut urlencoder)));
    }
    Ok(output)
}

/// A serializer for the `application/x-www-form-urlencoded` format.
///
/// * Supported top-level inputs are structs, maps and sequences of pairs,
///   with or without a given length.
///
/// * Supported keys and values are integers, bytes (if convertible to strings),
///   unit structs and unit variants.
///
/// * Newtype structs defer to their inner values.
pub struct Serializer<'output, T: 'output + UrlEncodedTarget> {
    urlencoder: &'output mut UrlEncodedSerializer<T>
}

impl<'output, T: 'output + UrlEncodedTarget> Serializer<'output, T> {
    /// Returns a new `Serializer`.
    pub fn new(urlencoder: &'output mut UrlEncodedSerializer<T>) -> Self {
        Serializer { urlencoder: urlencoder }
    }
}

/// Errors returned during serializing to `application/x-www-form-urlencoded`.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Error {
    Custom(Cow<'static, str>),
    InvalidValue(Cow<'static, str>),
    Utf8(str::Utf8Error),
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            Error::Custom(ref msg) => msg.fmt(f),
            Error::InvalidValue(ref msg) => write!(f, "invalid value: {}", msg),
            Error::Utf8(ref err) => write!(f, "invalid UTF-8: {}", err),
        }
    }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            Error::Custom(ref msg) => msg,
            Error::InvalidValue(ref msg) => msg,
            Error::Utf8(ref err) => error::Error::description(err),
        }
    }

    /// The lower-level cause of this error, in the case of a `Utf8` error.
    fn cause(&self) -> Option<&error::Error> {
        match *self {
            Error::Custom(_) | Error::InvalidValue(_) => None,
            Error::Utf8(ref err) => Some(err),
        }
    }
}

impl ser::Error for Error {
    fn custom<T: Into<String>>(msg: T) -> Self {
        Error::Custom(msg.into().into())
    }

    fn invalid_value(msg: &str) -> Self {
        Error::InvalidValue(String::from(msg).into())
    }
}

/// State used when serializing sequences.
pub struct SeqState {
    _state: (),
}

/// State used when serializing tuples.
pub struct TupleState {
    _state: (),
}

/// State used when serializing tuple structs.
pub struct TupleStructState {
    _state: (),
}

/// State used when serializing tuple variants.
pub struct TupleVariantState {
    _state: (),
}

/// State used when serializing maps.
pub struct MapState {
    key: Option<Cow<'static, str>>
}

/// State used when serializing structs.
pub struct StructState {
    _state: (),
}

/// State used when serializing struct variants.
pub struct StructVariantState {
    _state: (),
}

impl<'output, Target> ser::Serializer for Serializer<'output, Target>
    where Target: 'output + UrlEncodedTarget
{
    type Error = Error;

    /// State used when serializing sequences.
    type SeqState = SeqState;

    /// State used when serializing tuples.
    type TupleState = TupleState;

    /// State used when serializing tuple structs.
    type TupleStructState = TupleStructState;

    /// State used when serializing tuple variants.
    type TupleVariantState = TupleVariantState;

    /// State used when serializing maps.
    type MapState = MapState;

    /// State used when serializing structs.
    type StructState = StructState;

    /// State used when serializing struct variants.
    type StructVariantState = StructVariantState;

    /// Returns an error.
    fn serialize_bool(&mut self, _v: bool) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_isize(&mut self, _v: isize) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i8(&mut self, _v: i8) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i16(&mut self, _v: i16) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i32(&mut self, _v: i32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_i64(&mut self, _v: i64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_usize(&mut self, _v: usize) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u8(&mut self, _v: u8) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u16(&mut self, _v: u16) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u32(&mut self, _v: u32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_u64(&mut self, _v: u64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_f32(&mut self, _v: f32) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_f64(&mut self, _v: f64) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_char(&mut self, _v: char) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_str(&mut self, _value: &str) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_bytes(&mut self, _value: &[u8]) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit(&mut self) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit_struct(
            &mut self, _name: &'static str)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_unit_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Serializes the inner value, ignoring the newtype name.
    fn serialize_newtype_struct<T>(
            &mut self, _name: &'static str, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        value.serialize(self)
    }

    /// Returns an error.
    fn serialize_newtype_variant<T>(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_none(&mut self) -> Result<(), Error> {
        Ok(())
    }

    /// Returns an error.
    fn serialize_some<T>(&mut self, value: T) -> Result<(), Error>
        where T: ser::Serialize
    {
        value.serialize(self)
    }

    /// Begins to serialize a sequence, given length (if any) is ignored.
    fn serialize_seq(
            &mut self, _len: Option<usize>)
            -> Result<SeqState, Error> {
        Ok(SeqState { _state: () })
    }

    /// Serializes a sequence element.
    fn serialize_seq_elt<T>(
            &mut self, _state: &mut SeqState, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        value.serialize(&mut pair::PairSerializer::new(self.urlencoder))
    }

    /// Finishes serializing a sequence.
    fn serialize_seq_end(&mut self, _state: SeqState) -> Result<(), Error> {
        Ok(())
    }

    /// Begins to serialize a sequence, given length is ignored.
    fn serialize_seq_fixed_size(
            &mut self, _length: usize)
            -> Result<SeqState, Error> {
        Ok(SeqState { _state: () })
    }

    /// Returns an error.
    fn serialize_tuple(&mut self, _len: usize) -> Result<TupleState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_elt<T>(
            &mut self, _state: &mut TupleState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_end(&mut self, _state: TupleState) -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct(
            &mut self, _name: &'static str, _len: usize)
            -> Result<TupleStructState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct_elt<T>(
            &mut self, _state: &mut TupleStructState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_struct_end(
            &mut self, _state: TupleStructState)
            -> Result<(), Error>
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _len: usize)
            -> Result<TupleVariantState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant_elt<T>(
            &mut self, _state: &mut TupleVariantState, _value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_tuple_variant_end(
            &mut self, _state: TupleVariantState)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Begins to serialize a map, given length (if any) is ignored.
    fn serialize_map(
            &mut self, _len: Option<usize>)
            -> Result<MapState, Error> {
        Ok(MapState { key: None })
    }

    /// Serializes a map key.
    fn serialize_map_key<T>(
            &mut self, state: &mut MapState, key: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        key.serialize(&mut key::MapKeySerializer::new(&mut state.key))
    }

    /// Serializes a map value.
    fn serialize_map_value<T>(
            &mut self, state: &mut MapState, value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        let mut value_serializer =
            try!(value::ValueSerializer::new(&mut state.key, self.urlencoder));
        value.serialize(&mut value_serializer)
    }

    /// Finishes serializing a map.
    fn serialize_map_end(&mut self, _state: MapState) -> Result<(), Error> {
        Ok(())
    }

    /// Begins to serialize a struct, given length is ignored.
    fn serialize_struct(
            &mut self, _name: &'static str, _len: usize)
            -> Result<StructState, Error> {
        Ok(StructState { _state: () })
    }

    /// Serializes a struct element.
    fn serialize_struct_elt<T>(
            &mut self,
            _state: &mut StructState,
            key: &'static str,
            value: T)
            -> Result<(), Error>
        where T: ser::Serialize
    {
        let mut key = Some(key.into());
        let mut value_serializer =
            value::ValueSerializer::new(&mut key, self.urlencoder).unwrap();
        value.serialize(&mut value_serializer)
    }

    /// Finishes serializing a struct.
    fn serialize_struct_end(&mut self, _state: StructState)
                            -> Result<(), Error> {
        Ok(())
    }

    /// Returns an error.
    fn serialize_struct_variant(
            &mut self,
            _name: &'static str,
            _variant_index: usize,
            _variant: &'static str,
            _len: usize)
            -> Result<StructVariantState, Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_struct_variant_elt<T>(
            &mut self,
            _state: &mut StructVariantState,
            _key: &'static str,
            _value: T)
            -> Result<(), Error> {
        Err(Error::top_level())
    }

    /// Returns an error.
    fn serialize_struct_variant_end(
            &mut self, _state: StructVariantState)
            -> Result<(), Error> {
        Err(Error::top_level())
    }
}

impl Error {
    fn top_level() -> Self {
        Error::Custom(
            "top-level serializer supports only maps and structs".into())
    }
}
<|endoftext|>