<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);

                    // Return no match
                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn can_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<commit_msg>Add a simpler test for the problem that we’re seeing<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);

                    // Return no match
                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn can_distinguish_simple_tokens_with_zero_repeats() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(0), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(0), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn can_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<|endoftext|>


<commit_before>use std::io::{Read, ReadExt, Chars, CharsError};
use std::borrow::Cow;
use std::fmt;
use std::num;
use std::char;

use error::{ParserError, ParserErrorKind};

/// Token types
#[derive(PartialEq)]
pub enum Token {
    /// `(`
    OpenParen,
    /// `)`
    CloseParen,
    /// `#(`
    OpenVectorParen,
    /// `#vu8(`
    OpenBytesParen,
    /// `.`
    Dot,
    /// `'`
    Quote,
    Identifier(Cow<'static, str>),
    /// `#t`
    True,
    /// `#f`
    False,
    /// `#\<String>`
    Character(String),
    String(String),
    Numeric(String),
    /// End of character stream
    EOF
}

impl fmt::Debug for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::OpenParen => write!(f, "OpenParen"),
            Token::OpenVectorParen => write!(f, "OpenVectorParen"),
            Token::OpenBytesParen => write!(f, "OpenBytesParen"),
            Token::CloseParen => write!(f, "CloseParen"),
            Token::Dot => write!(f, "Dot"),
            Token::Quote => write!(f, "Quote"),
            Token::Identifier(ref name) => write!(f, "Identifier({})", name),
            Token::True => write!(f, "#t"),
            Token::False => write!(f, "#f"),
            Token::Character(ref name) => write!(f, "#\\{}", name),
            Token::Numeric(ref rep) => rep.fmt(f),
            Token::String(ref rep) => write!(f, "{:?}", rep),
            Token::EOF => write!(f, "EOF"),
        }
    }
}

/// TokenWrapper provides positional information to each token
pub struct TokenWrapper {
    pub line: usize,
    pub column: usize,
    pub token: Token
}

fn wrap(line: usize, column: usize, t: Token) -> TokenWrapper {
    TokenWrapper {
        line: line,
        column: column,
        token: t
    }
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ' => true,
        _ => false
    }
}

fn is_initial(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '!' | '$' | '%' | '&' | '*' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '_' | '~' => true,
        _ => false
    }
}

fn is_subsequent(c: char) -> bool {
    if is_initial(c) {
        true
    } else {
        match c {
            '0'...'9' | '+' | '-' | '.' | '@' => true,
            _ => false
        }
    }
}

fn is_delim(c: char) -> bool {
    match c {
        '(' | ')' | '[' | ']' | '"' | ';' => true,
        _ => c.is_whitespace()
    }
}

/// Lexer transforms character stream into a token stream
pub struct Lexer<R> {
    line: usize,
    column: usize,
    stream: Chars<R>,
    lookahead_buf: Option<char>,
}

macro_rules! try_consume {
    ($this:ident) => (
        match $this.consume() {
            Some(Ok(c)) => c,
            Some(Err(e)) => return Err($this.make_error(ParserErrorKind::UnderlyingError(e))),
            None => return Err($this.make_error(ParserErrorKind::UnexpectedEOF))
        }
    )
}

impl <R: Read + Sized> Lexer<R> {
    /// Creates new Lexer from io::Buffer
    pub fn new(stream: R) -> Lexer<R> {
        Lexer {
            line: 1,
            column: 1,
            stream: stream.chars(),
            lookahead_buf: None,
        }
    }

    /// return next token
    pub fn lex_token(&mut self) -> Result<TokenWrapper, ParserError> {
        try!(self.consume_whitespace());

        let line = self.line;
        let col = self.column;
        let c = match self.consume() {
            Some(Ok(c)) => c,
            None => return Ok(wrap(line, col, Token::EOF)),
            Some(Err(e)) =>
                return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
        };

        let end_of_token = try!(self.is_end_of_token());

        if is_initial(c) {
            let mut init = String::new();
            init.push(c);
            self.lex_ident(init).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s))))
        } else if c == '+' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("+"))))
            } else {
                self.lex_numeric("+".to_string()).map(|s| wrap(line, col, Token::Numeric(s)))
            }
        } else if c == '-' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
            } else {
                match self.lookahead() {
                    Some(Ok('>')) => self.lex_ident("-".to_string()).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s)))),
                    Some(Ok(_)) => self.lex_numeric("-".to_string()).map(|s| wrap(line, col, Token::Numeric(s))),
                    Some(Err(e)) => Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
                    None => Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
                }
            }
        } else if c == '(' {
            Ok(wrap(line, col, Token::OpenParen))
        } else if c == ')' {
            Ok(wrap(line, col, Token::CloseParen))
        } else if c == '.' && end_of_token {
            Ok(wrap(line, col, Token::Dot))
        } else if c == '\'' {
            Ok(wrap(line, col, Token::Quote))
        } else if c == '#' {
            let c0 = try_consume!(self);
            match c0 {
                't' | 'T' => Ok(wrap(line, col, Token::True)),
                'f' | 'F' => Ok(wrap(line, col, Token::False)),
                'b' | 'B' | 'o' | 'O' | 'd' | 'D' | 'x' | 'X' | 'i' | 'I' | 'e' | 'E' => {
                    let s = format!("{}{}", c, c0);
                    self.lex_numeric(s).map(|s| wrap(line, col, Token::Numeric(s)))
                },
                'v' | 'u' => {
                    let rest_prefix = try!(self.read_while(|c| !is_delim(c)));
                    let delim = try_consume!(self);
                    let prefix = format!("{}{}{}", c0, rest_prefix, delim);
                    if prefix.as_slice() == "vu8(" || prefix.as_slice() == "u8(" {
                        Ok(wrap(line, col, Token::OpenBytesParen))
                    } else {
                        Err(self.make_error(ParserErrorKind::InvalidToken(prefix)))
                    }
                },
                '\\' => self.lex_char().map(|s| wrap(line, col, Token::Character(s))),
                '(' => Ok(wrap(line, col, Token::OpenVectorParen)),
                _ => Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
            }
        } else if c == '"' {
            self.lex_string().map(|s| wrap(line, col, Token::String(s)))
        } else if c.is_numeric() {
            let s = format!("{}", c);
            self.lex_numeric(s).map(|s| wrap(line, col, Token::Numeric(s)))
        } else {
            Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
        }
    }

    fn is_end_of_token(&mut self) -> Result<bool, ParserError> {
        match self.lookahead() {
            Some(Ok(c)) => Ok(is_whitespace(c) || is_delim(c)),
            Some(Err(e)) => Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
            None => Ok(true)
        }
    }

    fn lex_ident(&mut self, initial: String) -> Result<String, ParserError> {
        let mut s = initial;
        let sub = try!(self.read_while(is_subsequent));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_char(&mut self) -> Result<String, ParserError> {
        let c = try_consume!(self);

        let mut s = String::new();
        s.push(c);
        let sub = try!(self.read_while(|c| c.is_alphanumeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_string(&mut self) -> Result<String, ParserError> {
        let mut s = String::new();
        loop {
            match try_consume!(self) {
                '"' => return Ok(s),
                '\\' => match try_consume!(self) {
                    'a' => s.push('\x07'),
                    'b' => s.push('\x08'),
                    't' => s.push('\t'),
                    'n' => s.push('\n'),
                    'v' => s.push('\x0b'),
                    'f' => s.push('\x0c'),
                    'r' => s.push('\r'),
                    '"' => s.push('"'),
                    '\\' => s.push('\\'),
                    'x' => {
                        let mut hex_str = String::new();
                        loop {
                            match try_consume!(self) {
                                ';' => break,
                                c => hex_str.push(c)
                            }
                        }
                        if hex_str.len() == 0 {
                            return Err(self.make_error(ParserErrorKind::InvalidStringEscape(hex_str)))
                        }
                        let code = match num::from_str_radix(hex_str.as_slice(), 16) {
                            Ok(n) => n,
                            Err(_) => return Err(self.make_error(ParserErrorKind::InvalidStringEscape(hex_str)))
                        };
                        match char::from_u32(code) {
                            Some(c) => s.push(c),
                            None => return Err(self.make_error(ParserErrorKind::InvalidUnicodeRange(code)))
                        }
                    },
                    c =>
                        if c.is_whitespace() {
                            try!(self.read_while(|c| c != '\n' && c.is_whitespace()));
                            if c != '\n' {
                                if let Some(Ok('\n')) = self.consume() {
                                    try!(self.read_while(|c| c != '\n' && c.is_whitespace()));
                                } else {
                                    return Err(self.make_error(ParserErrorKind::InvalidStringLiteral))
                                }
                            }
                        } else {
                            let mut s = String::new();
                            s.push(c);
                            return Err(self.make_error(ParserErrorKind::InvalidStringEscape(s)))
                        }
                },
                c => s.push(c),
            }
        }
    }

    fn lex_numeric(&mut self, init: String) -> Result<String, ParserError> {
        let mut s = init;
        let sub = try!(self.read_while(|c| !is_delim(c)));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn make_error(&self, kind: ParserErrorKind) -> ParserError {
        ParserError {
            line: self.line,
            column: self.column,
            kind: kind
        }
    }

    fn lookahead(&mut self) -> Option<Result<char, CharsError>> {
        match self.lookahead_buf {
            Some(c) => Some(Ok(c)),
            None => {
                let c = self.stream.next();
                if let Some(Ok(ch)) = c {
                    self.lookahead_buf = Some(ch);
                }
                c
            }
        }
    }

    fn advance(&mut self, c: char) {
        if c == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
    }

    fn read_while<F>(&mut self, f: F) -> Result<String, ParserError> where
        F: Fn(char) -> bool
    {
        let mut s = match self.lookahead_buf {
            None => String::new(),
            Some(c) => if f(c) {
                self.lookahead_buf = None;
                self.advance(c);
                let mut s = String::new();
                s.push(c);
                s
            } else {
                return Ok(String::new());
            }
        };

        loop {
            match self.stream.next() {
                Some(Ok(c)) => if f(c) {
                    self.advance(c);
                    s.push(c);
                } else {
                    self.lookahead_buf = Some(c);
                    return Ok(s);
                },
                Some(Err(e)) =>
                    return Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
                None => return Ok(s)
            }
        }
    }

    fn consume(&mut self) -> Option<Result<char, CharsError>> {
        let c = match self.lookahead_buf {
            Some(c) => {
                self.lookahead_buf = None;
                Some(Ok(c))
            },
            None => self.stream.next()
        };

        if let Some(Ok(ch)) = c {
            self.advance(ch);
        }

        c
    }

    fn consume_whitespace(&mut self) -> Result<bool, ParserError> {
        let mut consumed = false;
        loop {
            let whitespace = try!(self.read_while(is_whitespace));
            consumed = consumed || whitespace.len() > 0;
            match self.lookahead() {
                Some(Ok(';')) => {
                    consumed = true;
                    try!(self.read_while(|c| c != '\n'));
                    if self.lookahead_buf.is_some() {
                        self.lookahead_buf = None
                    }
                },
                Some(Ok(_)) | None => return Ok(consumed),
                Some(Err(e)) =>
                    return Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
            }
        }
    }
}
<commit_msg>Rust changes: ReadExt removed<commit_after>use std::io::{Read, Chars, CharsError};
use std::borrow::Cow;
use std::fmt;
use std::num;
use std::char;

use error::{ParserError, ParserErrorKind};

/// Token types
#[derive(PartialEq)]
pub enum Token {
    /// `(`
    OpenParen,
    /// `)`
    CloseParen,
    /// `#(`
    OpenVectorParen,
    /// `#vu8(`
    OpenBytesParen,
    /// `.`
    Dot,
    /// `'`
    Quote,
    Identifier(Cow<'static, str>),
    /// `#t`
    True,
    /// `#f`
    False,
    /// `#\<String>`
    Character(String),
    String(String),
    Numeric(String),
    /// End of character stream
    EOF
}

impl fmt::Debug for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::OpenParen => write!(f, "OpenParen"),
            Token::OpenVectorParen => write!(f, "OpenVectorParen"),
            Token::OpenBytesParen => write!(f, "OpenBytesParen"),
            Token::CloseParen => write!(f, "CloseParen"),
            Token::Dot => write!(f, "Dot"),
            Token::Quote => write!(f, "Quote"),
            Token::Identifier(ref name) => write!(f, "Identifier({})", name),
            Token::True => write!(f, "#t"),
            Token::False => write!(f, "#f"),
            Token::Character(ref name) => write!(f, "#\\{}", name),
            Token::Numeric(ref rep) => rep.fmt(f),
            Token::String(ref rep) => write!(f, "{:?}", rep),
            Token::EOF => write!(f, "EOF"),
        }
    }
}

/// TokenWrapper provides positional information to each token
pub struct TokenWrapper {
    pub line: usize,
    pub column: usize,
    pub token: Token
}

fn wrap(line: usize, column: usize, t: Token) -> TokenWrapper {
    TokenWrapper {
        line: line,
        column: column,
        token: t
    }
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ' => true,
        _ => false
    }
}

fn is_initial(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '!' | '$' | '%' | '&' | '*' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '_' | '~' => true,
        _ => false
    }
}

fn is_subsequent(c: char) -> bool {
    if is_initial(c) {
        true
    } else {
        match c {
            '0'...'9' | '+' | '-' | '.' | '@' => true,
            _ => false
        }
    }
}

fn is_delim(c: char) -> bool {
    match c {
        '(' | ')' | '[' | ']' | '"' | ';' => true,
        _ => c.is_whitespace()
    }
}

/// Lexer transforms character stream into a token stream
pub struct Lexer<R> {
    line: usize,
    column: usize,
    stream: Chars<R>,
    lookahead_buf: Option<char>,
}

macro_rules! try_consume {
    ($this:ident) => (
        match $this.consume() {
            Some(Ok(c)) => c,
            Some(Err(e)) => return Err($this.make_error(ParserErrorKind::UnderlyingError(e))),
            None => return Err($this.make_error(ParserErrorKind::UnexpectedEOF))
        }
    )
}

impl <R: Read + Sized> Lexer<R> {
    /// Creates new Lexer from io::Buffer
    pub fn new(stream: R) -> Lexer<R> {
        Lexer {
            line: 1,
            column: 1,
            stream: stream.chars(),
            lookahead_buf: None,
        }
    }

    /// return next token
    pub fn lex_token(&mut self) -> Result<TokenWrapper, ParserError> {
        try!(self.consume_whitespace());

        let line = self.line;
        let col = self.column;
        let c = match self.consume() {
            Some(Ok(c)) => c,
            None => return Ok(wrap(line, col, Token::EOF)),
            Some(Err(e)) =>
                return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
        };

        let end_of_token = try!(self.is_end_of_token());

        if is_initial(c) {
            let mut init = String::new();
            init.push(c);
            self.lex_ident(init).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s))))
        } else if c == '+' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("+"))))
            } else {
                self.lex_numeric("+".to_string()).map(|s| wrap(line, col, Token::Numeric(s)))
            }
        } else if c == '-' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
            } else {
                match self.lookahead() {
                    Some(Ok('>')) => self.lex_ident("-".to_string()).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s)))),
                    Some(Ok(_)) => self.lex_numeric("-".to_string()).map(|s| wrap(line, col, Token::Numeric(s))),
                    Some(Err(e)) => Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
                    None => Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
                }
            }
        } else if c == '(' {
            Ok(wrap(line, col, Token::OpenParen))
        } else if c == ')' {
            Ok(wrap(line, col, Token::CloseParen))
        } else if c == '.' && end_of_token {
            Ok(wrap(line, col, Token::Dot))
        } else if c == '\'' {
            Ok(wrap(line, col, Token::Quote))
        } else if c == '#' {
            let c0 = try_consume!(self);
            match c0 {
                't' | 'T' => Ok(wrap(line, col, Token::True)),
                'f' | 'F' => Ok(wrap(line, col, Token::False)),
                'b' | 'B' | 'o' | 'O' | 'd' | 'D' | 'x' | 'X' | 'i' | 'I' | 'e' | 'E' => {
                    let s = format!("{}{}", c, c0);
                    self.lex_numeric(s).map(|s| wrap(line, col, Token::Numeric(s)))
                },
                'v' | 'u' => {
                    let rest_prefix = try!(self.read_while(|c| !is_delim(c)));
                    let delim = try_consume!(self);
                    let prefix = format!("{}{}{}", c0, rest_prefix, delim);
                    if prefix.as_slice() == "vu8(" || prefix.as_slice() == "u8(" {
                        Ok(wrap(line, col, Token::OpenBytesParen))
                    } else {
                        Err(self.make_error(ParserErrorKind::InvalidToken(prefix)))
                    }
                },
                '\\' => self.lex_char().map(|s| wrap(line, col, Token::Character(s))),
                '(' => Ok(wrap(line, col, Token::OpenVectorParen)),
                _ => Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
            }
        } else if c == '"' {
            self.lex_string().map(|s| wrap(line, col, Token::String(s)))
        } else if c.is_numeric() {
            let s = format!("{}", c);
            self.lex_numeric(s).map(|s| wrap(line, col, Token::Numeric(s)))
        } else {
            Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
        }
    }

    fn is_end_of_token(&mut self) -> Result<bool, ParserError> {
        match self.lookahead() {
            Some(Ok(c)) => Ok(is_whitespace(c) || is_delim(c)),
            Some(Err(e)) => Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
            None => Ok(true)
        }
    }

    fn lex_ident(&mut self, initial: String) -> Result<String, ParserError> {
        let mut s = initial;
        let sub = try!(self.read_while(is_subsequent));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_char(&mut self) -> Result<String, ParserError> {
        let c = try_consume!(self);

        let mut s = String::new();
        s.push(c);
        let sub = try!(self.read_while(|c| c.is_alphanumeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_string(&mut self) -> Result<String, ParserError> {
        let mut s = String::new();
        loop {
            match try_consume!(self) {
                '"' => return Ok(s),
                '\\' => match try_consume!(self) {
                    'a' => s.push('\x07'),
                    'b' => s.push('\x08'),
                    't' => s.push('\t'),
                    'n' => s.push('\n'),
                    'v' => s.push('\x0b'),
                    'f' => s.push('\x0c'),
                    'r' => s.push('\r'),
                    '"' => s.push('"'),
                    '\\' => s.push('\\'),
                    'x' => {
                        let mut hex_str = String::new();
                        loop {
                            match try_consume!(self) {
                                ';' => break,
                                c => hex_str.push(c)
                            }
                        }
                        if hex_str.len() == 0 {
                            return Err(self.make_error(ParserErrorKind::InvalidStringEscape(hex_str)))
                        }
                        let code = match num::from_str_radix(hex_str.as_slice(), 16) {
                            Ok(n) => n,
                            Err(_) => return Err(self.make_error(ParserErrorKind::InvalidStringEscape(hex_str)))
                        };
                        match char::from_u32(code) {
                            Some(c) => s.push(c),
                            None => return Err(self.make_error(ParserErrorKind::InvalidUnicodeRange(code)))
                        }
                    },
                    c =>
                        if c.is_whitespace() {
                            try!(self.read_while(|c| c != '\n' && c.is_whitespace()));
                            if c != '\n' {
                                if let Some(Ok('\n')) = self.consume() {
                                    try!(self.read_while(|c| c != '\n' && c.is_whitespace()));
                                } else {
                                    return Err(self.make_error(ParserErrorKind::InvalidStringLiteral))
                                }
                            }
                        } else {
                            let mut s = String::new();
                            s.push(c);
                            return Err(self.make_error(ParserErrorKind::InvalidStringEscape(s)))
                        }
                },
                c => s.push(c),
            }
        }
    }

    fn lex_numeric(&mut self, init: String) -> Result<String, ParserError> {
        let mut s = init;
        let sub = try!(self.read_while(|c| !is_delim(c)));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn make_error(&self, kind: ParserErrorKind) -> ParserError {
        ParserError {
            line: self.line,
            column: self.column,
            kind: kind
        }
    }

    fn lookahead(&mut self) -> Option<Result<char, CharsError>> {
        match self.lookahead_buf {
            Some(c) => Some(Ok(c)),
            None => {
                let c = self.stream.next();
                if let Some(Ok(ch)) = c {
                    self.lookahead_buf = Some(ch);
                }
                c
            }
        }
    }

    fn advance(&mut self, c: char) {
        if c == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
    }

    fn read_while<F>(&mut self, f: F) -> Result<String, ParserError> where
        F: Fn(char) -> bool
    {
        let mut s = match self.lookahead_buf {
            None => String::new(),
            Some(c) => if f(c) {
                self.lookahead_buf = None;
                self.advance(c);
                let mut s = String::new();
                s.push(c);
                s
            } else {
                return Ok(String::new());
            }
        };

        loop {
            match self.stream.next() {
                Some(Ok(c)) => if f(c) {
                    self.advance(c);
                    s.push(c);
                } else {
                    self.lookahead_buf = Some(c);
                    return Ok(s);
                },
                Some(Err(e)) =>
                    return Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
                None => return Ok(s)
            }
        }
    }

    fn consume(&mut self) -> Option<Result<char, CharsError>> {
        let c = match self.lookahead_buf {
            Some(c) => {
                self.lookahead_buf = None;
                Some(Ok(c))
            },
            None => self.stream.next()
        };

        if let Some(Ok(ch)) = c {
            self.advance(ch);
        }

        c
    }

    fn consume_whitespace(&mut self) -> Result<bool, ParserError> {
        let mut consumed = false;
        loop {
            let whitespace = try!(self.read_while(is_whitespace));
            consumed = consumed || whitespace.len() > 0;
            match self.lookahead() {
                Some(Ok(';')) => {
                    consumed = true;
                    try!(self.read_while(|c| c != '\n'));
                    if self.lookahead_buf.is_some() {
                        self.lookahead_buf = None
                    }
                },
                Some(Ok(_)) | None => return Ok(consumed),
                Some(Err(e)) =>
                    return Err(self.make_error(ParserErrorKind::UnderlyingError(e))),
            }
        }
    }
}
<|endoftext|>


<commit_before>//! The tokenizer module implements a nock tokenizer.

use std::io::{Error, ErrorKind};
use std::convert::From;
use std::convert::Into;
use std::char;

#[derive(Debug)]
struct Token {
    line: usize,
    col: usize,
    val: String,
}

impl Token {
    pub fn new(c: char, line: usize, col: usize) -> Self {
        Token {
            line: line,
            col: col,
            val: c.to_string(),
        }
    }

    pub fn append_char(&mut self, c: char) {
        self.val.push(c);
    }
}

#[derive(Debug)]
pub struct TokenizerError {
    msg: String,
    cause: Option<Error>,
}

impl TokenizerError {
    pub fn new<S: Into<String>>(msg: S) -> Self {
        TokenizerError {
            msg: msg.into(),
            cause: None,
        }
    }

    pub fn from_io_error<S: Into<String>>(msg: S, err: Error) -> Self {
        TokenizerError {
            msg: msg.into(),
            cause: Some(err),
        }
    }
}

impl From<Error> for TokenizerError {
    fn from(err: Error) -> Self {
        Self::from_io_error("IO Error", err)
    }
}

// TODO(jwall): Tokenizer error.
pub trait ExpressionReader {
    fn read(&mut self) -> Result<Vec<String>, Error>;
}

pub struct Tokenizer {
    curr: Option<Vec<String>>,
    line: usize,
    col: usize,
    reader: Box<ExpressionReader>,
}

impl Tokenizer {
    pub fn new(reader: Box<ExpressionReader>) -> Self {
        Tokenizer {
            curr: None,
            line: 0,
            col: 0,
            reader: reader,
        }
    }

    pub fn reset_line_col(&mut self) {
        self.line = 0;
        self.col = 0;
    }

    pub fn next(&mut self) -> Result<Token, TokenizerError> {
        if !self.curr.is_some() {
            let expr = try!(self.reader.read());
            self.curr = Some(expr);
        }
        self.get_next_token()
    }

    fn get_next_char(&mut self) -> Result<(char, usize, usize), TokenizerError> {
        if let Some(ref lines) = self.curr {
            if self.col >= lines[self.line].len() {
                let (line, col) = (self.line, self.col);
                self.line += 1;
                self.col = 0;
                // We synthesize a newline character to simplify parsing.
                return Ok(('\n', line, col));
            }
            if self.line >= lines.len() {
                return Err(TokenizerError::new("End of stream"));
            }
            // TODO(jwall): Should we cache this?
            let bytes = &lines[self.line].as_bytes();
            // Since all nock syntax is valid ascii this is a
            // safe cast to do.
            let curr_col = self.col;
            self.col += 1;
            return Ok((bytes[curr_col] as char, self.line, curr_col));
        }
        return Err(TokenizerError::new("Empty Expression!"));
    }

    fn pushback(&mut self, len: usize) {
        // This is potentially unsafe but since we are in theory only
        // ever pushing back something that we have already consumed in
        // a single line this should be safe.
        self.col -= len;
    }

    fn gobble_atom(&mut self, mut tok: Token) -> Result<Token, TokenizerError> {
        loop {
            // char loop
            let (c, _, _) = match self.get_next_char() {
                Ok(tpl) => tpl,
                Err(_) => return Ok(tok),
            };
            if c.is_whitespace() {
                return Ok(tok);
            }
            if !c.is_digit(10) && c != '.' {
                // Technically this case is an error but we don't emit
                // error tokens here, ever, despite what the type signature
                // states.
                self.pushback(1);
                return Ok(tok);
            }
            tok.append_char(c);
        }
        return Ok(tok);
    }

    fn get_next_token(&mut self) -> Result<Token, TokenizerError> {
        let maybe_tok: Option<Token> = None;
        loop {
            // char loop
            let (c, line, col) = try!(self.get_next_char());
            match c {
                // open cell
                '[' => {
                    return Ok(Token::new(c, line, col));
                }
                // close cell
                ']' => {
                    return Ok(Token::new(c, line, col));
                }
                // Atom chars
                '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | '.' => {
                    return self.gobble_atom(Token::new(c, line, col));
                }
                // Whitespace
                ' ' | '\t' | '\n' | '\r' => {
                    // We skip these.
                    println!("Skipping whitespace char");
                    continue;
                }
                _ => return Err(TokenizerError::new(format!("Invalid Character: '{}'", c))),
            }
        }
    }
}

#[cfg(test)]
mod tokenizer_tests {
    use std::io::{Error, ErrorKind};
    use tokenizer::{ExpressionReader, Tokenizer, TokenizerError};

    struct MockReader {
        expr: Vec<String>,
        err: bool,
    }

    impl MockReader {
        pub fn new(expr: Vec<String>) -> Self {
            MockReader {
                expr: expr,
                err: false,
            }
        }
    }

    impl ExpressionReader for MockReader {
        fn read(&mut self) -> Result<Vec<String>, Error> {
            if !self.err {
                self.err = true;
                return Ok(self.expr.clone());
            } else {
                return Err(Error::new(ErrorKind::Other, "End Of Stream"));
            }
        }
    }

    #[test]
    fn mock_reader_sanity_check() {
        let mut reader = MockReader::new(vec![
                "[1 2 3]".to_string(),
            ]);
        let expr = reader.read();
        assert!(expr.is_ok());
        let expr = reader.read();
        assert!(expr.is_err());
    }

    fn assert_token_stream(toker: &mut Tokenizer, expect: Vec<(&str, usize, usize)>) {
        let mut i = 0;
        for (v, l, c) in expect {
            let tok = toker.next();
            println!("tok: {:?}", tok);
            assert!(tok.is_ok());
            let tok = tok.unwrap();
            assert_eq!(tok.line, l);
            assert_eq!(tok.col, c);
            assert_eq!(tok.val, *v);
            i += 1;
        }
    }

    #[test]
    fn test_tokenizer_simple_one_liner() {
        let reader = MockReader::new(vec![
                "[1 2 3]".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("[", 0, 0), ("1", 0, 1), ("2", 0, 3), ("3", 0, 5), ("]", 0, 6)];
        assert_token_stream(&mut toker, expect);
    }

    #[test]
    fn test_tokenizer_simple_multi_line() {
        let reader = MockReader::new(vec![
                "[1 2 3".to_string(),
                "]".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("[", 0, 0), ("1", 0, 1), ("2", 0, 3), ("3", 0, 5), ("]", 1, 0)];
        assert_token_stream(&mut toker, expect);
    }

    #[test]
    fn test_tokenizer_simple_multi_char_atoms() {
        let reader = MockReader::new(vec![
                "1234567890".to_string(),
                "123  1".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("1234567890", 0, 0),
                          ("123", 1, 0),
                          ("1", 1, 5),
        ];
        assert_token_stream(&mut toker, expect);
    }
}
<commit_msg>Fix dot-handling for numbers to in the tokenizer.<commit_after>//! The tokenizer module implements a nock tokenizer.

use std::io::{Error, ErrorKind};
use std::convert::From;
use std::convert::Into;
use std::char;

#[derive(Debug)]
struct Token {
    line: usize,
    col: usize,
    val: String,
}

impl Token {
    pub fn new(c: char, line: usize, col: usize) -> Self {
        Token {
            line: line,
            col: col,
            val: c.to_string(),
        }
    }

    pub fn append_char(&mut self, c: char) {
        self.val.push(c);
    }
}

#[derive(Debug)]
pub struct TokenizerError {
    msg: String,
    cause: Option<Error>,
}

impl TokenizerError {
    pub fn new<S: Into<String>>(msg: S) -> Self {
        TokenizerError {
            msg: msg.into(),
            cause: None,
        }
    }

    pub fn from_io_error<S: Into<String>>(msg: S, err: Error) -> Self {
        TokenizerError {
            msg: msg.into(),
            cause: Some(err),
        }
    }
}

impl From<Error> for TokenizerError {
    fn from(err: Error) -> Self {
        Self::from_io_error("IO Error", err)
    }
}

// TODO(jwall): Tokenizer error.
pub trait ExpressionReader {
    fn read(&mut self) -> Result<Vec<String>, Error>;
}

pub struct Tokenizer {
    curr: Option<Vec<String>>,
    line: usize,
    col: usize,
    reader: Box<ExpressionReader>,
}

impl Tokenizer {
    pub fn new(reader: Box<ExpressionReader>) -> Self {
        Tokenizer {
            curr: None,
            line: 0,
            col: 0,
            reader: reader,
        }
    }

    pub fn reset_line_col(&mut self) {
        self.line = 0;
        self.col = 0;
    }

    pub fn next(&mut self) -> Result<Token, TokenizerError> {
        if !self.curr.is_some() {
            let expr = try!(self.reader.read());
            self.curr = Some(expr);
        }
        self.get_next_token()
    }

    fn get_next_char(&mut self) -> Result<(char, usize, usize), TokenizerError> {
        if let Some(ref lines) = self.curr {
            if self.col >= lines[self.line].len() {
                let (line, col) = (self.line, self.col);
                self.line += 1;
                self.col = 0;
                // We synthesize a newline character to simplify parsing.
                return Ok(('\n', line, col));
            }
            if self.line >= lines.len() {
                return Err(TokenizerError::new("End of stream"));
            }
            // TODO(jwall): Should we cache this?
            let bytes = &lines[self.line].as_bytes();
            // Since all nock syntax is valid ascii this is a
            // safe cast to do.
            let curr_col = self.col;
            self.col += 1;
            return Ok((bytes[curr_col] as char, self.line, curr_col));
        }
        return Err(TokenizerError::new("Empty Expression!"));
    }

    fn pushback(&mut self, len: usize) {
        // This is potentially unsafe but since we are in theory only
        // ever pushing back something that we have already consumed in
        // a single line this should be safe.
        self.col -= len;
    }

    fn gobble_atom(&mut self, mut tok: Token) -> Result<Token, TokenizerError> {
        loop {
            // char loop
            let (c, _, _) = match self.get_next_char() {
                Ok(tpl) => tpl,
                Err(_) => return Ok(tok),
            };
            if c.is_whitespace() {
                return Ok(tok);
            }
            if c == '.' {
                // treat . as whitespace inside of an atom.
                // Currently this is pretty dumb and doesn't
                // enforce the right syntax of dotting as comma.
                // i.e. every 3 digits. This is deemed acceptable
                // for now.
                continue;
            }
            if !c.is_digit(10) {
                // Technically this case is an error but we don't emit
                // error tokens here, ever, despite what the type signature
                // states.
                self.pushback(1);
                return Ok(tok);
            }
            tok.append_char(c);
        }
        return Ok(tok);
    }

    fn get_next_token(&mut self) -> Result<Token, TokenizerError> {
        let maybe_tok: Option<Token> = None;
        loop {
            // char loop
            let (c, line, col) = try!(self.get_next_char());
            match c {
                // open cell
                '[' => {
                    return Ok(Token::new(c, line, col));
                }
                // close cell
                ']' => {
                    return Ok(Token::new(c, line, col));
                }
                // Atom chars
                '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | '.' => {
                    return self.gobble_atom(Token::new(c, line, col));
                }
                // Whitespace
                ' ' | '\t' | '\n' | '\r' => {
                    // We skip these.
                    println!("Skipping whitespace char");
                    continue;
                }
                _ => return Err(TokenizerError::new(format!("Invalid Character: '{}'", c))),
            }
        }
    }
}

#[cfg(test)]
mod tokenizer_tests {
    use std::io::{Error, ErrorKind};
    use tokenizer::{ExpressionReader, Tokenizer, TokenizerError};

    struct MockReader {
        expr: Vec<String>,
        err: bool,
    }

    impl MockReader {
        pub fn new(expr: Vec<String>) -> Self {
            MockReader {
                expr: expr,
                err: false,
            }
        }
    }

    impl ExpressionReader for MockReader {
        fn read(&mut self) -> Result<Vec<String>, Error> {
            if !self.err {
                self.err = true;
                return Ok(self.expr.clone());
            } else {
                return Err(Error::new(ErrorKind::Other, "End Of Stream"));
            }
        }
    }

    #[test]
    fn mock_reader_sanity_check() {
        let mut reader = MockReader::new(vec![
                "[1 2 3]".to_string(),
            ]);
        let expr = reader.read();
        assert!(expr.is_ok());
        let expr = reader.read();
        assert!(expr.is_err());
    }

    fn assert_token_stream(toker: &mut Tokenizer, expect: Vec<(&str, usize, usize)>) {
        let mut i = 0;
        for (v, l, c) in expect {
            let tok = toker.next();
            println!("tok: {:?}", tok);
            assert!(tok.is_ok());
            let tok = tok.unwrap();
            assert_eq!(tok.line, l);
            assert_eq!(tok.col, c);
            assert_eq!(tok.val, *v);
            i += 1;
        }
    }

    #[test]
    fn test_tokenizer_simple_one_liner() {
        let reader = MockReader::new(vec![
                "[1 2 3]".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("[", 0, 0), ("1", 0, 1), ("2", 0, 3), ("3", 0, 5), ("]", 0, 6)];
        assert_token_stream(&mut toker, expect);
    }

    #[test]
    fn test_tokenizer_simple_multi_line() {
        let reader = MockReader::new(vec![
                "[1 2 3".to_string(),
                "]".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("[", 0, 0), ("1", 0, 1), ("2", 0, 3), ("3", 0, 5), ("]", 1, 0)];
        assert_token_stream(&mut toker, expect);
    }

    #[test]
    fn test_tokenizer_simple_multi_char_atoms() {
        let reader = MockReader::new(vec![
                "1234567890".to_string(),
                "123  1".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("1234567890", 0, 0),
                          ("123", 1, 0),
                          ("1", 1, 5),
        ];
        assert_token_stream(&mut toker, expect);
    }

    #[test]
    fn test_tokenizer_simple_multi_char_atoms_with_dot() {
        let reader = MockReader::new(vec![
                "123.4567.890".to_string(),
                "123  1".to_string(),
            ]);
        let boxed = Box::new(reader);
        let mut toker = Tokenizer::new(boxed);
        let expect = vec![("1234567890", 0, 0),
                          ("123", 1, 0),
                          ("1", 1, 5),
        ];
        assert_token_stream(&mut toker, expect);
    }
}
<|endoftext|>


<commit_before>use std::io::{IoError, IoErrorKind};
use std::string::CowString;
use std::borrow::Cow;
use std::fmt;

use error::{ParserError, ParserErrorKind};

/// Token types
#[derive(PartialEq)]
pub enum Token {
    /// `(`
    OpenParen,
    /// `)`
    CloseParen,
    /// `.`
    Dot,
    Identifier(CowString<'static>),
    /// `#t`
    True,
    /// `#f`
    False,
    /// `#\<String>`
    Character(String),
    Numeric(String),
    /// End of character stream
    EOF
}

impl fmt::Show for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::OpenParen => write!(f, "OpenParen"),
            Token::CloseParen => write!(f, "CloseParen"),
            Token::Dot => write!(f, "Dot"),
            Token::Identifier(ref name) => write!(f, "Identifier({})", name),
            Token::True => write!(f, "#t"),
            Token::False => write!(f, "#f"),
            Token::Character(ref name) => write!(f, "#\\{}", name),
            Token::Numeric(ref rep) => rep.fmt(f),
            Token::EOF => write!(f, "EOF"),
        }
    }
}

/// TokenWrapper provides positional information to each token
pub struct TokenWrapper {
    pub line: usize,
    pub column: usize,
    pub token: Token
}

fn wrap(line: usize, column: usize, t: Token) -> TokenWrapper {
    TokenWrapper {
        line: line,
        column: column,
        token: t
    }
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ' => true,
        _ => false
    }
}

fn is_initial(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '!' | '$' | '%' | '&' | '*' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '_' | '~' => true,
        _ => false
    }
}

fn is_subsequent(c: char) -> bool {
    if is_initial(c) {
        true
    } else {
        match c {
            '0'...'9' | '+' | '-' | '.' | '@' => true,
            _ => false
        }
    }
}

/// Lexer transforms character stream into a token stream
pub struct Lexer<'a> {
    line: usize,
    column: usize,
    stream: &'a mut (Buffer+'a),
    lookahead_buf: Option<char>,
}

impl <'a> Lexer<'a> {
    /// Creates new Lexer from io::Buffer
    pub fn new<'r>(stream: &'r mut Buffer) -> Lexer<'r> {
        Lexer {
            line: 1,
            column: 1,
            stream: stream,
            lookahead_buf: None,
        }
    }

    /// return next token
    pub fn lex_token(&mut self) -> Result<TokenWrapper, ParserError> {
        try!(self.consume_whitespace());

        let line = self.line;
        let col = self.column;
        let c = match self.consume() {
            Err(e) => match e.kind {
                IoErrorKind::EndOfFile => return Ok(wrap(line, col, Token::EOF)),
                _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
            },
            Ok(c) => c
        };

        let end_of_token = try!(self.is_end_of_token());

        if is_initial(c) {
            let mut init = String::new();
            init.push(c);
            self.lex_ident(init).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s))))
        } else if c == '+' && end_of_token {
            Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("+"))))
        } else if c == '-' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
            } else {
                match self.lookahead() {
                    Ok('>') => self.lex_ident("-".to_string()).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s)))),
                    Ok(c) => Err(self.make_error(ParserErrorKind::InvalidCharacter(c))),
                    Err(e) => match e.kind {
                        IoErrorKind::EndOfFile => Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-")))),
                        _ => Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                    }
                }
            }
        } else if c == '(' {
            Ok(wrap(line, col, Token::OpenParen))
        } else if c == ')' {
            Ok(wrap(line, col, Token::CloseParen))
        } else if c == '.' && end_of_token {
            Ok(wrap(line, col, Token::Dot))
        } else if c == '#' {
            let c0 = match self.consume() {
                Err(e) => return Err(match e.kind {
                    IoErrorKind::EndOfFile => self.make_error(ParserErrorKind::UnexpectedEOF),
                    _ => self.make_error(ParserErrorKind::UnderlyingError(e))
                }),
                Ok(x) => x
            };
            match c0 {
                't' | 'T' => Ok(wrap(line, col, Token::True)),
                'f' | 'F' => Ok(wrap(line, col, Token::False)),
                '\\' => self.lex_char().map(|s| wrap(line, col, Token::Character(s))),
                _ => Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
            }
        } else if c.is_numeric() {
            self.lex_numeric(c).map(|s| wrap(line, col, Token::Numeric(s)))
        } else {
            Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
        }
    }

    fn is_end_of_token(&mut self) -> Result<bool, ParserError> {
        match self.lookahead() {
            Ok(c) => Ok(is_whitespace(c)),
            Err(e) => match e.kind {
                IoErrorKind::EndOfFile => Ok(true),
                _ => Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
            }
        }
    }

    fn lex_ident(&mut self, initial: String) -> Result<String, ParserError> {
        let mut s = initial;
        let sub = try!(self.read_while(is_subsequent));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_char(&mut self) -> Result<String, ParserError> {
        let c = match self.consume() {
            Ok(c) => c,
            Err(e) => return Err(self.make_error(match e.kind {
                IoErrorKind::EndOfFile => ParserErrorKind::UnexpectedEOF,
                _ => ParserErrorKind::UnderlyingError(e)
            }))
        };

        let mut s = String::new();
        s.push(c);
        let sub = try!(self.read_while(|c| c.is_alphanumeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_numeric(&mut self, init: char) -> Result<String, ParserError> {
        let mut s = String::new();
        s.push(init);
        let sub = try!(self.read_while(|c| c.is_numeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn make_error(&self, kind: ParserErrorKind) -> ParserError {
        ParserError {
            line: self.line,
            column: self.column,
            kind: kind
        }
    }

    fn lookahead(&mut self) -> Result<char, IoError> {
        Ok(match self.lookahead_buf {
            Some(c) => c,
            None => {
                let c = try!(self.stream.read_char());
                self.lookahead_buf = Some(c);
                c
            }
        })
    }

    fn advance(&mut self, c: char) {
        if c == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
    }

    fn read_while<F>(&mut self, f: F) -> Result<String, ParserError> where
        F: Fn(char) -> bool
    {
        let mut s = match self.lookahead_buf {
            None => String::new(),
            Some(c) => if f(c) {
                self.lookahead_buf = None;
                self.advance(c);
                let mut s = String::new();
                s.push(c);
                s
            } else {
                return Ok(String::new());
            }
        };

        loop {
            match self.stream.read_char() {
                Ok(c) => if f(c) {
                    self.advance(c);
                    s.push(c);
                } else {
                    self.lookahead_buf = Some(c);
                    return Ok(s);
                },
                Err(e) => match e.kind {
                    IoErrorKind::EndOfFile => return Ok(s),
                    _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                }
            }
        }
    }

    fn consume(&mut self) -> Result<char, IoError> {
        let c = match self.lookahead_buf {
            Some(c) => {
                self.lookahead_buf = None;
                c
            },
            None => try!(self.stream.read_char())
        };

        self.advance(c);
        Ok(c)
    }

    fn consume_whitespace(&mut self) -> Result<bool, ParserError> {
        let mut consumed = false;
        loop {
            let whitespace = try!(self.read_while(is_whitespace));
            consumed = consumed || whitespace.len() > 0;
            match self.lookahead() {
                Ok(';') => {
                    consumed = true;
                    try!(self.read_while(|c| c != '\n'));
                    if self.lookahead_buf.is_some() {
                        self.lookahead_buf = None
                    }
                },
                Ok(_) => return Ok(consumed),
                Err(e) => match e.kind {
                    IoErrorKind::EndOfFile => return Ok(consumed),
                    _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                }
            }
        }
    }
}
<commit_msg>Add lexer support for negative int<commit_after>use std::io::{IoError, IoErrorKind};
use std::string::CowString;
use std::borrow::Cow;
use std::fmt;

use error::{ParserError, ParserErrorKind};

/// Token types
#[derive(PartialEq)]
pub enum Token {
    /// `(`
    OpenParen,
    /// `)`
    CloseParen,
    /// `.`
    Dot,
    Identifier(CowString<'static>),
    /// `#t`
    True,
    /// `#f`
    False,
    /// `#\<String>`
    Character(String),
    Numeric(String),
    /// End of character stream
    EOF
}

impl fmt::Show for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::OpenParen => write!(f, "OpenParen"),
            Token::CloseParen => write!(f, "CloseParen"),
            Token::Dot => write!(f, "Dot"),
            Token::Identifier(ref name) => write!(f, "Identifier({})", name),
            Token::True => write!(f, "#t"),
            Token::False => write!(f, "#f"),
            Token::Character(ref name) => write!(f, "#\\{}", name),
            Token::Numeric(ref rep) => rep.fmt(f),
            Token::EOF => write!(f, "EOF"),
        }
    }
}

/// TokenWrapper provides positional information to each token
pub struct TokenWrapper {
    pub line: usize,
    pub column: usize,
    pub token: Token
}

fn wrap(line: usize, column: usize, t: Token) -> TokenWrapper {
    TokenWrapper {
        line: line,
        column: column,
        token: t
    }
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ' => true,
        _ => false
    }
}

fn is_initial(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '!' | '$' | '%' | '&' | '*' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '_' | '~' => true,
        _ => false
    }
}

fn is_subsequent(c: char) -> bool {
    if is_initial(c) {
        true
    } else {
        match c {
            '0'...'9' | '+' | '-' | '.' | '@' => true,
            _ => false
        }
    }
}

/// Lexer transforms character stream into a token stream
pub struct Lexer<'a> {
    line: usize,
    column: usize,
    stream: &'a mut (Buffer+'a),
    lookahead_buf: Option<char>,
}

impl <'a> Lexer<'a> {
    /// Creates new Lexer from io::Buffer
    pub fn new<'r>(stream: &'r mut Buffer) -> Lexer<'r> {
        Lexer {
            line: 1,
            column: 1,
            stream: stream,
            lookahead_buf: None,
        }
    }

    /// return next token
    pub fn lex_token(&mut self) -> Result<TokenWrapper, ParserError> {
        try!(self.consume_whitespace());

        let line = self.line;
        let col = self.column;
        let c = match self.consume() {
            Err(e) => match e.kind {
                IoErrorKind::EndOfFile => return Ok(wrap(line, col, Token::EOF)),
                _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
            },
            Ok(c) => c
        };

        let end_of_token = try!(self.is_end_of_token());

        if is_initial(c) {
            let mut init = String::new();
            init.push(c);
            self.lex_ident(init).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s))))
        } else if c == '+' && end_of_token {
            Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("+"))))
        } else if c == '-' {
            if end_of_token {
                Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-"))))
            } else {
                match self.lookahead() {
                    Ok('>') => self.lex_ident("-".to_string()).map(|s| wrap(line, col, Token::Identifier(Cow::Owned(s)))),
                    Ok(c) => if c.is_numeric() {
                            self.lex_numeric('-').map(|s| wrap(line, col, Token::Numeric(s)))
                        } else {
                            Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
                        },
                    Err(e) => match e.kind {
                        IoErrorKind::EndOfFile => Ok(wrap(line, col, Token::Identifier(Cow::Borrowed("-")))),
                        _ => Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                    }
                }
            }
        } else if c == '(' {
            Ok(wrap(line, col, Token::OpenParen))
        } else if c == ')' {
            Ok(wrap(line, col, Token::CloseParen))
        } else if c == '.' && end_of_token {
            Ok(wrap(line, col, Token::Dot))
        } else if c == '#' {
            let c0 = match self.consume() {
                Err(e) => return Err(match e.kind {
                    IoErrorKind::EndOfFile => self.make_error(ParserErrorKind::UnexpectedEOF),
                    _ => self.make_error(ParserErrorKind::UnderlyingError(e))
                }),
                Ok(x) => x
            };
            match c0 {
                't' | 'T' => Ok(wrap(line, col, Token::True)),
                'f' | 'F' => Ok(wrap(line, col, Token::False)),
                '\\' => self.lex_char().map(|s| wrap(line, col, Token::Character(s))),
                _ => Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
            }
        } else if c.is_numeric() {
            self.lex_numeric(c).map(|s| wrap(line, col, Token::Numeric(s)))
        } else {
            Err(self.make_error(ParserErrorKind::InvalidCharacter(c)))
        }
    }

    fn is_end_of_token(&mut self) -> Result<bool, ParserError> {
        match self.lookahead() {
            Ok(c) => Ok(is_whitespace(c)),
            Err(e) => match e.kind {
                IoErrorKind::EndOfFile => Ok(true),
                _ => Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
            }
        }
    }

    fn lex_ident(&mut self, initial: String) -> Result<String, ParserError> {
        let mut s = initial;
        let sub = try!(self.read_while(is_subsequent));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_char(&mut self) -> Result<String, ParserError> {
        let c = match self.consume() {
            Ok(c) => c,
            Err(e) => return Err(self.make_error(match e.kind {
                IoErrorKind::EndOfFile => ParserErrorKind::UnexpectedEOF,
                _ => ParserErrorKind::UnderlyingError(e)
            }))
        };

        let mut s = String::new();
        s.push(c);
        let sub = try!(self.read_while(|c| c.is_alphanumeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn lex_numeric(&mut self, init: char) -> Result<String, ParserError> {
        let mut s = String::new();
        s.push(init);
        let sub = try!(self.read_while(|c| c.is_numeric()));
        s.push_str(sub.as_slice());
        return Ok(s);
    }

    fn make_error(&self, kind: ParserErrorKind) -> ParserError {
        ParserError {
            line: self.line,
            column: self.column,
            kind: kind
        }
    }

    fn lookahead(&mut self) -> Result<char, IoError> {
        Ok(match self.lookahead_buf {
            Some(c) => c,
            None => {
                let c = try!(self.stream.read_char());
                self.lookahead_buf = Some(c);
                c
            }
        })
    }

    fn advance(&mut self, c: char) {
        if c == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
    }

    fn read_while<F>(&mut self, f: F) -> Result<String, ParserError> where
        F: Fn(char) -> bool
    {
        let mut s = match self.lookahead_buf {
            None => String::new(),
            Some(c) => if f(c) {
                self.lookahead_buf = None;
                self.advance(c);
                let mut s = String::new();
                s.push(c);
                s
            } else {
                return Ok(String::new());
            }
        };

        loop {
            match self.stream.read_char() {
                Ok(c) => if f(c) {
                    self.advance(c);
                    s.push(c);
                } else {
                    self.lookahead_buf = Some(c);
                    return Ok(s);
                },
                Err(e) => match e.kind {
                    IoErrorKind::EndOfFile => return Ok(s),
                    _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                }
            }
        }
    }

    fn consume(&mut self) -> Result<char, IoError> {
        let c = match self.lookahead_buf {
            Some(c) => {
                self.lookahead_buf = None;
                c
            },
            None => try!(self.stream.read_char())
        };

        self.advance(c);
        Ok(c)
    }

    fn consume_whitespace(&mut self) -> Result<bool, ParserError> {
        let mut consumed = false;
        loop {
            let whitespace = try!(self.read_while(is_whitespace));
            consumed = consumed || whitespace.len() > 0;
            match self.lookahead() {
                Ok(';') => {
                    consumed = true;
                    try!(self.read_while(|c| c != '\n'));
                    if self.lookahead_buf.is_some() {
                        self.lookahead_buf = None
                    }
                },
                Ok(_) => return Ok(consumed),
                Err(e) => match e.kind {
                    IoErrorKind::EndOfFile => return Ok(consumed),
                    _ => return Err(self.make_error(ParserErrorKind::UnderlyingError(e)))
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>//! Adaptors for object-safe hashing.
//!
//! `std::hash::Hash` is not object-safe because trait function `hash` takes
//! a type parameter that specifies the concrete hash implementation used.
//!
//! This module defines a trait [`Hashable`](trait.Hashable.html), which can be
//! implemented identically to `std::hash::Hash` but accepts the hash-state
//! object as a trait object (`state: &Hasher`) and thus requires no type
//! parameters on function `hash`.
//!
//! For this reason, `Hashable` is entirely agnostic to the hash implementation being
//! used; any trait object that specifies `Hashable` as a supertrait, and
//! implements `Hash` in terms of Hashable, can be used may be safely used as
//! a (boxed) type parameter to containers like `HashSet`.
//!
//! TL;DR => if you want to hash instances of a type using dynamic dispatch
//! through a trait it implements, you must:
//!
//!
//!   1. ensure that the trait lists `Hashable` as a supertrait, and implements
//!      `std::hash::Hash` in terms of `Hashable`;
//!
//!      ```rust,ignore
//!      trait MyTrait: Hashable {
//!          // ...
//!      }
//!
//!      impl std::hash::Hash for MyTrait {
//!          fn hash<H: Hasher>(&self, state: &mut H) {
//!              <Self as Hashable>::hash(self, state);
//!          }
//!      }
//!      ```
//!
//!      and
//!
//!   2. implement `Hashable` for each concrete type that implements the trait.
//!
//!      ```rust,ignore
//!      impl Hashable for MyType {
//!          fn hash(&self, state: &mut std::hash::Hasher) {
//!              // function body should be identical to that of
//!              // an implementation of `Hash::hash`.
//!          }
//!      }
//!      ```
//!
//!
//! ## Complete Example
//!
//! ```rust
//! #![feature(hash)]
//! use std::cmp::{Eq,PartialEq};
//! use std::hash::{Hash,Hasher,SipHasher};
//! use std::collections::HashSet;
//!
//! use cripes::util::hash::{stdhash,hash,Hashable};
//!
//! trait MyInterface: Hashable {}
//!
//! impl Eq for MyInterface {}
//! impl PartialEq<MyInterface> for MyInterface {
//!     fn eq(&self, other: &MyInterface) -> bool {
//!         hash::<_,SipHasher>(self) == hash::<_,SipHasher>(other)
//!     }
//! }
//!
//! impl std::hash::Hash for MyInterface {
//!     fn hash<H: Hasher>(&self, state: &mut H) {
//!         <Self as Hashable>::hash(self, state)
//!     }
//! }
//!
//!
//! #[derive(Eq,PartialEq)]
//! struct MyType(u64, &'static str);
//!
//! impl MyInterface for MyType {}
//! impl Hashable for MyType {
//!     fn hash(&self, state: &mut Hasher) {
//!         state.write_u64(self.0);
//!         state.write(self.1.as_bytes());
//!     }
//! }
//!
//! fn main() {
//!     use std::collections::HashSet;
//!
//!     let mut hs: HashSet<Box<MyInterface>> = HashSet::new();
//!     hs.insert(Box::new(MyType(123, "abc")));
//!
//!     // ...
//!
//!     for x in &hs {
//!         println!("{}", stdhash::<_,SipHasher>(x));
//!     }
//! }
//!
//! ```
//!
use std::default::Default;
use std::hash::{Hash,Hasher};

/// Object-safe hash adaptor.  See the [module-level documentation](index.html)
/// for details.
pub trait Hashable {
    fn hash(&self, state: &mut Hasher);
}

/// Hash `char` as a `u32`.
impl Hashable for char {
    #[inline(always)]
    fn hash(&self, state: &mut Hasher) {
        state.write_u32(*self as u32);
    }
}

pub fn stdhash<T: Hash,H: Default + Hasher>(t: &T) -> u64 {
    let mut s = H::default();
    t.hash(&mut s);
    s.finish()
}

/// Hash an object by using the Hashable interface directly.
///
///
pub fn hash<T: ?Sized, H>(obj: &T) -> u64 where H: Hasher + Default, T: Hashable {
    let mut state: H = Default::default();
    obj.hash(&mut state);
    state.finish()
}
<commit_msg>util/hash.rs: refill a paragraph in module-level doc-comment<commit_after>//! Adaptors for object-safe hashing.
//!
//! `std::hash::Hash` is not object-safe because trait function `hash` takes
//! a type parameter that specifies the concrete hash implementation used.
//!
//! This module defines a trait [`Hashable`](trait.Hashable.html), which can be
//! implemented identically to `std::hash::Hash` but accepts the hash-state
//! object as a trait object (`state: &Hasher`) and thus requires no type
//! parameters on function `hash`.
//!
//! For this reason, `Hashable` is entirely agnostic to the hash implementation
//! being used; any trait object that specifies `Hashable` as a supertrait, and
//! implements `Hash` in terms of Hashable, may be safely used as a (boxed)
//! type parameter to containers like `HashSet`.
//!
//! TL;DR => if you want to hash instances of a type using dynamic dispatch
//! through a trait it implements, you must:
//!
//!
//!   1. ensure that the trait lists `Hashable` as a supertrait, and implements
//!      `std::hash::Hash` in terms of `Hashable`;
//!
//!      ```rust,ignore
//!      trait MyTrait: Hashable {
//!          // ...
//!      }
//!
//!      impl std::hash::Hash for MyTrait {
//!          fn hash<H: Hasher>(&self, state: &mut H) {
//!              <Self as Hashable>::hash(self, state);
//!          }
//!      }
//!      ```
//!
//!      and
//!
//!   2. implement `Hashable` for each concrete type that implements the trait.
//!
//!      ```rust,ignore
//!      impl Hashable for MyType {
//!          fn hash(&self, state: &mut std::hash::Hasher) {
//!              // function body should be identical to that of
//!              // an implementation of `Hash::hash`.
//!          }
//!      }
//!      ```
//!
//!
//! ## Complete Example
//!
//! ```rust
//! #![feature(hash)]
//! use std::cmp::{Eq,PartialEq};
//! use std::hash::{Hash,Hasher,SipHasher};
//! use std::collections::HashSet;
//!
//! use cripes::util::hash::{stdhash,hash,Hashable};
//!
//! trait MyInterface: Hashable {}
//!
//! impl Eq for MyInterface {}
//! impl PartialEq<MyInterface> for MyInterface {
//!     fn eq(&self, other: &MyInterface) -> bool {
//!         hash::<_,SipHasher>(self) == hash::<_,SipHasher>(other)
//!     }
//! }
//!
//! impl std::hash::Hash for MyInterface {
//!     fn hash<H: Hasher>(&self, state: &mut H) {
//!         <Self as Hashable>::hash(self, state)
//!     }
//! }
//!
//!
//! #[derive(Eq,PartialEq)]
//! struct MyType(u64, &'static str);
//!
//! impl MyInterface for MyType {}
//! impl Hashable for MyType {
//!     fn hash(&self, state: &mut Hasher) {
//!         state.write_u64(self.0);
//!         state.write(self.1.as_bytes());
//!     }
//! }
//!
//! fn main() {
//!     use std::collections::HashSet;
//!
//!     let mut hs: HashSet<Box<MyInterface>> = HashSet::new();
//!     hs.insert(Box::new(MyType(123, "abc")));
//!
//!     // ...
//!
//!     for x in &hs {
//!         println!("{}", stdhash::<_,SipHasher>(x));
//!     }
//! }
//!
//! ```
//!
use std::default::Default;
use std::hash::{Hash,Hasher};

/// Object-safe hash adaptor.  See the [module-level documentation](index.html)
/// for details.
pub trait Hashable {
    fn hash(&self, state: &mut Hasher);
}

/// Hash `char` as a `u32`.
impl Hashable for char {
    #[inline(always)]
    fn hash(&self, state: &mut Hasher) {
        state.write_u32(*self as u32);
    }
}

pub fn stdhash<T: Hash,H: Default + Hasher>(t: &T) -> u64 {
    let mut s = H::default();
    t.hash(&mut s);
    s.finish()
}

/// Hash an object by using the Hashable interface directly.
///
///
pub fn hash<T: ?Sized, H>(obj: &T) -> u64 where H: Hasher + Default, T: Hashable {
    let mut state: H = Default::default();
    obj.hash(&mut state);
    state.finish()
}
<|endoftext|>