<commit_before>use std::collections::HashMap;
use std::string;

use self::TokenizerErr::*;

use super::token::XPathToken;
use super::token::XPathToken::*;

fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

pub struct XPathTokenizer {
    xpath: XPathString,
    start: uint,
    prefer_recognition_of_operator_names: bool,
}

pub type TokenResult = Result<XPathToken, TokenizerErr>;

#[deriving(Show,PartialEq,Clone,Copy)]
pub enum TokenizerErr {
    MissingLocalName,
    MismatchedQuoteCharacters,
    UnableToCreateToken,
}

struct XPathString {
    xpath: Vec<char>,
}

impl XPathString {
    fn new(xpath: &str) -> XPathString {
        XPathString {
            xpath: xpath.chars().collect(),
        }
    }

    fn len(& self) -> uint {
        self.xpath.len()
    }

    fn str_at_is(& self, offset: uint, needle: &[char]) -> bool {
        let s_len = needle.len();

        if self.xpath.len() < offset + s_len { return false; }

        let xpath_chars = self.xpath.slice(offset, offset + s_len);

        needle == xpath_chars
    }

    fn valid_ncname_start_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if c >= 'A' && c <= 'Z' { return true }
        if c == '_' { return true }
        if c >= 'a' && c <= 'z' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn valid_ncname_follow_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if self.valid_ncname_start_char(offset) { return true }
        if c == '-' { return true }
        if c == '.' { return true }
        if c >= '0' && c <= '9' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn while_valid_string(& self, offset: uint) -> uint {
        let mut offset = offset;

        if offset < self.xpath.len() && self.valid_ncname_start_char(offset) {
            offset += 1;

            while offset < self.xpath.len() && self.valid_ncname_follow_char(offset) {
                offset += 1;
            }
        }

        return offset;
    }

    fn while_valid_number(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && is_number_char(self.xpath[offset]) {
            offset += 1;
        }

        return offset;
    }

    fn while_not_character(& self, offset: uint, end_char: char) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.xpath[offset] != end_char {
            offset += 1;
        }

        return offset;
    }

    fn substr(& self, start: uint, end: uint) -> string::String {
        string::String::from_chars(self.xpath.slice(start, end))
    }

    fn safe_substr(& self, start: uint, end: uint) -> Option<string::String> {
        if self.xpath.len() >= end {
            Some(self.substr(start, end))
        } else {
            None
        }
    }

    fn char_at(&self, offset: uint) -> char {
        self.xpath[offset]
    }

    fn char_at_is(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        has_one_more && self.xpath[offset] == c
    }

    fn char_at_is_not(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        ! has_one_more || self.xpath[offset] != c
    }

    fn char_at_is_not_digit(& self, offset: uint) -> bool {
        let has_more_chars = self.xpath.len() >= offset + 1;

        ! has_more_chars || ! is_digit(self.xpath[offset])
    }

    fn is_xml_space(&self, offset: uint) -> bool {
        let c = self.xpath[offset];

        return
            c == ' '  ||
            c == '\t' ||
            c == '\n' ||
            c == '\r';
    }

    fn end_of_whitespace(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.is_xml_space(offset) {
            offset += 1;
        }

        offset
    }
}

static QUOTE_CHARS: [char, .. 2] =  ['\'', '\"'];

impl XPathTokenizer {
    pub fn new(xpath: & str) -> XPathTokenizer {
        XPathTokenizer {
            xpath: XPathString::new(xpath),
            start: 0,
            prefer_recognition_of_operator_names: false,
        }
    }

    pub fn has_more_tokens(& self) -> bool {
        self.xpath.len() > self.start
    }

    fn two_char_tokens(& self) -> HashMap<string::String, XPathToken> {
        let mut m = HashMap::new();
        m.insert("<=".to_string(), XPathToken::LessThanOrEqual);
        m.insert(">=".to_string(), XPathToken::GreaterThanOrEqual);
        m.insert("!=".to_string(), XPathToken::NotEqual);
        m.insert("::".to_string(), XPathToken::DoubleColon);
        m.insert("//".to_string(), XPathToken::DoubleSlash);
        m.insert("..".to_string(), XPathToken::ParentNode);
        m
    }

    fn single_char_tokens(&self) -> HashMap<char, XPathToken> {
        let mut m = HashMap::new();
        m.insert('/', XPathToken::Slash);
        m.insert('(', XPathToken::LeftParen);
        m.insert(')', XPathToken::RightParen);
        m.insert('[', XPathToken::LeftBracket);
        m.insert(']', XPathToken::RightBracket);
        m.insert('@', XPathToken::AtSign);
        m.insert('$', XPathToken::DollarSign);
        m.insert('+', XPathToken::PlusSign);
        m.insert('-', XPathToken::MinusSign);
        m.insert('|', XPathToken::Pipe);
        m.insert('=', XPathToken::Equal);
        m.insert('<', XPathToken::LessThan);
        m.insert('>', XPathToken::GreaterThan);
        m
    }

    fn named_operators(& self) -> Vec<(& 'static str, XPathToken)> {
        vec!(("and", XPathToken::And),
             ("or",  XPathToken::Or),
             ("mod", XPathToken::Remainder),
             ("div", XPathToken::Divide),
             ("*",   XPathToken::Multiply))
    }

    fn tokenize_literal(& mut self, quote_char: char) -> TokenResult {
        let mut offset = self.start;

        offset += 1; // Skip over the starting quote
        let start_of_string = offset;

        offset = self.xpath.while_not_character(offset, quote_char);
        let end_of_string = offset;

        if self.xpath.char_at_is_not(offset, quote_char) {
            return Err(MismatchedQuoteCharacters);
        }
        offset += 1; // Skip over ending quote

        self.start = offset;
        return Ok(XPathToken::Literal(self.xpath.substr(start_of_string, end_of_string)));
    }

    fn raw_next_token(& mut self) -> TokenResult {
        if let Some(first_two) = self.xpath.safe_substr(self.start, self.start + 2) {
            if let Some(token) = self.two_char_tokens().get(&first_two) {
                self.start += 2;
                return Ok(token.clone());
            }
        }

        let c = self.xpath.char_at(self.start);

        if let Some(token) = self.single_char_tokens().get(&c) {
            self.start += 1;
            return Ok(token.clone());
        }

        for quote_char in QUOTE_CHARS.iter() {
            if *quote_char == c {
                return self.tokenize_literal(*quote_char);
            }
        }

        if '.' == c {
            if self.xpath.char_at_is_not_digit(self.start + 1) {
                // Ugly. Should we use START / FOLLOW constructs?
                self.start += 1;
                return Ok(XPathToken::CurrentNode);
            }
        }

        if is_number_char(c) {
            let mut offset = self.start;
            let current_start = self.start;

            offset = self.xpath.while_valid_number(offset);

            self.start = offset;
            let substr = self.xpath.substr(current_start, offset);
            match from_str(substr.as_slice()) {
                Some(value) => Ok(XPathToken::Number(value)),
                None => panic!("Not really a number!")
            }
        } else {
            let mut offset = self.start;
            let current_start = self.start;

            if self.prefer_recognition_of_operator_names {
                for &(ref name, ref token) in self.named_operators().iter() {
                    let name_chars: Vec<char> = name.chars().collect();
                    let name_chars_slice = name_chars.as_slice();

                    if self.xpath.str_at_is(offset, name_chars_slice) {
                        self.start += name_chars.len();
                        return Ok(token.clone());
                    }
                }
            }

            if self.xpath.char_at_is(offset, '*') {
                self.start = offset + 1;
                return Ok(XPathToken::String("*".to_string()));
            }

            offset = self.xpath.while_valid_string(offset);

            if self.xpath.char_at_is(offset, ':') && self.xpath.char_at_is_not(offset + 1, ':') {
                let prefix = self.xpath.substr(current_start, offset);

                offset += 1;

                let current_start = offset;
                offset = self.xpath.while_valid_string(offset);

                if current_start == offset {
                    return Err(MissingLocalName);
                }

                let name = self.xpath.substr(current_start, offset);

                self.start = offset;
                return Ok(XPathToken::PrefixedName(prefix, name));

            } else {
                self.start = offset;
                return Ok(XPathToken::String(self.xpath.substr(current_start, offset)));
            }
        }
    }

    fn consume_whitespace(& mut self) {
        self.start = self.xpath.end_of_whitespace(self.start);
    }

    fn next_token(& mut self) -> TokenResult {
        self.consume_whitespace();

        let old_start = self.start;
        let token = self.raw_next_token();
        if token.is_err() { return token; }

        let token = token.unwrap();

        if old_start == self.start {
            return Err(UnableToCreateToken);
        }

        self.consume_whitespace();

        if ! (token.precedes_node_test() ||
              token.precedes_expression() ||
              token.is_operator()) {
            // See http://www.w3.org/TR/xpath/#exprlex
            self.prefer_recognition_of_operator_names = true;
        } else {
            self.prefer_recognition_of_operator_names = false;
        }

        return Ok(token);
    }
}

impl Iterator<TokenResult> for XPathTokenizer {
    fn next(&mut self) -> Option<TokenResult> {
        if self.has_more_tokens() {
            Some(self.next_token())
        } else {
            None
        }
    }
}

fn is_number_char(c: char) -> bool {
    return is_digit(c) || '.' == c;
}

pub struct XPathTokenDisambiguator<T, I> {
    source: ::std::iter::Peekable<T, I>,
}

impl<T, I: Iterator<T>> XPathTokenDisambiguator<T, I> {
    pub fn new(source: I) -> XPathTokenDisambiguator<T, I> {
        XPathTokenDisambiguator{
            source: source.peekable(),
        }
    }
}

static NODE_TEST_NAMES : [&'static str, .. 4] =
    [ "comment", "text", "processing-instruction", "node" ];

impl<I: Iterator<TokenResult>> Iterator<TokenResult> for XPathTokenDisambiguator<TokenResult, I> {
    fn next(&mut self) -> Option<TokenResult> {
        let token = self.source.next();
        let next  = self.source.peek();

        match (token, next) {
            (Some(Ok(XPathToken::String(val))), Some(&Ok(XPathToken::LeftParen))) => {
                if NODE_TEST_NAMES.contains(&val.as_slice()) {
                    Some(Ok(XPathToken::NodeTest(val)))
                } else {
                    Some(Ok(XPathToken::Function(val)))
                }
            },
            (Some(Ok(XPathToken::String(val))), Some(&Ok(XPathToken::DoubleColon))) => {
                Some(Ok(XPathToken::Axis(val)))
            },
            (token, _) => token,
        }
    }
}

pub struct XPathTokenDeabbreviator<I> {
    source: I,
    buffer: Vec<XPathToken>,
}

impl<I> XPathTokenDeabbreviator<I> {
    pub fn new(source: I) -> XPathTokenDeabbreviator<I> {
        XPathTokenDeabbreviator {
            source: source,
            buffer: vec!(),
        }
    }

    fn push(&mut self, token: XPathToken) {
        self.buffer.push(token);
    }

    fn expand_token(&mut self, token: XPathToken) {
        match token {
            XPathToken::AtSign => {
                self.push(XPathToken::String("attribute".to_string()));
                self.push(XPathToken::DoubleColon);
            }
            XPathToken::DoubleSlash => {
                self.push(XPathToken::Slash);
                self.push(XPathToken::String("descendant-or-self".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
                self.push(XPathToken::Slash);
            }
            XPathToken::CurrentNode => {
                self.push(XPathToken::String("self".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
            }
            XPathToken::ParentNode => {
                self.push(XPathToken::String("parent".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
            }
            _ => {
                self.push(token);
            }
        }
    }
}

impl<I: Iterator<TokenResult>> Iterator<TokenResult> for XPathTokenDeabbreviator<I> {
    fn next(&mut self) -> Option<TokenResult> {
        if self.buffer.is_empty() {
            let token = self.source.next();

            match token {
                None => return token,
                Some(Err(_)) => return token,
                Some(Ok(token)) => self.expand_token(token),
            }
        }

        match self.buffer.remove(0) {
            Some(t) => Some(Ok(t)),
            None => panic!("No tokens left to return"), // Can't happen, we always add one
        }
    }
}
<commit_msg>Create tokenization lookup tables only once per tokenizer<commit_after>use std::collections::HashMap;
use std::string;

use self::TokenizerErr::*;

use super::token::XPathToken;
use super::token::XPathToken::*;

fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

pub struct XPathTokenizer {
    xpath: XPathString,
    start: uint,
    prefer_recognition_of_operator_names: bool,
    single_char_tokens: HashMap<char, XPathToken>,
    two_char_tokens: HashMap<&'static str, XPathToken>,
    named_operators: Vec<(&'static str, XPathToken)>,
}

pub type TokenResult = Result<XPathToken, TokenizerErr>;

#[deriving(Show,PartialEq,Clone,Copy)]
pub enum TokenizerErr {
    MissingLocalName,
    MismatchedQuoteCharacters,
    UnableToCreateToken,
}

struct XPathString {
    xpath: Vec<char>,
}

impl XPathString {
    fn new(xpath: &str) -> XPathString {
        XPathString {
            xpath: xpath.chars().collect(),
        }
    }

    fn len(& self) -> uint {
        self.xpath.len()
    }

    fn str_at_is(& self, offset: uint, needle: &[char]) -> bool {
        let s_len = needle.len();

        if self.xpath.len() < offset + s_len { return false; }

        let xpath_chars = self.xpath.slice(offset, offset + s_len);

        needle == xpath_chars
    }

    fn valid_ncname_start_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if c >= 'A' && c <= 'Z' { return true }
        if c == '_' { return true }
        if c >= 'a' && c <= 'z' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn valid_ncname_follow_char(& self, offset: uint) -> bool {
        let c = self.xpath[offset];
        if self.valid_ncname_start_char(offset) { return true }
        if c == '-' { return true }
        if c == '.' { return true }
        if c >= '0' && c <= '9' { return true }
        // TODO: All non-ASCII codepoints
        return false;
    }

    fn while_valid_string(& self, offset: uint) -> uint {
        let mut offset = offset;

        if offset < self.xpath.len() && self.valid_ncname_start_char(offset) {
            offset += 1;

            while offset < self.xpath.len() && self.valid_ncname_follow_char(offset) {
                offset += 1;
            }
        }

        return offset;
    }

    fn while_valid_number(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && is_number_char(self.xpath[offset]) {
            offset += 1;
        }

        return offset;
    }

    fn while_not_character(& self, offset: uint, end_char: char) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.xpath[offset] != end_char {
            offset += 1;
        }

        return offset;
    }

    fn substr(& self, start: uint, end: uint) -> string::String {
        string::String::from_chars(self.xpath.slice(start, end))
    }

    fn safe_substr(& self, start: uint, end: uint) -> Option<string::String> {
        if self.xpath.len() >= end {
            Some(self.substr(start, end))
        } else {
            None
        }
    }

    fn char_at(&self, offset: uint) -> char {
        self.xpath[offset]
    }

    fn char_at_is(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        has_one_more && self.xpath[offset] == c
    }

    fn char_at_is_not(&self, offset: uint, c: char) -> bool {
        let has_one_more = self.xpath.len() >= offset + 1;

        ! has_one_more || self.xpath[offset] != c
    }

    fn char_at_is_not_digit(& self, offset: uint) -> bool {
        let has_more_chars = self.xpath.len() >= offset + 1;

        ! has_more_chars || ! is_digit(self.xpath[offset])
    }

    fn is_xml_space(&self, offset: uint) -> bool {
        let c = self.xpath[offset];

        return
            c == ' '  ||
            c == '\t' ||
            c == '\n' ||
            c == '\r';
    }

    fn end_of_whitespace(& self, offset: uint) -> uint {
        let mut offset = offset;

        while offset < self.xpath.len() && self.is_xml_space(offset) {
            offset += 1;
        }

        offset
    }
}

static QUOTE_CHARS: [char, .. 2] =  ['\'', '\"'];

impl XPathTokenizer {
    pub fn new(xpath: & str) -> XPathTokenizer {
        let single_char_tokens = {
            let mut m = HashMap::new();
            m.insert('/', XPathToken::Slash);
            m.insert('(', XPathToken::LeftParen);
            m.insert(')', XPathToken::RightParen);
            m.insert('[', XPathToken::LeftBracket);
            m.insert(']', XPathToken::RightBracket);
            m.insert('@', XPathToken::AtSign);
            m.insert('$', XPathToken::DollarSign);
            m.insert('+', XPathToken::PlusSign);
            m.insert('-', XPathToken::MinusSign);
            m.insert('|', XPathToken::Pipe);
            m.insert('=', XPathToken::Equal);
            m.insert('<', XPathToken::LessThan);
            m.insert('>', XPathToken::GreaterThan);
            m
        };

        let two_char_tokens = {
            let mut m = HashMap::new();
            m.insert("<=", XPathToken::LessThanOrEqual);
            m.insert(">=", XPathToken::GreaterThanOrEqual);
            m.insert("!=", XPathToken::NotEqual);
            m.insert("::", XPathToken::DoubleColon);
            m.insert("//", XPathToken::DoubleSlash);
            m.insert("..", XPathToken::ParentNode);
            m
        };

        let named_operators = vec![
            ("and", XPathToken::And),
            ("or",  XPathToken::Or),
            ("mod", XPathToken::Remainder),
            ("div", XPathToken::Divide),
            ("*",   XPathToken::Multiply)
        ];

        XPathTokenizer {
            xpath: XPathString::new(xpath),
            start: 0,
            prefer_recognition_of_operator_names: false,
            single_char_tokens: single_char_tokens,
            two_char_tokens: two_char_tokens,
            named_operators: named_operators,
        }
    }

    pub fn has_more_tokens(& self) -> bool {
        self.xpath.len() > self.start
    }

    fn tokenize_literal(& mut self, quote_char: char) -> TokenResult {
        let mut offset = self.start;

        offset += 1; // Skip over the starting quote
        let start_of_string = offset;

        offset = self.xpath.while_not_character(offset, quote_char);
        let end_of_string = offset;

        if self.xpath.char_at_is_not(offset, quote_char) {
            return Err(MismatchedQuoteCharacters);
        }
        offset += 1; // Skip over ending quote

        self.start = offset;
        return Ok(XPathToken::Literal(self.xpath.substr(start_of_string, end_of_string)));
    }

    fn raw_next_token(& mut self) -> TokenResult {
        if let Some(first_two) = self.xpath.safe_substr(self.start, self.start + 2) {
            if let Some(token) = self.two_char_tokens.get(first_two.as_slice()) {
                self.start += 2;
                return Ok(token.clone());
            }
        }

        let c = self.xpath.char_at(self.start);

        if let Some(token) = self.single_char_tokens.get(&c) {
            self.start += 1;
            return Ok(token.clone());
        }

        for quote_char in QUOTE_CHARS.iter() {
            if *quote_char == c {
                return self.tokenize_literal(*quote_char);
            }
        }

        if '.' == c {
            if self.xpath.char_at_is_not_digit(self.start + 1) {
                // Ugly. Should we use START / FOLLOW constructs?
                self.start += 1;
                return Ok(XPathToken::CurrentNode);
            }
        }

        if is_number_char(c) {
            let mut offset = self.start;
            let current_start = self.start;

            offset = self.xpath.while_valid_number(offset);

            self.start = offset;
            let substr = self.xpath.substr(current_start, offset);
            match from_str(substr.as_slice()) {
                Some(value) => Ok(XPathToken::Number(value)),
                None => panic!("Not really a number!")
            }
        } else {
            let mut offset = self.start;
            let current_start = self.start;

            if self.prefer_recognition_of_operator_names {
                for &(ref name, ref token) in self.named_operators.iter() {
                    let name_chars: Vec<char> = name.chars().collect();
                    let name_chars_slice = name_chars.as_slice();

                    if self.xpath.str_at_is(offset, name_chars_slice) {
                        self.start += name_chars.len();
                        return Ok(token.clone());
                    }
                }
            }

            if self.xpath.char_at_is(offset, '*') {
                self.start = offset + 1;
                return Ok(XPathToken::String("*".to_string()));
            }

            offset = self.xpath.while_valid_string(offset);

            if self.xpath.char_at_is(offset, ':') && self.xpath.char_at_is_not(offset + 1, ':') {
                let prefix = self.xpath.substr(current_start, offset);

                offset += 1;

                let current_start = offset;
                offset = self.xpath.while_valid_string(offset);

                if current_start == offset {
                    return Err(MissingLocalName);
                }

                let name = self.xpath.substr(current_start, offset);

                self.start = offset;
                return Ok(XPathToken::PrefixedName(prefix, name));

            } else {
                self.start = offset;
                return Ok(XPathToken::String(self.xpath.substr(current_start, offset)));
            }
        }
    }

    fn consume_whitespace(& mut self) {
        self.start = self.xpath.end_of_whitespace(self.start);
    }

    fn next_token(& mut self) -> TokenResult {
        self.consume_whitespace();

        let old_start = self.start;
        let token = self.raw_next_token();
        if token.is_err() { return token; }

        let token = token.unwrap();

        if old_start == self.start {
            return Err(UnableToCreateToken);
        }

        self.consume_whitespace();

        if ! (token.precedes_node_test() ||
              token.precedes_expression() ||
              token.is_operator()) {
            // See http://www.w3.org/TR/xpath/#exprlex
            self.prefer_recognition_of_operator_names = true;
        } else {
            self.prefer_recognition_of_operator_names = false;
        }

        return Ok(token);
    }
}

impl Iterator<TokenResult> for XPathTokenizer {
    fn next(&mut self) -> Option<TokenResult> {
        if self.has_more_tokens() {
            Some(self.next_token())
        } else {
            None
        }
    }
}

fn is_number_char(c: char) -> bool {
    return is_digit(c) || '.' == c;
}

pub struct XPathTokenDisambiguator<T, I> {
    source: ::std::iter::Peekable<T, I>,
}

impl<T, I: Iterator<T>> XPathTokenDisambiguator<T, I> {
    pub fn new(source: I) -> XPathTokenDisambiguator<T, I> {
        XPathTokenDisambiguator{
            source: source.peekable(),
        }
    }
}

static NODE_TEST_NAMES : [&'static str, .. 4] =
    [ "comment", "text", "processing-instruction", "node" ];

impl<I: Iterator<TokenResult>> Iterator<TokenResult> for XPathTokenDisambiguator<TokenResult, I> {
    fn next(&mut self) -> Option<TokenResult> {
        let token = self.source.next();
        let next  = self.source.peek();

        match (token, next) {
            (Some(Ok(XPathToken::String(val))), Some(&Ok(XPathToken::LeftParen))) => {
                if NODE_TEST_NAMES.contains(&val.as_slice()) {
                    Some(Ok(XPathToken::NodeTest(val)))
                } else {
                    Some(Ok(XPathToken::Function(val)))
                }
            },
            (Some(Ok(XPathToken::String(val))), Some(&Ok(XPathToken::DoubleColon))) => {
                Some(Ok(XPathToken::Axis(val)))
            },
            (token, _) => token,
        }
    }
}

pub struct XPathTokenDeabbreviator<I> {
    source: I,
    buffer: Vec<XPathToken>,
}

impl<I> XPathTokenDeabbreviator<I> {
    pub fn new(source: I) -> XPathTokenDeabbreviator<I> {
        XPathTokenDeabbreviator {
            source: source,
            buffer: vec!(),
        }
    }

    fn push(&mut self, token: XPathToken) {
        self.buffer.push(token);
    }

    fn expand_token(&mut self, token: XPathToken) {
        match token {
            XPathToken::AtSign => {
                self.push(XPathToken::String("attribute".to_string()));
                self.push(XPathToken::DoubleColon);
            }
            XPathToken::DoubleSlash => {
                self.push(XPathToken::Slash);
                self.push(XPathToken::String("descendant-or-self".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
                self.push(XPathToken::Slash);
            }
            XPathToken::CurrentNode => {
                self.push(XPathToken::String("self".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
            }
            XPathToken::ParentNode => {
                self.push(XPathToken::String("parent".to_string()));
                self.push(XPathToken::DoubleColon);
                self.push(XPathToken::String("node".to_string()));
                self.push(XPathToken::LeftParen);
                self.push(XPathToken::RightParen);
            }
            _ => {
                self.push(token);
            }
        }
    }
}

impl<I: Iterator<TokenResult>> Iterator<TokenResult> for XPathTokenDeabbreviator<I> {
    fn next(&mut self) -> Option<TokenResult> {
        if self.buffer.is_empty() {
            let token = self.source.next();

            match token {
                None => return token,
                Some(Err(_)) => return token,
                Some(Ok(token)) => self.expand_token(token),
            }
        }

        match self.buffer.remove(0) {
            Some(t) => Some(Ok(t)),
            None => panic!("No tokens left to return"), // Can't happen, we always add one
        }
    }
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);
                    self.tape.next_symbol();
                    self.tape.cut();

                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn will_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<commit_msg>Make behaviour be manual skipping<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A tokenizer is a pattern matcher that is intended to turn a stream of symbols into another stream of symbols based on the patterns
//! that are matched. Every pattern can produce a different output symbol. If two input strings can ndfa in two different output
//! symbols, then the output symbol that is ordered lower is the one that's produced (ie, if the output symbols are numbers, then '0' will
//! be produced instead of '1' in the event of a clash)
//!

use super::countable::*;
use super::symbol_range::*;
use super::regular_pattern::*;
use super::state_machine::*;
use super::ndfa::*;
use super::prepare::*;
use super::symbol_range_dfa::*;
use super::symbol_reader::*;
use super::pattern_matcher::*;
use super::matches::*;
use super::tape::*;

///
/// Used for generating tokenizing pattern matchers
///
pub struct TokenMatcher<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord> {
    patterns: Vec<(Pattern<InputSymbol>, OutputSymbol)>
}

impl<InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> TokenMatcher<InputSymbol, OutputSymbol> {
    ///
    /// Creates a new TokenMatcher
    ///
    pub fn new() -> TokenMatcher<InputSymbol, OutputSymbol> {
        TokenMatcher { patterns: vec![] }
    }

    ///
    /// Adds a new pattern that will generate the specified output symbol
    ///
    pub fn add_pattern<TPattern: ToPattern<InputSymbol>>(&mut self, pattern: TPattern, output: OutputSymbol) {
        self.patterns.push((pattern.to_pattern(), output));
    }

    ///
    /// Compiles an NDFA from this TokenMatcher
    ///
    pub fn to_ndfa(&self) -> Box<StateMachine<SymbolRange<InputSymbol>, OutputSymbol>> {
        let mut ndfa = Ndfa::new();

        for &(ref pattern, ref output) in &self.patterns {
            // Compile each pattern starting at state 0
            let end_state = pattern.compile(&mut ndfa, 0);

            // Set the output for this pattern
            ndfa.set_output_symbol(end_state, output.clone());
        }

        // Clear out any overlapping ranges so we can build a valid DFA
        ndfa.fix_overlapping_ranges();

        Box::new(ndfa)
    }
}

impl<'a, InputSymbol: Clone+Ord+Countable+'static, OutputSymbol: Clone+Ord+'static> PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>
for &'a TokenMatcher<InputSymbol, OutputSymbol> {
    #[inline]
    fn prepare_to_match(self) -> SymbolRangeDfa<InputSymbol, OutputSymbol> {
        let ndfa = self.to_ndfa();

        ndfa.prepare_to_match()
    }
}

///
/// A tokenizer is a type of symbol stream that uses a pattern matcher to convert a symbol stream into a stream of tokens
///
pub struct Tokenizer<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> {
    /// The pattern matcher for this tokenizer
    dfa: SymbolRangeDfa<InputSymbol, OutputSymbol>,

    /// Tape of input symbols that will be used to generate the result
    tape: Tape<InputSymbol, Reader>,
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord, Reader: SymbolReader<InputSymbol>> Tokenizer<InputSymbol, OutputSymbol, Reader> {
    ///
    /// Creates a new tokenizer from a pattern (usually a TokenMatcher)
    ///
    pub fn new<Prepare: PrepareToMatch<SymbolRangeDfa<InputSymbol, OutputSymbol>>>(source: Reader, pattern: Prepare) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern.prepare_to_match(), tape: Tape::new(source) }
    }

    ///
    /// Creates a new tokenizer from a prepared pattern
    ///
    pub fn new_prepared(source: Reader, pattern: SymbolRangeDfa<InputSymbol, OutputSymbol>) -> Tokenizer<InputSymbol, OutputSymbol, Reader> {
        Tokenizer { dfa: pattern, tape: Tape::new(source) }
    }

    ///
    /// Returns the current position in the source (the position after the last matched symbol)
    ///
    pub fn get_source_position(&self) -> usize {
        self.tape.get_source_position()
    }

    ///
    /// Skips an input symbol (returning the symbol that was skipped)
    ///
    pub fn skip_input(&mut self) -> Option<InputSymbol> {
        self.tape.next_symbol()
    }

    ///
    /// True if we've reached the end of the source reader
    ///
    /// If `next_symbol` returns `None` and `at_end_of_reader` is false, then the input stream does not contain a symbol matching the DFA
    ///
    pub fn at_end_of_reader(&self) -> bool {
        self.tape.at_end_of_reader()
    }
}

impl<InputSymbol: Clone+Ord+Countable, OutputSymbol: Clone+Ord+'static, Reader: SymbolReader<InputSymbol>> SymbolReader<OutputSymbol> for Tokenizer<InputSymbol, OutputSymbol, Reader> {
    fn next_symbol(&mut self) -> Option<OutputSymbol> {
        // Start of the next symbol
        let start_pos = self.tape.get_source_position();

        // Match against it
        let match_result = match_pattern(self.dfa.start(), &mut self.tape);

        let end_pos = self.tape.get_source_position();
        match match_result {
            Accept(length, outputsymbol) => {
                if length > 0 {
                    // Rewind the tape to after the accepted symbol
                    self.tape.rewind(end_pos-start_pos - length);

                    // Won't try to match anything before this position
                    self.tape.cut();

                    // Result is the oputput symbol
                    Some(outputsymbol.clone())
                } else {
                    // Zero-length match
                    // If we accepted matches of length 0 we'd get an infinite stream when we hit a symbol that doesn't match
                    self.tape.rewind(end_pos-start_pos);

                    // Return no match
                    None
                }
            },

            Reject => {
                // Rewind back to the start position
                self.tape.rewind(end_pos-start_pos);

                // No match
                None
            },

            _ => {
                panic!("Unexpected output state of ");
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_match_tokens_like_any_other_pattern() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        assert!(matches("aaaa", &token_matcher) == Some(4));
        assert!(matches("bbbbb", &token_matcher) == Some(5));
        assert!(matches("abbb", &token_matcher) == Some(1));
        assert!(matches("bbaaa", &token_matcher) == Some(2));
    }

    #[test]
    fn can_distinguish_simple_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            AllAs,
            AllBs
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1), TestToken::AllAs);
        token_matcher.add_pattern("b".repeat_forever(1), TestToken::AllBs);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaaaa".read_symbols()).is_accepted(&TestToken::AllAs));
        assert!(match_pattern(matcher.start(), &mut "bbbb".read_symbols()).is_accepted(&TestToken::AllBs));
    }

    #[test]
    fn clashes_producer_lower_tokens() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Abbb,
            Aaab
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern("a".repeat_forever(1).append("b"), TestToken::Aaab);
        token_matcher.add_pattern("a".append("b".repeat_forever(1)), TestToken::Abbb);

        let matcher = token_matcher.prepare_to_match();

        assert!(match_pattern(matcher.start(), &mut "aaab".read_symbols()).is_accepted(&TestToken::Aaab));
        assert!(match_pattern(matcher.start(), &mut "ab".read_symbols()).is_accepted(&TestToken::Abbb));
        assert!(match_pattern(matcher.start(), &mut "abbbb".read_symbols()).is_accepted(&TestToken::Abbb));
    }

    #[test]
    fn can_match_number_stream() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 390  32 ".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 2);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 3);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 6);
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.get_source_position() == 8);
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.get_source_position() == 10);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == None);
    }

    #[test]
    fn will_skip_bad_symbols() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(1), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(1), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }

    #[test]
    fn wont_match_zero_length() {
        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
        enum TestToken {
            Digit,
            Whitespace
        }

        let mut token_matcher = TokenMatcher::new();
        token_matcher.add_pattern(MatchRange('0', '9').repeat_forever(0), TestToken::Digit);
        token_matcher.add_pattern(" ".repeat_forever(0), TestToken::Whitespace);

        let mut tokenizer = Tokenizer::new("12 ab 12".read_symbols(), &token_matcher);

        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('a'));
        assert!(tokenizer.next_symbol() == None);
        assert!(!tokenizer.at_end_of_reader());
        assert!(tokenizer.skip_input() == Some('b'));
        assert!(tokenizer.next_symbol() == Some(TestToken::Whitespace));
        assert!(tokenizer.next_symbol() == Some(TestToken::Digit));
        assert!(tokenizer.next_symbol() == None);
        assert!(tokenizer.at_end_of_reader());
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The CGMath Developers. For a full listing of the authors,
// refer to the Cargo.toml file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::fmt;

use structure::*;

use approx::ApproxEq;
use matrix::{Matrix2, Matrix3, Matrix4};
use num::{BaseFloat, BaseNum};
use point::{Point2, Point3};
use rotation::*;
use vector::{Vector2, Vector3};

/// A trait representing an [affine
/// transformation](https://en.wikipedia.org/wiki/Affine_transformation) that
/// can be applied to points or vectors. An affine transformation is one which
pub trait Transform<P: EuclideanSpace>: Sized {
    /// Create an identity transformation. That is, a transformation which
    /// does nothing.
    fn one() -> Self;

    /// Create a transformation that rotates a vector to look at `center` from
    /// `eye`, using `up` for orientation.
    fn look_at(eye: P, center: P, up: P::Diff) -> Self;

    /// Transform a vector using this transform.
    fn transform_vector(&self, vec: P::Diff) -> P::Diff;

    /// Transform a point using this transform.
    fn transform_point(&self, point: P) -> P;

    /// Transform a vector as a point using this transform.
    #[inline]
    fn transform_as_point(&self, vec: P::Diff) -> P::Diff {
        self.transform_point(P::from_vec(vec)).to_vec()
    }

    /// Combine this transform with another, yielding a new transformation
    /// which has the effects of both.
    fn concat(&self, other: &Self) -> Self;

    /// Create a transform that "un-does" this one.
    fn inverse_transform(&self) -> Option<Self>;

    /// Combine this transform with another, in-place.
    #[inline]
    fn concat_self(&mut self, other: &Self) {
        *self = Self::concat(self, other);
    }
}

/// A generic transformation consisting of a rotation,
/// displacement vector and scale amount.
#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable)]
pub struct Decomposed<V: VectorSpace, R> {
    pub scale: V::Scalar,
    pub rot: R,
    pub disp: V,
}

impl<P: EuclideanSpace, R: Rotation<P>> Transform<P> for Decomposed<P::Diff, R> where
    // FIXME: Ugly type signatures - blocked by rust-lang/rust#24092
    <P as EuclideanSpace>::Scalar: BaseFloat,
    // FIXME: Investigate why this is needed!
    <P as EuclideanSpace>::Diff: VectorSpace,
{
    #[inline]
    fn one() -> Decomposed<P::Diff, R> {
        Decomposed {
            scale: P::Scalar::one(),
            rot: R::one(),
            disp: P::Diff::zero(),
        }
    }

    #[inline]
    fn look_at(eye: P, center: P, up: P::Diff) -> Decomposed<P::Diff, R> {
        let rot = R::look_at(center - eye, up);
        let disp = rot.rotate_vector(P::origin() - eye);
        Decomposed {
            scale: P::Scalar::one(),
            rot: rot,
            disp: disp,
        }
    }

    #[inline]
    fn transform_vector(&self, vec: P::Diff) -> P::Diff {
        self.rot.rotate_vector(vec * self.scale)
    }

    #[inline]
    fn transform_point(&self, point: P) -> P {
        self.rot.rotate_point(point * self.scale) + self.disp
    }

    fn concat(&self, other: &Decomposed<P::Diff, R>) -> Decomposed<P::Diff, R> {
        Decomposed {
            scale: self.scale * other.scale,
            rot: self.rot.concat(&other.rot),
            disp: self.transform_as_point(other.disp.clone()),
        }
    }

    fn inverse_transform(&self) -> Option<Decomposed<P::Diff, R>> {
        if self.scale.approx_eq(&P::Scalar::zero()) {
            None
        } else {
            let s = P::Scalar::one() / self.scale;
            let r = self.rot.invert();
            let d = r.rotate_vector(self.disp.clone()) * -s;
            Some(Decomposed {
                scale: s,
                rot: r,
                disp: d,
            })
        }
    }
}

pub trait Transform2<S: BaseNum>: Transform<Point2<S>> + Into<Matrix3<S>> {}
pub trait Transform3<S: BaseNum>: Transform<Point3<S>> + Into<Matrix4<S>> {}

impl<S: BaseFloat, R: Rotation2<S>> From<Decomposed<Vector2<S>, R>> for Matrix3<S> {
    fn from(dec: Decomposed<Vector2<S>, R>) -> Matrix3<S> {
        let m: Matrix2<_> = dec.rot.into();
        let mut m: Matrix3<_> = (&m * dec.scale).into();
        m.z = dec.disp.extend(S::one());
        m
    }
}

impl<S: BaseFloat, R: Rotation3<S>> From<Decomposed<Vector3<S>, R>> for Matrix4<S> {
    fn from(dec: Decomposed<Vector3<S>, R>) -> Matrix4<S> {
        let m: Matrix3<_> = dec.rot.into();
        let mut m: Matrix4<_> = (&m * dec.scale).into();
        m.w = dec.disp.extend(S::one());
        m
    }
}

impl<S: BaseFloat, R: Rotation2<S>> Transform2<S> for Decomposed<Vector2<S>, R> {}

impl<S: BaseFloat, R: Rotation3<S>> Transform3<S> for Decomposed<Vector3<S>, R> {}

/// A homogeneous transformation matrix.
#[derive(Copy, Clone, RustcEncodable, RustcDecodable)]
pub struct AffineMatrix3<S> {
    pub mat: Matrix4<S>,
}

impl<S: BaseFloat> Transform<Point3<S>> for AffineMatrix3<S> {
    #[inline]
    fn one() -> AffineMatrix3<S> {
       AffineMatrix3 { mat: Matrix4::identity() }
    }

    #[inline]
    fn look_at(eye: Point3<S>, center: Point3<S>, up: Vector3<S>) -> AffineMatrix3<S> {
        AffineMatrix3 { mat: Matrix4::look_at(eye, center, up) }
    }

    #[inline]
    fn transform_vector(&self, vec: Vector3<S>) -> Vector3<S> {
        (self.mat * vec.extend(S::zero())).truncate()
    }

    #[inline]
    fn transform_point(&self, point: Point3<S>) -> Point3<S> {
        Point3::from_homogeneous(self.mat * point.to_homogeneous())
    }

    #[inline]
    fn concat(&self, other: &AffineMatrix3<S>) -> AffineMatrix3<S> {
        AffineMatrix3 { mat: self.mat * other.mat }
    }

    #[inline]
    fn inverse_transform(&self) -> Option<AffineMatrix3<S>> {
        SquareMatrix::invert(& self.mat).map(|m| AffineMatrix3{ mat: m })
    }
}

impl<S: BaseNum> From<AffineMatrix3<S>> for Matrix4<S> {
    #[inline] fn from(aff: AffineMatrix3<S>) -> Matrix4<S> { aff.mat }
}

impl<S: BaseFloat> Transform3<S> for AffineMatrix3<S> {}

impl<S: fmt::Debug> fmt::Debug for AffineMatrix3<S> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(write!(f, "AffineMatrix3 "));
        <[[S; 4]; 4] as fmt::Debug>::fmt(self.mat.as_ref(), f)
    }
}
<commit_msg>Remove AffineMatrix3<commit_after>// Copyright 2014 The CGMath Developers. For a full listing of the authors,
// refer to the Cargo.toml file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use structure::*;

use approx::ApproxEq;
use matrix::{Matrix2, Matrix3, Matrix4};
use num::{BaseFloat, BaseNum};
use point::{Point2, Point3};
use rotation::*;
use vector::{Vector2, Vector3};

/// A trait representing an [affine
/// transformation](https://en.wikipedia.org/wiki/Affine_transformation) that
/// can be applied to points or vectors. An affine transformation is one which
pub trait Transform<P: EuclideanSpace>: Sized {
    /// Create an identity transformation. That is, a transformation which
    /// does nothing.
    fn one() -> Self;

    /// Create a transformation that rotates a vector to look at `center` from
    /// `eye`, using `up` for orientation.
    fn look_at(eye: P, center: P, up: P::Diff) -> Self;

    /// Transform a vector using this transform.
    fn transform_vector(&self, vec: P::Diff) -> P::Diff;

    /// Transform a point using this transform.
    fn transform_point(&self, point: P) -> P;

    /// Transform a vector as a point using this transform.
    #[inline]
    fn transform_as_point(&self, vec: P::Diff) -> P::Diff {
        self.transform_point(P::from_vec(vec)).to_vec()
    }

    /// Combine this transform with another, yielding a new transformation
    /// which has the effects of both.
    fn concat(&self, other: &Self) -> Self;

    /// Create a transform that "un-does" this one.
    fn inverse_transform(&self) -> Option<Self>;

    /// Combine this transform with another, in-place.
    #[inline]
    fn concat_self(&mut self, other: &Self) {
        *self = Self::concat(self, other);
    }
}

/// A generic transformation consisting of a rotation,
/// displacement vector and scale amount.
#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable)]
pub struct Decomposed<V: VectorSpace, R> {
    pub scale: V::Scalar,
    pub rot: R,
    pub disp: V,
}

impl<P: EuclideanSpace, R: Rotation<P>> Transform<P> for Decomposed<P::Diff, R> where
    // FIXME: Ugly type signatures - blocked by rust-lang/rust#24092
    <P as EuclideanSpace>::Scalar: BaseFloat,
    // FIXME: Investigate why this is needed!
    <P as EuclideanSpace>::Diff: VectorSpace,
{
    #[inline]
    fn one() -> Decomposed<P::Diff, R> {
        Decomposed {
            scale: P::Scalar::one(),
            rot: R::one(),
            disp: P::Diff::zero(),
        }
    }

    #[inline]
    fn look_at(eye: P, center: P, up: P::Diff) -> Decomposed<P::Diff, R> {
        let rot = R::look_at(center - eye, up);
        let disp = rot.rotate_vector(P::origin() - eye);
        Decomposed {
            scale: P::Scalar::one(),
            rot: rot,
            disp: disp,
        }
    }

    #[inline]
    fn transform_vector(&self, vec: P::Diff) -> P::Diff {
        self.rot.rotate_vector(vec * self.scale)
    }

    #[inline]
    fn transform_point(&self, point: P) -> P {
        self.rot.rotate_point(point * self.scale) + self.disp
    }

    fn concat(&self, other: &Decomposed<P::Diff, R>) -> Decomposed<P::Diff, R> {
        Decomposed {
            scale: self.scale * other.scale,
            rot: self.rot.concat(&other.rot),
            disp: self.transform_as_point(other.disp.clone()),
        }
    }

    fn inverse_transform(&self) -> Option<Decomposed<P::Diff, R>> {
        if self.scale.approx_eq(&P::Scalar::zero()) {
            None
        } else {
            let s = P::Scalar::one() / self.scale;
            let r = self.rot.invert();
            let d = r.rotate_vector(self.disp.clone()) * -s;
            Some(Decomposed {
                scale: s,
                rot: r,
                disp: d,
            })
        }
    }
}

pub trait Transform2<S: BaseNum>: Transform<Point2<S>> + Into<Matrix3<S>> {}
pub trait Transform3<S: BaseNum>: Transform<Point3<S>> + Into<Matrix4<S>> {}

impl<S: BaseFloat, R: Rotation2<S>> From<Decomposed<Vector2<S>, R>> for Matrix3<S> {
    fn from(dec: Decomposed<Vector2<S>, R>) -> Matrix3<S> {
        let m: Matrix2<_> = dec.rot.into();
        let mut m: Matrix3<_> = (&m * dec.scale).into();
        m.z = dec.disp.extend(S::one());
        m
    }
}

impl<S: BaseFloat, R: Rotation3<S>> From<Decomposed<Vector3<S>, R>> for Matrix4<S> {
    fn from(dec: Decomposed<Vector3<S>, R>) -> Matrix4<S> {
        let m: Matrix3<_> = dec.rot.into();
        let mut m: Matrix4<_> = (&m * dec.scale).into();
        m.w = dec.disp.extend(S::one());
        m
    }
}

impl<S: BaseFloat, R: Rotation2<S>> Transform2<S> for Decomposed<Vector2<S>, R> {}

impl<S: BaseFloat, R: Rotation3<S>> Transform3<S> for Decomposed<Vector3<S>, R> {}
<|endoftext|>


<commit_before>use Request;

use std::io::Read;
use std::error::Error;

/// Request and response transport abstraction.
///
/// The `Transport` trait provides a way to send a `Request` to a server and to receive the
/// corresponding response. A `Transport` implementor is passed to `Request::call` in order to use
/// it to perform that request.
///
/// The most commonly used transport is simple HTTP: If the `reqwest` feature is enabled (it is by
/// default), the reqwest `RequestBuilder` will implement this trait and send the XML-RPC `Request`
/// via HTTP.
///
/// You can implement this trait for your own types if you want to customize how requests are sent.
/// You can modify HTTP headers or wrap requests in a completely different protocol.
pub trait Transport {
    // FIXME replace with `impl Trait` when stable
    /// The response stream returned by `transmit`.
    type Stream: Read;

    /// Transmits an XML-RPC request and returns the server's response.
    ///
    /// The response is returned as a `Self::Stream` - some type implementing the `Read` trait. The
    /// library will read all of the data and parse it as a response. It must be UTF-8 encoded XML,
    /// otherwise the call will fail.
    ///
    /// If a transport error occurs, this should return it as a boxed error - the library will
    /// interpret it as a transport error and return an appropriate `RequestError`.
    fn transmit(self, request: &Request) -> Result<Self::Stream, Box<Error>>;
}

/// Provides helpers for implementing custom `Transport`s using reqwest.
///
/// This module will be disabled if the `reqwest` feature is not enabled.
///
/// The default `Transport` implementation for `RequestBuilder` looks roughly like this:
///
/// ```notrust
/// // serialize request into `body` (a `Vec<u8>`)
///
/// build_headers(builder, body.len());
///
/// // send `body` using `builder`, ensure server returned 200 OK
///
/// check_response(&response)?;
/// ```
///
/// From this, you can build your own custom transports.
#[cfg(feature = "reqwest")]
pub mod http {
    extern crate reqwest;

    use {Request, Transport};
    use self::reqwest::{RequestBuilder, mime};
    use self::reqwest::header::{ContentType, ContentLength, UserAgent};

    use std::error::Error;

    /// Appends all HTTP headers required by the XML-RPC specification to the `RequestBuilder`.
    ///
    /// More specifically, the following headers are set:
    ///
    /// ```notrust
    /// User-Agent: Rust xmlrpc
    /// Content-Type: text/xml; charset=utf-8
    /// Content-Length: $body_len
    /// ```
    pub fn build_headers(builder: &mut RequestBuilder, body_len: u64) {
        // Set all required request headers
        // NB: The `Host` header is also required, but reqwest adds it automatically, since
        // HTTP/1.1 requires it.
        builder
            .header(UserAgent::new("Rust xmlrpc"))
            .header(ContentType("text/xml; charset=utf-8".parse().unwrap()))
            .header(ContentLength(body_len));
    }

    /// Checks that a reqwest `Response` contains all headers mandated by the spec.
    ///
    /// This is done by default to ensure compliance.
    pub fn check_response(response: &reqwest::Response) -> Result<(), Box<Error>> {
        // Check response headers
        // "The Content-Type is text/xml. Content-Length must be present and correct."
        if let Some(content) = response.headers().get::<ContentType>() {
            // (we ignore this if the header is missing completely)
            match (content.type_(), content.subtype()) {
                (mime::TEXT, mime::XML) => {},
                (ty, sub) => return Err(
                    format!("expected Content-Type 'text/xml', got '{}/{}'", ty, sub).into()
                ),
            }
        }

        response.headers().get::<ContentLength>()
            .ok_or_else(|| format!("expected Content-Length header, but none was found"))?;

        Ok(())
    }

    /// Use a `RequestBuilder` as the transport.
    ///
    /// The request will be sent as specified in the XML-RPC specification: A default `User-Agent`
    /// will be set, along with the correct `Content-Type` and `Content-Length`.
    impl Transport for RequestBuilder {
        type Stream = reqwest::Response;

        fn transmit(mut self, request: &Request) -> Result<Self::Stream, Box<Error>> {
            // First, build the body XML
            let mut body = Vec::new();
            // This unwrap never panics as we are using `Vec<u8>` as a `Write` implementor,
            // and not doing anything else that could return an `Err` in `write_as_xml()`.
            request.write_as_xml(&mut body).unwrap();

            build_headers(&mut self, body.len() as u64);

            let response = self
                .body(body)
                .send()?
                .error_for_status()?;

            check_response(&response)?;

            Ok(response)
        }
    }
}
<commit_msg>Make `check_response` also check the status code<commit_after>use Request;

use std::io::Read;
use std::error::Error;

/// Request and response transport abstraction.
///
/// The `Transport` trait provides a way to send a `Request` to a server and to receive the
/// corresponding response. A `Transport` implementor is passed to `Request::call` in order to use
/// it to perform that request.
///
/// The most commonly used transport is simple HTTP: If the `reqwest` feature is enabled (it is by
/// default), the reqwest `RequestBuilder` will implement this trait and send the XML-RPC `Request`
/// via HTTP.
///
/// You can implement this trait for your own types if you want to customize how requests are sent.
/// You can modify HTTP headers or wrap requests in a completely different protocol.
pub trait Transport {
    // FIXME replace with `impl Trait` when stable
    /// The response stream returned by `transmit`.
    type Stream: Read;

    /// Transmits an XML-RPC request and returns the server's response.
    ///
    /// The response is returned as a `Self::Stream` - some type implementing the `Read` trait. The
    /// library will read all of the data and parse it as a response. It must be UTF-8 encoded XML,
    /// otherwise the call will fail.
    ///
    /// If a transport error occurs, this should return it as a boxed error - the library will
    /// interpret it as a transport error and return an appropriate `RequestError`.
    fn transmit(self, request: &Request) -> Result<Self::Stream, Box<Error>>;
}

// FIXME: Link to `Transport` and `RequestBuilder` using intra-rustdoc links. Relative links break
// everything and abs. links don't work locally.
/// Provides helpers for implementing custom `Transport`s using reqwest.
///
/// This module will be disabled if the `reqwest` feature is not enabled.
///
/// The default [`Transport`][transport] implementation for `RequestBuilder` looks roughly like
/// this:
///
/// ```notrust
/// // serialize request into `body` (a `Vec<u8>`)
///
/// build_headers(builder, body.len());
///
/// // send `body` using `builder` and get response
///
/// check_response(&response)?;
/// ```
///
/// From this, you can build your own custom transports.
///
/// [transport]: trait.Transport.html
#[cfg(feature = "reqwest")]
pub mod http {
    extern crate reqwest;

    use {Request, Transport};
    use self::reqwest::{RequestBuilder, mime};
    use self::reqwest::header::{ContentType, ContentLength, UserAgent};

    use std::error::Error;

    /// Appends all HTTP headers required by the XML-RPC specification to the `RequestBuilder`.
    ///
    /// More specifically, the following headers are set:
    ///
    /// ```notrust
    /// User-Agent: Rust xmlrpc
    /// Content-Type: text/xml; charset=utf-8
    /// Content-Length: $body_len
    /// ```
    pub fn build_headers(builder: &mut RequestBuilder, body_len: u64) {
        // Set all required request headers
        // NB: The `Host` header is also required, but reqwest adds it automatically, since
        // HTTP/1.1 requires it.
        builder
            .header(UserAgent::new("Rust xmlrpc"))
            .header(ContentType("text/xml; charset=utf-8".parse().unwrap()))
            .header(ContentLength(body_len));
    }

    /// Checks that a reqwest `Response` has a status code indicating success and contains all
    /// headers mandated by the spec.
    ///
    /// This is done by default to ensure compliance.
    pub fn check_response(response: &reqwest::Response) -> Result<(), Box<Error>> {
        // This is essentially an open-coded version of `Response::error_for_status` that does not
        // consume the response.
        if response.status().is_client_error() || response.status().is_server_error() {
            return Err(format!("server response indicates error: {}", response.status()).into());
        }

        // Check response headers
        // "The Content-Type is text/xml. Content-Length must be present and correct."
        if let Some(content) = response.headers().get::<ContentType>() {
            // (we ignore this if the header is missing completely)
            match (content.type_(), content.subtype()) {
                (mime::TEXT, mime::XML) => {},
                (ty, sub) => return Err(
                    format!("expected Content-Type 'text/xml', got '{}/{}'", ty, sub).into()
                ),
            }
        }

        response.headers().get::<ContentLength>()
            .ok_or_else(|| format!("expected Content-Length header, but none was found"))?;

        Ok(())
    }

    /// Use a `RequestBuilder` as the transport.
    ///
    /// The request will be sent as specified in the XML-RPC specification: A default `User-Agent`
    /// will be set, along with the correct `Content-Type` and `Content-Length`.
    impl Transport for RequestBuilder {
        type Stream = reqwest::Response;

        fn transmit(mut self, request: &Request) -> Result<Self::Stream, Box<Error>> {
            // First, build the body XML
            let mut body = Vec::new();
            // This unwrap never panics as we are using `Vec<u8>` as a `Write` implementor,
            // and not doing anything else that could return an `Err` in `write_as_xml()`.
            request.write_as_xml(&mut body).unwrap();

            build_headers(&mut self, body.len() as u64);

            let response = self
                .body(body)
                .send()?;

            check_response(&response)?;

            Ok(response)
        }
    }
}
<|endoftext|>


<commit_before>//! Customisable, user-implementable traversals for tries.
//!
//! This module contains 4 traits that allow you to write your own recursive trie traversals. Each
//! trait handles the key-splitting logic and result handling required to write a traversal. All of
//! the library's important operations are implemented as traversals (insert, remove, get_ancestor,
//! etc).
//!
//! The trait is basically a massively higher-order function. The `Input` type allows you
//! to define values to be passed down the traversal as it proceeds, while the `Output` type
//! allows you to propogate a value back up the call stack. Look at the source for `run`.
//!
//! # Which Trait?
//!
//! * `Traversal` - immutable trie, references not allowed in `Output` type.
//! * `RefTraversal` - immutable trie, references allowed in `Output` type.
//! * `TraversalMut` - mutable trie, references not allowed in `Output` type.
//! * `RefTraversalMut` - mutable trie, references allowed in `Output` type, no `action_fn`.

use {Trie, TrieKey, NibbleVec};
use keys::{match_keys, KeyMatch};

/// Identity macro to allow expansion of the "mutability" token tree.
#[macro_export]
macro_rules! id {
    ($e:item) => { $e }
}

macro_rules! if_else {
    (false, $x:expr, $y:expr) => { $y };
    (true, $x:expr, $y:expr) => { $x };
}

// FIXME: Use () default input type once 1.2/1.3 lands.
macro_rules! make_traversal_trait {
    (
        name: $name:ident,
        trie_type: $trie_type:ty,
        with_action: $with_action:tt,
        mutability: $($mut_:tt)*
    ) => { id! {
pub trait $name<'a, K: 'a, V: 'a> where K: TrieKey {
    type Input: 'a;
    type Output;

    fn default_result() -> Self::Output;

    #[allow(unused)]
    fn match_fn(trie: $trie_type, input: Self::Input) -> Self::Output {
        Self::default_result()
    }

    #[allow(unused)]
    fn no_child_fn
    (
        trie: $trie_type, input: Self::Input,
        nv: NibbleVec, bucket: usize
    ) -> Self::Output {
        Self::default_result()
    }

    #[doc = "Defaults to `match_fn`."]
    #[allow(unused)]
    fn child_match_fn(child: $trie_type, input: Self::Input, nv: NibbleVec) -> Self::Output {
        Self::match_fn(child, input)
    }

    #[allow(unused)]
    fn partial_match_fn(child: $trie_type, input: Self::Input, nv: NibbleVec, idx: usize)
    -> Self::Output {
        Self::default_result()
    }

    #[allow(unused)]
    fn first_prefix_fn(trie: $trie_type, input: Self::Input, nv: NibbleVec) -> Self::Output {
        Self::default_result()
    }

    // FIXME: Don't generate action_fn at all (cf. Rust issue #4621).
    #[doc = "Note: this function isn't called in a `RefTraversalMut`."]
    #[allow(unused)]
    fn action_fn(trie: $trie_type, intermediate: Self::Output, bucket: usize) -> Self::Output {
        intermediate
    }

    #[doc = "Run the traversal, returning the result."]
    fn run(trie: $trie_type, input: Self::Input, mut key_fragments: NibbleVec) -> Self::Output {
        if key_fragments.len() == 0 {
            return Self::match_fn(trie, input);
        }

        let bucket = key_fragments.get(0) as usize;

        let intermediate = match trie.children[bucket] {
            None => return Self::no_child_fn(trie, input, key_fragments, bucket),
            Some(ref $($mut_)* child) => {
                match match_keys(&key_fragments, &child.key) {
                    KeyMatch::Full =>
                        Self::child_match_fn(child, input, key_fragments),
                    KeyMatch::Partial(i) =>
                        Self::partial_match_fn(child, input, key_fragments, i),
                    KeyMatch::FirstPrefix =>
                        Self::first_prefix_fn(child, input, key_fragments),
                    KeyMatch::SecondPrefix => {
                        let new_key = key_fragments.split(child.key.len());
                        Self::run(child, input, new_key)
                    }
                }
            }
        };

        if_else!($with_action, Self::action_fn(trie, intermediate, bucket), intermediate)
    }
} // end trait.
}} // end macro body.
}

make_traversal_trait!(name: Traversal, trie_type: &Trie<K, V>, with_action: true, mutability: );
make_traversal_trait!(name: RefTraversal, trie_type: &'a Trie<K, V>, with_action: true, mutability: );

make_traversal_trait!(name: TraversalMut, trie_type: &mut Trie<K, V>, with_action: true, mutability: mut);
make_traversal_trait!(name: RefTraversalMut, trie_type: &'a mut Trie<K, V>, with_action: false, mutability: mut);
<commit_msg>Update note about using default types.<commit_after>//! Customisable, user-implementable traversals for tries.
//!
//! This module contains 4 traits that allow you to write your own recursive trie traversals. Each
//! trait handles the key-splitting logic and result handling required to write a traversal. All of
//! the library's important operations are implemented as traversals (insert, remove, get_ancestor,
//! etc).
//!
//! The trait is basically a massively higher-order function. The `Input` type allows you
//! to define values to be passed down the traversal as it proceeds, while the `Output` type
//! allows you to propogate a value back up the call stack. Look at the source for `run`.
//!
//! # Which Trait?
//!
//! * `Traversal` - immutable trie, references not allowed in `Output` type.
//! * `RefTraversal` - immutable trie, references allowed in `Output` type.
//! * `TraversalMut` - mutable trie, references not allowed in `Output` type.
//! * `RefTraversalMut` - mutable trie, references allowed in `Output` type, no `action_fn`.

use {Trie, TrieKey, NibbleVec};
use keys::{match_keys, KeyMatch};

/// Identity macro to allow expansion of the "mutability" token tree.
#[macro_export]
macro_rules! id {
    ($e:item) => { $e }
}

macro_rules! if_else {
    (false, $x:expr, $y:expr) => { $y };
    (true, $x:expr, $y:expr) => { $x };
}

// NOTE: Use () default input type once `associated_type_defaults` stabilises.
macro_rules! make_traversal_trait {
    (
        name: $name:ident,
        trie_type: $trie_type:ty,
        with_action: $with_action:tt,
        mutability: $($mut_:tt)*
    ) => { id! {
pub trait $name<'a, K: 'a, V: 'a> where K: TrieKey {
    type Input: 'a;
    type Output;

    fn default_result() -> Self::Output;

    #[allow(unused)]
    fn match_fn(trie: $trie_type, input: Self::Input) -> Self::Output {
        Self::default_result()
    }

    #[allow(unused)]
    fn no_child_fn
    (
        trie: $trie_type, input: Self::Input,
        nv: NibbleVec, bucket: usize
    ) -> Self::Output {
        Self::default_result()
    }

    #[doc = "Defaults to `match_fn`."]
    #[allow(unused)]
    fn child_match_fn(child: $trie_type, input: Self::Input, nv: NibbleVec) -> Self::Output {
        Self::match_fn(child, input)
    }

    #[allow(unused)]
    fn partial_match_fn(child: $trie_type, input: Self::Input, nv: NibbleVec, idx: usize)
    -> Self::Output {
        Self::default_result()
    }

    #[allow(unused)]
    fn first_prefix_fn(trie: $trie_type, input: Self::Input, nv: NibbleVec) -> Self::Output {
        Self::default_result()
    }

    // NOTE: Don't generate action_fn at all (cf. Rust issue #4621).
    #[doc = "Note: this function isn't called in a `RefTraversalMut`."]
    #[allow(unused)]
    fn action_fn(trie: $trie_type, intermediate: Self::Output, bucket: usize) -> Self::Output {
        intermediate
    }

    #[doc = "Run the traversal, returning the result."]
    fn run(trie: $trie_type, input: Self::Input, mut key_fragments: NibbleVec) -> Self::Output {
        if key_fragments.len() == 0 {
            return Self::match_fn(trie, input);
        }

        let bucket = key_fragments.get(0) as usize;

        let intermediate = match trie.children[bucket] {
            None => return Self::no_child_fn(trie, input, key_fragments, bucket),
            Some(ref $($mut_)* child) => {
                match match_keys(&key_fragments, &child.key) {
                    KeyMatch::Full =>
                        Self::child_match_fn(child, input, key_fragments),
                    KeyMatch::Partial(i) =>
                        Self::partial_match_fn(child, input, key_fragments, i),
                    KeyMatch::FirstPrefix =>
                        Self::first_prefix_fn(child, input, key_fragments),
                    KeyMatch::SecondPrefix => {
                        let new_key = key_fragments.split(child.key.len());
                        Self::run(child, input, new_key)
                    }
                }
            }
        };

        if_else!($with_action, Self::action_fn(trie, intermediate, bucket), intermediate)
    }
} // end trait.
}} // end macro body.
}

make_traversal_trait!(name: Traversal, trie_type: &Trie<K, V>, with_action: true, mutability: );
make_traversal_trait!(name: RefTraversal, trie_type: &'a Trie<K, V>, with_action: true, mutability: );

make_traversal_trait!(name: TraversalMut, trie_type: &mut Trie<K, V>, with_action: true, mutability: mut);
make_traversal_trait!(name: RefTraversalMut, trie_type: &'a mut Trie<K, V>, with_action: false, mutability: mut);
<|endoftext|>


<commit_before><commit_msg>refactor: Simplify compiler_pipeline<commit_after><|endoftext|>


<commit_before>// of course this method is too slow
// pub fn reverse_pairs(nums: Vec<i32>) -> i32 {
//     if nums.len() < 2 {
//         return 0;
//     }

//     let nums: Vec<i64> = nums.iter().map(|&s| s as i64).collect();

//     let mut count = 0;
//     for i in 0..nums.len() {
//         for j in 0..i {
//             if nums[j] > nums[i] * 2 {
//                 count += 1
//             }
//         }
//     }
//     count
// }

use std::cell::RefCell;
use std::rc::Rc;
#[derive(Debug)]
struct BST {
    this: i64,
    count: usize, // count number of larger or equal number in this tree
    left: Option<Rc<RefCell<BST>>>,

    right: Option<Rc<RefCell<BST>>>,
}

impl BST {
    fn new(n: i64) -> Self {
        BST {
            this: n,
            count: 1,
            left: None,
            right: None,
        }
    }

    fn insert(&mut self, n: i64) {
        if n > self.this {
            self.count += 1;
            if let Some(t) = self.right.as_ref() {
                t.borrow_mut().insert(n);
            } else {
                self.right = Some(Rc::new(RefCell::new(BST::new(n))))
            }
        } else if n < self.this {
            if let Some(t) = self.left.as_ref() {
                t.borrow_mut().insert(n);
            } else {
                self.left = Some(Rc::new(RefCell::new(BST::new(n))))
            }
        } else {
            self.count += 1;
        }
    }

    fn search(&self, n: i64) -> usize {
        if self.this == n {
            self.count
        } else if n < self.this {
            if let None = self.left {
                self.count
            } else {
                self.count + self.left.as_ref().unwrap().borrow().search(n)
            }
        } else {
            if let None = self.right {
                0
            } else {
                self.right.as_ref().unwrap().borrow().search(n)
            }
        }
    }
}

// this method is O(n^2)
pub fn reverse_pairs(nums: Vec<i32>) -> i32 {
    if nums.len() < 2 {
        return 0;
    }

    let mut nums = nums.iter().map(|&s| s as i64);

    let mut count = 0;
    let mut bst = BST::new(nums.next().unwrap());

    while let Some(n) = nums.next() {
        count += bst.search(2 * n + 1);
        bst.insert(n);
    }

    count as i32
}

fn main() {
    // assert_eq!(2, reverse_pairs(vec![1, 3, 2, 3, 1]));
    // assert_eq!(3, reverse_pairs(vec![2, 4, 3, 5, 1]));
    // assert_eq!(0, reverse_pairs(vec![1]));
    // assert_eq!(4, reverse_pairs(vec![5, 4, 3, 2, 1]));
    // assert_eq!(
    //     0,
    //     reverse_pairs(vec![
    //         2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647
    //     ])
    // );
    assert_eq!(
        9,
        reverse_pairs(vec![
            2147483647,
            2147483647,
            -2147483647,
            -2147483647,
            -2147483647,
            2147483647
        ])
    );

    assert_eq!(
        40,
        reverse_pairs(vec![
            233, 2000000001, 234, 2000000006, 235, 2000000003, 236, 2000000007, 237, 2000000002,
            2000000005, 233, 233, 233, 233, 233, 2000000004
        ])
    );
}
<commit_msg>finish BIT version<commit_after>// of course this method is too slow
// pub fn reverse_pairs(nums: Vec<i32>) -> i32 {
//     if nums.len() < 2 {
//         return 0;
//     }

//     let nums: Vec<i64> = nums.iter().map(|&s| s as i64).collect();

//     let mut count = 0;
//     for i in 0..nums.len() {
//         for j in 0..i {
//             if nums[j] > nums[i] * 2 {
//                 count += 1
//             }
//         }
//     }
//     count
// }

///////////////////////////////////////
//////////////////////////////////////

// also BTS is slow
// use std::cell::RefCell;
// use std::rc::Rc;
// #[derive(Debug)]
// struct BST {
//     this: i64,
//     count: usize, // count number of larger or equal number in this tree
//     left: Option<Rc<RefCell<BST>>>,

//     right: Option<Rc<RefCell<BST>>>,
// }

// impl BST {
//     fn new(n: i64) -> Self {
//         BST {
//             this: n,
//             count: 1,
//             left: None,
//             right: None,
//         }
//     }

//     fn insert(&mut self, n: i64) {
//         if n > self.this {
//             self.count += 1;
//             if let Some(t) = self.right.as_ref() {
//                 t.borrow_mut().insert(n);
//             } else {
//                 self.right = Some(Rc::new(RefCell::new(BST::new(n))))
//             }
//         } else if n < self.this {
//             if let Some(t) = self.left.as_ref() {
//                 t.borrow_mut().insert(n);
//             } else {
//                 self.left = Some(Rc::new(RefCell::new(BST::new(n))))
//             }
//         } else {
//             self.count += 1;
//         }
//     }

//     fn search(&self, n: i64) -> usize {
//         if self.this == n {
//             self.count
//         } else if n < self.this {
//             if let None = self.left {
//                 self.count
//             } else {
//                 self.count + self.left.as_ref().unwrap().borrow().search(n)
//             }
//         } else {
//             if let None = self.right {
//                 0
//             } else {
//                 self.right.as_ref().unwrap().borrow().search(n)
//             }
//         }
//     }
// }

// this method is O(n^2)
// pub fn reverse_pairs(nums: Vec<i32>) -> i32 {
//     if nums.len() < 2 {
//         return 0;
//     }

//     let mut nums = nums.iter().map(|&s| s as i64);

//     let mut count = 0;
//     let mut bst = BST::new(nums.next().unwrap());

//     while let Some(n) = nums.next() {
//         count += bst.search(2 * n + 1);
//         bst.insert(n);
//     }

//     count as i32
// }

////////////////////////////////////////////////
///////////////////////////////////////////////

use std::io::{Error, ErrorKind, Result};
use std::ops::{Add, AddAssign};

#[derive(Debug)]
struct BIT<T> {
    inner: Vec<T>,
}

/// https://zh.wikipedia.org/wiki/树状数组
impl<T> BIT<T>
where
    T: Add + Default + Clone + Copy + AddAssign,
{
    fn new(a: &Vec<T>) -> Self {
        let mut inner = vec![Default::default(); a.len() + 1];

        for i in 1..=a.len() {
            inner[i] = a[i - 1];
            let mut j = i as i32 - 2;
            while j >= i as i32 - ((i as i32) & (-(i as i32))) {
                inner[i] += a[j as usize];
                j -= 1;
            }
        }

        Self { inner }
    }

    fn update(&mut self, mut ind: usize, delta: T) -> Result<()> {
        let len = self.inner.len();
        if ind >= len {
            return Err(Error::new(
                ErrorKind::InvalidInput,
                "ind beyond inner vec length",
            ));
        }

        while ind < len {
            self.inner[ind] += delta;
            ind += ((ind as i32) & (-(ind as i32))) as usize
        }

        Ok(())
    }

    fn sum(&self, ind: usize) -> Result<T> {
        let len = self.inner.len();
        if ind >= len {
            return Err(Error::new(
                ErrorKind::InvalidInput,
                "ind beyond inner vec length",
            ));
        }

        let mut result = Default::default();
        let mut ind = ind as i32;
        while ind > 0 {
            result += self.inner[ind as usize];
            ind -= (ind as i32) & (-(ind as i32))
        }

        Ok(result)
    }
}

pub fn reverse_pairs(nums: Vec<i32>) -> i32 {
    use std::collections::{HashMap, HashSet};
    if nums.len() < 2 {
        return 0;
    }

    // incase i32 overflow
    let nums: Vec<i64> = nums.iter().map(|&s| s as i64).collect();

    // make a new nums
    let mut new_nums = nums.clone();
    new_nums.append(&mut nums.iter().cloned().map(|x| x * 2).collect());

    let mut set: Vec<i64> = new_nums
        .iter()
        .collect::<HashSet<&i64>>()
        .iter()
        .map(|&&x| x)
        .collect();
    set.sort();

    // make map for faster query
    let position_map = {
        let mut map: HashMap<i64, usize> = HashMap::new();
        for (ind, n) in set.iter().enumerate() {
            map.insert(*n, ind);
        }
        map
    };

    let mut bit = BIT {
        inner: vec![0; set.len() + 1],
    };

    let mut count = 0;

    for i in (0..nums.len()).rev() {
        // BIT sum/query, position of nums[i] in BIT[i+1]
        // So sum from BIT[i-1]
        count += bit.sum(*position_map.get(&nums[i]).unwrap()).unwrap();
        bit.update(*position_map.get(&(nums[i] * 2)).unwrap() + 1, 1)
            .unwrap();
    }

    count
}

fn main() {
    assert_eq!(2, reverse_pairs(vec![1, 3, 2, 3, 1]));
    assert_eq!(3, reverse_pairs(vec![2, 4, 3, 5, 1]));
    assert_eq!(0, reverse_pairs(vec![1]));
    assert_eq!(4, reverse_pairs(vec![5, 4, 3, 2, 1]));
    assert_eq!(
        0,
        reverse_pairs(vec![
            2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647
        ])
    );
    assert_eq!(
        9,
        reverse_pairs(vec![
            2147483647,
            2147483647,
            -2147483647,
            -2147483647,
            -2147483647,
            2147483647
        ])
    );

    assert_eq!(
        40,
        reverse_pairs(vec![
            233, 2000000001, 234, 2000000006, 235, 2000000003, 236, 2000000007, 237, 2000000002,
            2000000005, 233, 233, 233, 233, 233, 2000000004
        ])
    );
}
<|endoftext|>