<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    SymbolCannotBeNumber,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }

    fn next_symbol(&mut self, start: char) -> TokenResult {
        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else {
                try!(self.scanner.unget_char()
                    .map_err(|e| TokenError::ScannerError(e.to_owned())));
                break;
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol.chars().next().map_or(false, |c| c.is_digit(10)) {
            symbol.parse::<f64>()
                .map(|value| Token::Number(value))
                .map_err(|_| TokenError::SymbolCannotBeNumber)
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            '\\' => Token::Backslash,
            '(' => Token::ParenOpen,
            ')' => Token::ParenClose,
            '"' => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.next_symbol(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);

        test_number("1233.14", 1233.14f64);

        test_number("3.14159e-3", 3.14159e-3f64);
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 1e-1;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::SymbolCannotBeNumber);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "a12[$3abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        // Ignore first token
        assert!(tokenizer.next().unwrap().is_ok());
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().unwrap().is_ok());
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<commit_msg>Grabbed the tests from the other branch for numeric literal parsing<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    SymbolCannotBeNumber,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }

    fn next_symbol(&mut self, start: char) -> TokenResult {
        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else {
                try!(self.scanner.unget_char()
                    .map_err(|e| TokenError::ScannerError(e.to_owned())));
                break;
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol.chars().next().map_or(false, |c| c.is_digit(10)) {
            symbol.parse::<f64>()
                .map(|value| Token::Number(value))
                .map_err(|_| TokenError::SymbolCannotBeNumber)
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            '\\' => Token::Backslash,
            '(' => Token::ParenOpen,
            ')' => Token::ParenClose,
            '"' => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.next_symbol(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);

        test_number("1233.14", 1233.14f64);

        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('1'));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        // Ignore first token
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';
const BACKSLASH_CHAR: char = '\\';
const PAREN_OPEN_CHAR: char = '(';
const PAREN_CLOSE_CHAR: char = ')';
const STRING_BOUNDARY_CHAR: char = '"';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '=',
    ].into_iter().collect();

    static ref TOKEN_BOUNDARY_CHARS: HashSet<char> = vec![
        EOL_CHAR,
        BACKSLASH_CHAR,
        PAREN_OPEN_CHAR,
        PAREN_CLOSE_CHAR,
        STRING_BOUNDARY_CHAR,
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    InvalidNumericLiteral,
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                self.scanner.unget_char();
                break;
            }
        }
    }

    fn match_advanced_token(&mut self, start: char) -> TokenResult {
        if self.must_match_numeric(start) {
            return self.match_numeric(start);
        }

        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::UnrecognizedCharacter(c));
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn must_match_numeric(&mut self, start: char) -> bool {
        // Text must match a numeric literal if it could possibly
        // be interpreted as a number.
        // More formally, text must match a number if any of the
        // following conditions are met:
        // 1. start is a digit
        // 2. start is either '-' or '.' and the second character is a digit
        // 3. start is '-', the second character is '.' and the third character is a digit
        //TODO: There must be some way to optimize this significantly
        if start.is_digit(10) {
            return true;
        }

        let start_dash = start == '-';
        let start_dot = start == '.';

        let second = self.scanner.get_char();
        // Cannot undo this mutation right away just in case we need to get a third character after this

        let second_is_digit = second.map_or(false, |c| c.is_digit(10));

        if (start_dash || start_dot) && second_is_digit {
            // Undo any extra mutations that occurred
            if !second.is_none() { self.scanner.unget_char(); }
            return true;
        }

        let third = self.scanner.get_char();
        // Undo any extra mutations that occurred
        if !third.is_none() { self.scanner.unget_char(); }
        if !second.is_none() { self.scanner.unget_char(); }

        let second_is_dot = second.map_or(false, |c| c == '.');
        let third_is_digit = third.map_or(false, |c| c.is_digit(10));

        if start_dash && second_is_dot && third_is_digit {
            return true;
        }

        false
    }

    fn match_numeric(&mut self, start: char) -> TokenResult {
        let mut literal = String::new();
        literal.push(start);

        loop {
            let next_char = self.scanner.get_char();
            if next_char.is_none() {
                break;
            }

            let c = next_char.unwrap();
            // This is quite naive because parse() will do most of the work
            if c.is_digit(10) || c == '.' || c == 'e' || c == '-' {
                literal.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::InvalidNumericLiteral);
            }
        }

        literal.parse::<f64>()
            .map(|value| Token::Number(value))
            .map_err(|_| TokenError::InvalidNumericLiteral)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }

    fn is_token_boundary(&self, c: &char) -> bool {
        TOKEN_BOUNDARY_CHARS.contains(&c) || c.is_whitespace()
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        self.ignore_whitespace();

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            BACKSLASH_CHAR => Token::Backslash,
            PAREN_OPEN_CHAR => Token::ParenOpen,
            PAREN_CLOSE_CHAR => Token::ParenClose,
            STRING_BOUNDARY_CHAR => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.match_advanced_token(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);
        test_number("0.5", 0.5f64);
        test_number(".5", 0.5f64);
        test_number("-.5", -0.5f64);
        test_number("1233.14", 1233.14f64);
        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_in_symbol() {
        // This test is important because it may fail if numeric parsing changes
        let mut tokenizer = tokenizer_for("x - 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_only_minus_dot_in_symbol() {
        // This test is important because it may fail if numeric parsing changes since -.5 is a valid numeric literal
        let mut tokenizer = tokenizer_for("x -. 1");
        let expected = [
            Symbol("x".to_owned()),
            Symbol("-.".to_owned()),
            Number(1f64),
        ];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_numbers() {
        let symbol = "192.168.0.1".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_symbols_containing_numbers() {
        let symbol = "ms^-2";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<commit_msg>Refactoring tests to stop repeating so much code<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';
const BACKSLASH_CHAR: char = '\\';
const PAREN_OPEN_CHAR: char = '(';
const PAREN_CLOSE_CHAR: char = ')';
const STRING_BOUNDARY_CHAR: char = '"';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '=',
    ].into_iter().collect();

    static ref TOKEN_BOUNDARY_CHARS: HashSet<char> = vec![
        EOL_CHAR,
        BACKSLASH_CHAR,
        PAREN_OPEN_CHAR,
        PAREN_CLOSE_CHAR,
        STRING_BOUNDARY_CHAR,
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    InvalidNumericLiteral,
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                self.scanner.unget_char();
                break;
            }
        }
    }

    fn match_advanced_token(&mut self, start: char) -> TokenResult {
        if self.must_match_numeric(start) {
            return self.match_numeric(start);
        }

        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::UnrecognizedCharacter(c));
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn must_match_numeric(&mut self, start: char) -> bool {
        // Text must match a numeric literal if it could possibly
        // be interpreted as a number.
        // More formally, text must match a number if any of the
        // following conditions are met:
        // 1. start is a digit
        // 2. start is either '-' or '.' and the second character is a digit
        // 3. start is '-', the second character is '.' and the third character is a digit
        //TODO: There must be some way to optimize this significantly
        if start.is_digit(10) {
            return true;
        }

        let start_dash = start == '-';
        let start_dot = start == '.';

        let second = self.scanner.get_char();
        // Cannot undo this mutation right away just in case we need to get a third character after this

        let second_is_digit = second.map_or(false, |c| c.is_digit(10));

        if (start_dash || start_dot) && second_is_digit {
            // Undo any extra mutations that occurred
            if !second.is_none() { self.scanner.unget_char(); }
            return true;
        }

        let third = self.scanner.get_char();
        // Undo any extra mutations that occurred
        if !third.is_none() { self.scanner.unget_char(); }
        if !second.is_none() { self.scanner.unget_char(); }

        let second_is_dot = second.map_or(false, |c| c == '.');
        let third_is_digit = third.map_or(false, |c| c.is_digit(10));

        if start_dash && second_is_dot && third_is_digit {
            return true;
        }

        false
    }

    fn match_numeric(&mut self, start: char) -> TokenResult {
        let mut literal = String::new();
        literal.push(start);

        loop {
            let next_char = self.scanner.get_char();
            if next_char.is_none() {
                break;
            }

            let c = next_char.unwrap();
            // This is quite naive because parse() will do most of the work
            if c.is_digit(10) || c == '.' || c == 'e' || c == '-' {
                literal.push(c)
            }
            else if self.is_token_boundary(&c) {
                self.scanner.unget_char();
                break;
            }
            else {
                return Err(TokenError::InvalidNumericLiteral);
            }
        }

        literal.parse::<f64>()
            .map(|value| Token::Number(value))
            .map_err(|_| TokenError::InvalidNumericLiteral)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }

    fn is_token_boundary(&self, c: &char) -> bool {
        TOKEN_BOUNDARY_CHARS.contains(&c) || c.is_whitespace()
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        self.ignore_whitespace();

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            BACKSLASH_CHAR => Token::Backslash,
            PAREN_OPEN_CHAR => Token::ParenOpen,
            PAREN_CLOSE_CHAR => Token::ParenClose,
            STRING_BOUNDARY_CHAR => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.match_advanced_token(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        test_single_token("\\", Backslash);
    }

    #[test]
    fn equals() {
        test_single_token("=", Equals);
    }

    #[test]
    fn paren_open() {
        test_single_token("(", ParenOpen);
    }

    #[test]
    fn paren_close() {
        test_single_token(")", ParenClose);
    }

    #[test]
    fn string_boundary() {
        test_single_token("\"", StringBoundary);
    }

    #[test]
    fn eol() {
        test_single_token("\n", EOL);
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);
        test_number("0.5", 0.5f64);
        test_number(".5", 0.5f64);
        test_number("-.5", -0.5f64);
        test_number("1233.14", 1233.14f64);
        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn groups_characters_into_symbols() {
        test_symbol("abqq$$^&/|!");
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        test_symbol("-abqq$$^&/|!");
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        test_symbol("=ifsaoi=hfsdMMDS,,,,~~~:=");
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        test_symbol("abcdefghijklmnopqrstuvwxyzABCDEFGHIJ\
                    KLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=");
    }

    #[test]
    fn recognizes_symbols_containing_numbers() {
        test_symbol("ms^-2");
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        test_expected_tokens("\\ ) \n ( \\ = = )", &[
            Backslash,
            ParenClose,
            EOL,
            ParenOpen,
            Backslash,
            Equals,
            Equals,
            ParenClose
        ]);
    }

    #[test]
    fn allows_only_minus_in_symbol() {
        // This test is important because it may fail if numeric parsing changes
        test_expected_tokens("x - 1", &[
            Symbol("x".to_owned()),
            Symbol("-".to_owned()),
            Number(1f64),
        ]);
    }

    #[test]
    fn allows_only_minus_dot_in_symbol() {
        // This test is important because it may fail if numeric parsing changes since -.5 is a valid numeric literal
        test_expected_tokens("x -. 1", &[
            Symbol("x".to_owned()),
            Symbol("-.".to_owned()),
            Number(1f64),
        ]);
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        test_expected_tokens("(x + y)())", &[
            ParenOpen,
            Symbol("x".to_owned()),
            Symbol("+".to_owned()),
            Symbol("y".to_owned()),
            ParenClose,
            ParenOpen,
            ParenClose,
            ParenClose
        ]);
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        test_expected_tokens("\"x + y\"\"\"\"", &[
            StringBoundary,
            Symbol("x".to_owned()),
            Symbol("+".to_owned()),
            Symbol("y".to_owned()),
            StringBoundary,
            StringBoundary,
            StringBoundary,
            StringBoundary
        ]);
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        test_symbol_error("123abc~", TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        test_symbol_error("-123abc~", TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_numbers() {
        test_symbol_error("192.168.0.1", TokenError::InvalidNumericLiteral);
    }

    #[test]
    fn disallows_invalid_characters() {
        test_symbol_error("]", TokenError::UnrecognizedCharacter(']'));

        // Make sure invalid character is caught among valid characters too
        test_symbol_error("a[$abc~", TokenError::UnrecognizedCharacter('['));
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    // Test utility functions

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_single_token(input: &str, expected_token: Token) {
        let mut tokenizer = tokenizer_for(input);
        assert_eq!(tokenizer.next().unwrap().unwrap(), expected_token);
        assert!(tokenizer.next().is_none());
    }

    fn test_symbol(symbol: &str) {
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        // Ensure no extra tokens other than what is expected
        assert!(tokenizer.next().is_none());
    }

    fn test_symbol_error(symbol: &str, err: TokenError) {
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), err);
        // Behaviour of tokens after an error are undefined
    }

    fn test_expected_tokens(input: &str, expected: &[Token]) {
        let mut tokenizer = tokenizer_for(input);

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern mod freetype;

use font::{CSSFontWeight, FontHandleMethods, FontMetrics, FontTableMethods};
use font::{FontTableTag, FractionalPixel, SpecifiedFontStyle, UsedFontStyle, FontWeight100};
use font::{FontWeight200, FontWeight300, FontWeight400, FontWeight500, FontWeight600};
use font::{FontWeight700, FontWeight800, FontWeight900};
use geometry::Au;
use geometry;
use platform::font_context::FontContextHandle;
use text::glyph::GlyphIndex;
use text::util::{float_to_fixed, fixed_to_float};

use freetype::freetype::bindgen::{FT_Get_Char_Index, FT_Get_Postscript_Name};
use freetype::freetype::bindgen::{FT_Load_Glyph, FT_Set_Char_Size};
use freetype::freetype::bindgen::{FT_New_Face, FT_Get_Sfnt_Table};
use freetype::freetype::bindgen::{FT_New_Memory_Face, FT_Done_Face};
use freetype::freetype::{FTErrorMethods, FT_F26Dot6, FT_Face, FT_FaceRec};
use freetype::freetype::{FT_GlyphSlot, FT_Library, FT_Long, FT_ULong};
use freetype::freetype::{FT_STYLE_FLAG_ITALIC, FT_STYLE_FLAG_BOLD};
use freetype::freetype::{FT_SizeRec, FT_UInt, FT_Size_Metrics};
use freetype::freetype::{ft_sfnt_os2};
use freetype::tt_os2::TT_OS2;

fn float_to_fixed_ft(f: float) -> i32 {
    float_to_fixed(6, f)
}

fn fixed_to_float_ft(f: i32) -> float {
    fixed_to_float(6, f)
}

pub struct FontTable {
    bogus: ()
}

impl FontTableMethods for FontTable {
    fn with_buffer(&self, _blk: &fn(*u8, uint)) {
        fail!()
    }
}

enum FontSource {
    FontSourceMem(~[u8]),
    FontSourceFile(~str)
}

pub struct FontHandle {
    // The font binary. This must stay valid for the lifetime of the font,
    // if the font is created using FT_Memory_Face.
    source: FontSource,
    face: FT_Face,
}

#[unsafe_destructor]
impl Drop for FontHandle {
    fn finalize(&self) {
        assert!(self.face.is_not_null());
        if !FT_Done_Face(self.face).succeeded() {
            fail!(~"FT_Done_Face failed");
        }
    }
}

impl FontHandleMethods for FontHandle {
    pub fn new_from_buffer(fctx: &FontContextHandle,
                           buf: ~[u8],
                           style: &SpecifiedFontStyle)
                        -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let face_result = do vec::as_imm_buf(buf) |bytes: *u8, len: uint| {
            create_face_from_buffer(ft_ctx, bytes, len, style.pt_size)
        };

        // TODO: this could be more simply written as result::chain
        // and moving buf into the struct ctor, but cant' move out of
        // captured binding.
        return match face_result {
            Ok(face) => Ok(FontHandle { face: face, source: FontSourceMem(buf) }),
            Err(()) => Err(())
        };

         fn create_face_from_buffer(lib: FT_Library,
                                    cbuf: *u8, cbuflen: uint, pt_size: float)
             -> Result<FT_Face, ()> {

             let mut face: FT_Face = ptr::null();
             let face_index = 0 as FT_Long;
             let result = FT_New_Memory_Face(lib, cbuf, cbuflen as FT_Long,
                                             face_index, ptr::to_mut_unsafe_ptr(&mut face));

             if !result.succeeded() || face.is_null() {
                 return Err(());
             }
             if FontHandle::set_char_size(face, pt_size).is_ok() {
                 Ok(face)
             } else {
                 Err(())
             }
         }
    }

    // an identifier usable by FontContextHandle to recreate this FontHandle.
    fn face_identifier(&self) -> ~str {
        /* FT_Get_Postscript_Name seems like a better choice here, but it
           doesn't give usable results for fontconfig when deserializing. */
        unsafe { str::raw::from_c_str((*self.face).family_name) }
    }
    fn family_name(&self) -> ~str {
        unsafe { str::raw::from_c_str((*self.face).family_name) }
    }
    fn face_name(&self) -> ~str {
        unsafe { str::raw::from_c_str(FT_Get_Postscript_Name(self.face)) }
    }
    fn is_italic(&self) -> bool {
        unsafe { (*self.face).style_flags & FT_STYLE_FLAG_ITALIC != 0 }
    }
    fn boldness(&self) -> CSSFontWeight {
        let default_weight = FontWeight400;
        if unsafe { (*self.face).style_flags & FT_STYLE_FLAG_BOLD == 0 } {
            default_weight
        } else {
            let os2 = FT_Get_Sfnt_Table(self.face, ft_sfnt_os2) as *TT_OS2;
            let valid = os2.is_not_null() && unsafe { (*os2).version != 0xffff };
            if valid {
                let weight = unsafe { (*os2).usWeightClass };
                match weight {
                  1 | 100..199 => FontWeight100,
                  2 | 200..299 => FontWeight200,
                  3 | 300..399 => FontWeight300,
                  4 | 400..499 => FontWeight400,
                  5 | 500..599 => FontWeight500,
                  6 | 600..699 => FontWeight600,
                  7 | 700..799 => FontWeight700,
                  8 | 800..899 => FontWeight800,
                  9 | 900..999 => FontWeight900,
                  _ => default_weight
                }
            } else {
                default_weight
            }
        }
    }

    fn clone_with_style(&self,
                        fctx: &FontContextHandle,
                        style: &UsedFontStyle) -> Result<FontHandle, ()> {
        match self.source {
            FontSourceMem(ref buf) => {
                FontHandleMethods::new_from_buffer(fctx, buf.clone(), style)
            }
            FontSourceFile(copy file) => {
                FontHandle::new_from_file(fctx, file, style)
            }
        }
    }

    pub fn glyph_index(&self,
                       codepoint: char) -> Option<GlyphIndex> {
        assert!(self.face.is_not_null());
        let idx = FT_Get_Char_Index(self.face, codepoint as FT_ULong);
        return if idx != 0 as FT_UInt {
            Some(idx as GlyphIndex)
        } else {
            debug!("Invalid codepoint: %?", codepoint);
            None
        };
    }

    pub fn glyph_h_advance(&self,
                           glyph: GlyphIndex) -> Option<FractionalPixel> {
        assert!(self.face.is_not_null());
        let res =  FT_Load_Glyph(self.face, glyph as FT_UInt, 0);
        if res.succeeded() {
            unsafe {
                let void_glyph = (*self.face).glyph;
                let slot: FT_GlyphSlot = cast::transmute(void_glyph);
                assert!(slot.is_not_null());
                debug!("metrics: %?", (*slot).metrics);
                let advance = (*slot).metrics.horiAdvance;
                debug!("h_advance for %? is %?", glyph, advance);
                let advance = advance as i32;
                return Some(fixed_to_float_ft(advance) as FractionalPixel);
            }
        } else {
            debug!("Unable to load glyph %?. reason: %?", glyph, res);
            return None;
        }
    }

    pub fn get_metrics(&self) -> FontMetrics {
        /* TODO(Issue #76): complete me */
        let face = self.get_face_rec();

        let underline_size = self.font_units_to_au(face.underline_thickness as float);
        let underline_offset = self.font_units_to_au(face.underline_position as float);
        let em_size = self.font_units_to_au(face.units_per_EM as float);
        let ascent = self.font_units_to_au(face.ascender as float);
        let descent = self.font_units_to_au(face.descender as float);
        let max_advance = self.font_units_to_au(face.max_advance_width as float);

        return FontMetrics {
            underline_size:   underline_size,
            underline_offset: underline_offset,
            leading:          geometry::from_pt(0.0), //FIXME
            x_height:         geometry::from_pt(0.0), //FIXME
            em_size:          em_size,
            ascent:           ascent,
            descent:          descent,
            max_advance:      max_advance
        }
    }

    fn get_table_for_tag(&self, _: FontTableTag) -> Option<FontTable> {
        None
    }
}

pub impl<'self> FontHandle {
    priv fn set_char_size(face: FT_Face, pt_size: float) -> Result<(), ()>{
        let char_width = float_to_fixed_ft(pt_size) as FT_F26Dot6;
        let char_height = float_to_fixed_ft(pt_size) as FT_F26Dot6;
        let h_dpi = 72;
        let v_dpi = 72;

        let result = FT_Set_Char_Size(face, char_width, char_height, h_dpi, v_dpi);
        if result.succeeded() { Ok(()) } else { Err(()) }
    }

    pub fn new_from_file(fctx: &FontContextHandle, file: ~str,
                         style: &SpecifiedFontStyle) -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let mut face: FT_Face = ptr::null();
        let face_index = 0 as FT_Long;
        do str::as_c_str(file) |file_str| {
            FT_New_Face(ft_ctx, file_str,
                        face_index, ptr::to_mut_unsafe_ptr(&mut face));
        }
        if face.is_null() {
            return Err(());
        }
        if FontHandle::set_char_size(face, style.pt_size).is_ok() {
            Ok(FontHandle { source: FontSourceFile(file), face: face })
        } else {
            Err(())
        }
    }

    pub fn new_from_file_unstyled(fctx: &FontContextHandle, file: ~str)
                               -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let mut face: FT_Face = ptr::null();
        let face_index = 0 as FT_Long;
        do str::as_c_str(file) |file_str| {
            FT_New_Face(ft_ctx, file_str,
                        face_index, ptr::to_mut_unsafe_ptr(&mut face));
        }
        if face.is_null() {
            return Err(());
        }

        Ok(FontHandle { source: FontSourceFile(file), face: face })
    }

    priv fn get_face_rec(&'self self) -> &'self FT_FaceRec {
        unsafe {
            &(*self.face)
        }
    }

    priv fn font_units_to_au(&self, value: float) -> Au {
        let face = self.get_face_rec();

        // face.size is a *c_void in the bindings, presumably to avoid
        // recursive structural types
        let size: &FT_SizeRec = unsafe { cast::transmute(&(*face.size)) };
        let metrics: &FT_Size_Metrics = &(*size).metrics;

        let em_size = face.units_per_EM as float;
        let x_scale = (metrics.x_ppem as float) / em_size as float;

        // If this isn't true then we're scaling one of the axes wrong
        assert!(metrics.x_ppem == metrics.y_ppem);

        return geometry::from_frac_px(value * x_scale);
    }
}

<commit_msg>Ensure FontHandles on Linux don't outlast the FreeType library reference.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern mod freetype;

use font::{CSSFontWeight, FontHandleMethods, FontMetrics, FontTableMethods};
use font::{FontTableTag, FractionalPixel, SpecifiedFontStyle, UsedFontStyle, FontWeight100};
use font::{FontWeight200, FontWeight300, FontWeight400, FontWeight500, FontWeight600};
use font::{FontWeight700, FontWeight800, FontWeight900};
use geometry::Au;
use geometry;
use platform::font_context::FontContextHandle;
use text::glyph::GlyphIndex;
use text::util::{float_to_fixed, fixed_to_float};

use freetype::freetype::bindgen::{FT_Get_Char_Index, FT_Get_Postscript_Name};
use freetype::freetype::bindgen::{FT_Load_Glyph, FT_Set_Char_Size};
use freetype::freetype::bindgen::{FT_New_Face, FT_Get_Sfnt_Table};
use freetype::freetype::bindgen::{FT_New_Memory_Face, FT_Done_Face};
use freetype::freetype::{FTErrorMethods, FT_F26Dot6, FT_Face, FT_FaceRec};
use freetype::freetype::{FT_GlyphSlot, FT_Library, FT_Long, FT_ULong};
use freetype::freetype::{FT_STYLE_FLAG_ITALIC, FT_STYLE_FLAG_BOLD};
use freetype::freetype::{FT_SizeRec, FT_UInt, FT_Size_Metrics};
use freetype::freetype::{ft_sfnt_os2};
use freetype::tt_os2::TT_OS2;

fn float_to_fixed_ft(f: float) -> i32 {
    float_to_fixed(6, f)
}

fn fixed_to_float_ft(f: i32) -> float {
    fixed_to_float(6, f)
}

pub struct FontTable {
    bogus: ()
}

impl FontTableMethods for FontTable {
    fn with_buffer(&self, _blk: &fn(*u8, uint)) {
        fail!()
    }
}

enum FontSource {
    FontSourceMem(~[u8]),
    FontSourceFile(~str)
}

pub struct FontHandle {
    // The font binary. This must stay valid for the lifetime of the font,
    // if the font is created using FT_Memory_Face.
    source: FontSource,
    face: FT_Face,
    handle: FontContextHandle
}

#[unsafe_destructor]
impl Drop for FontHandle {
    fn finalize(&self) {
        assert!(self.face.is_not_null());
        if !FT_Done_Face(self.face).succeeded() {
            fail!(~"FT_Done_Face failed");
        }
    }
}

impl FontHandleMethods for FontHandle {
    pub fn new_from_buffer(fctx: &FontContextHandle,
                           buf: ~[u8],
                           style: &SpecifiedFontStyle)
                        -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let face_result = do vec::as_imm_buf(buf) |bytes: *u8, len: uint| {
            create_face_from_buffer(ft_ctx, bytes, len, style.pt_size)
        };

        // TODO: this could be more simply written as result::chain
        // and moving buf into the struct ctor, but cant' move out of
        // captured binding.
        return match face_result {
            Ok(face) => {
              let handle = FontHandle {
                  face: face,
                  source: FontSourceMem(buf),
                  handle: *fctx
              };
              Ok(handle)
            }
            Err(()) => Err(())
        };

         fn create_face_from_buffer(lib: FT_Library,
                                    cbuf: *u8, cbuflen: uint, pt_size: float)
             -> Result<FT_Face, ()> {

             let mut face: FT_Face = ptr::null();
             let face_index = 0 as FT_Long;
             let result = FT_New_Memory_Face(lib, cbuf, cbuflen as FT_Long,
                                             face_index, ptr::to_mut_unsafe_ptr(&mut face));

             if !result.succeeded() || face.is_null() {
                 return Err(());
             }
             if FontHandle::set_char_size(face, pt_size).is_ok() {
                 Ok(face)
             } else {
                 Err(())
             }
         }
    }

    // an identifier usable by FontContextHandle to recreate this FontHandle.
    fn face_identifier(&self) -> ~str {
        /* FT_Get_Postscript_Name seems like a better choice here, but it
           doesn't give usable results for fontconfig when deserializing. */
        unsafe { str::raw::from_c_str((*self.face).family_name) }
    }
    fn family_name(&self) -> ~str {
        unsafe { str::raw::from_c_str((*self.face).family_name) }
    }
    fn face_name(&self) -> ~str {
        unsafe { str::raw::from_c_str(FT_Get_Postscript_Name(self.face)) }
    }
    fn is_italic(&self) -> bool {
        unsafe { (*self.face).style_flags & FT_STYLE_FLAG_ITALIC != 0 }
    }
    fn boldness(&self) -> CSSFontWeight {
        let default_weight = FontWeight400;
        if unsafe { (*self.face).style_flags & FT_STYLE_FLAG_BOLD == 0 } {
            default_weight
        } else {
            let os2 = FT_Get_Sfnt_Table(self.face, ft_sfnt_os2) as *TT_OS2;
            let valid = os2.is_not_null() && unsafe { (*os2).version != 0xffff };
            if valid {
                let weight = unsafe { (*os2).usWeightClass };
                match weight {
                  1 | 100..199 => FontWeight100,
                  2 | 200..299 => FontWeight200,
                  3 | 300..399 => FontWeight300,
                  4 | 400..499 => FontWeight400,
                  5 | 500..599 => FontWeight500,
                  6 | 600..699 => FontWeight600,
                  7 | 700..799 => FontWeight700,
                  8 | 800..899 => FontWeight800,
                  9 | 900..999 => FontWeight900,
                  _ => default_weight
                }
            } else {
                default_weight
            }
        }
    }

    fn clone_with_style(&self,
                        fctx: &FontContextHandle,
                        style: &UsedFontStyle) -> Result<FontHandle, ()> {
        match self.source {
            FontSourceMem(ref buf) => {
                FontHandleMethods::new_from_buffer(fctx, buf.clone(), style)
            }
            FontSourceFile(copy file) => {
                FontHandle::new_from_file(fctx, file, style)
            }
        }
    }

    pub fn glyph_index(&self,
                       codepoint: char) -> Option<GlyphIndex> {
        assert!(self.face.is_not_null());
        let idx = FT_Get_Char_Index(self.face, codepoint as FT_ULong);
        return if idx != 0 as FT_UInt {
            Some(idx as GlyphIndex)
        } else {
            debug!("Invalid codepoint: %?", codepoint);
            None
        };
    }

    pub fn glyph_h_advance(&self,
                           glyph: GlyphIndex) -> Option<FractionalPixel> {
        assert!(self.face.is_not_null());
        let res =  FT_Load_Glyph(self.face, glyph as FT_UInt, 0);
        if res.succeeded() {
            unsafe {
                let void_glyph = (*self.face).glyph;
                let slot: FT_GlyphSlot = cast::transmute(void_glyph);
                assert!(slot.is_not_null());
                debug!("metrics: %?", (*slot).metrics);
                let advance = (*slot).metrics.horiAdvance;
                debug!("h_advance for %? is %?", glyph, advance);
                let advance = advance as i32;
                return Some(fixed_to_float_ft(advance) as FractionalPixel);
            }
        } else {
            debug!("Unable to load glyph %?. reason: %?", glyph, res);
            return None;
        }
    }

    pub fn get_metrics(&self) -> FontMetrics {
        /* TODO(Issue #76): complete me */
        let face = self.get_face_rec();

        let underline_size = self.font_units_to_au(face.underline_thickness as float);
        let underline_offset = self.font_units_to_au(face.underline_position as float);
        let em_size = self.font_units_to_au(face.units_per_EM as float);
        let ascent = self.font_units_to_au(face.ascender as float);
        let descent = self.font_units_to_au(face.descender as float);
        let max_advance = self.font_units_to_au(face.max_advance_width as float);

        return FontMetrics {
            underline_size:   underline_size,
            underline_offset: underline_offset,
            leading:          geometry::from_pt(0.0), //FIXME
            x_height:         geometry::from_pt(0.0), //FIXME
            em_size:          em_size,
            ascent:           ascent,
            descent:          descent,
            max_advance:      max_advance
        }
    }

    fn get_table_for_tag(&self, _: FontTableTag) -> Option<FontTable> {
        None
    }
}

pub impl<'self> FontHandle {
    priv fn set_char_size(face: FT_Face, pt_size: float) -> Result<(), ()>{
        let char_width = float_to_fixed_ft(pt_size) as FT_F26Dot6;
        let char_height = float_to_fixed_ft(pt_size) as FT_F26Dot6;
        let h_dpi = 72;
        let v_dpi = 72;

        let result = FT_Set_Char_Size(face, char_width, char_height, h_dpi, v_dpi);
        if result.succeeded() { Ok(()) } else { Err(()) }
    }

    pub fn new_from_file(fctx: &FontContextHandle, file: ~str,
                         style: &SpecifiedFontStyle) -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let mut face: FT_Face = ptr::null();
        let face_index = 0 as FT_Long;
        do str::as_c_str(file) |file_str| {
            FT_New_Face(ft_ctx, file_str,
                        face_index, ptr::to_mut_unsafe_ptr(&mut face));
        }
        if face.is_null() {
            return Err(());
        }
        if FontHandle::set_char_size(face, style.pt_size).is_ok() {
            Ok(FontHandle {
                source: FontSourceFile(file),
                face: face,
                handle: *fctx
            })
        } else {
            Err(())
        }
    }

    pub fn new_from_file_unstyled(fctx: &FontContextHandle, file: ~str)
                               -> Result<FontHandle, ()> {
        let ft_ctx: FT_Library = fctx.ctx.ctx;
        if ft_ctx.is_null() { return Err(()); }

        let mut face: FT_Face = ptr::null();
        let face_index = 0 as FT_Long;
        do str::as_c_str(file) |file_str| {
            FT_New_Face(ft_ctx, file_str,
                        face_index, ptr::to_mut_unsafe_ptr(&mut face));
        }
        if face.is_null() {
            return Err(());
        }

        Ok(FontHandle {
            source: FontSourceFile(file),
            face: face,
            handle: *fctx
        })
    }

    priv fn get_face_rec(&'self self) -> &'self FT_FaceRec {
        unsafe {
            &(*self.face)
        }
    }

    priv fn font_units_to_au(&self, value: float) -> Au {
        let face = self.get_face_rec();

        // face.size is a *c_void in the bindings, presumably to avoid
        // recursive structural types
        let size: &FT_SizeRec = unsafe { cast::transmute(&(*face.size)) };
        let metrics: &FT_Size_Metrics = &(*size).metrics;

        let em_size = face.units_per_EM as float;
        let x_scale = (metrics.x_ppem as float) / em_size as float;

        // If this isn't true then we're scaling one of the axes wrong
        assert!(metrics.x_ppem == metrics.y_ppem);

        return geometry::from_frac_px(value * x_scale);
    }
}

<|endoftext|>


<commit_before>#![feature(bool_to_option)]
#![feature(crate_visibility_modifier)]
#![feature(decl_macro)]
#![feature(or_patterns)]
#![feature(proc_macro_diagnostic)]
#![feature(proc_macro_internals)]
#![feature(proc_macro_span)]
#![feature(try_blocks)]

#[macro_use]
extern crate rustc_macros;

extern crate proc_macro as pm;

mod placeholders;
mod proc_macro_server;

pub use mbe::macro_rules::compile_declarative_macro;
crate use rustc_span::hygiene;
pub mod base;
pub mod build;
#[macro_use]
pub mod config;
pub mod expand;
pub mod module;
pub mod proc_macro;

crate mod mbe;

// HACK(Centril, #64197): These shouldn't really be here.
// Rather, they should be with their respective modules which are defined in other crates.
// However, since for now constructing a `ParseSess` sorta requires `config` from this crate,
// these tests will need to live here in the iterim.

#[cfg(test)]
mod tests;
#[cfg(test)]
mod parse {
    #[cfg(test)]
    mod tests;
}
#[cfg(test)]
mod tokenstream {
    #[cfg(test)]
    mod tests;
}
#[cfg(test)]
mod mut_visit {
    #[cfg(test)]
    mod tests;
}
<commit_msg>Remove some no longer necessary `#[cfg(test)]`s<commit_after>#![feature(bool_to_option)]
#![feature(crate_visibility_modifier)]
#![feature(decl_macro)]
#![feature(or_patterns)]
#![feature(proc_macro_diagnostic)]
#![feature(proc_macro_internals)]
#![feature(proc_macro_span)]
#![feature(try_blocks)]

#[macro_use]
extern crate rustc_macros;

extern crate proc_macro as pm;

mod placeholders;
mod proc_macro_server;

pub use mbe::macro_rules::compile_declarative_macro;
crate use rustc_span::hygiene;
pub mod base;
pub mod build;
#[macro_use]
pub mod config;
pub mod expand;
pub mod module;
pub mod proc_macro;

crate mod mbe;

// HACK(Centril, #64197): These shouldn't really be here.
// Rather, they should be with their respective modules which are defined in other crates.
// However, since for now constructing a `ParseSess` sorta requires `config` from this crate,
// these tests will need to live here in the iterim.

#[cfg(test)]
mod tests;
#[cfg(test)]
mod parse {
    mod tests;
}
#[cfg(test)]
mod tokenstream {
    mod tests;
}
#[cfg(test)]
mod mut_visit {
    mod tests;
}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::request_manager::RequestManager;
use diem_config::config::{PeerNetworkId, UpstreamConfig};
use netcore::transport::ConnectionOrigin;
use std::{collections::HashMap, time::Duration};

const NUM_CHUNKS_TO_PROCESS: u64 = 50;
const NUM_PICKS_TO_MAKE: u64 = 1000;

#[test]
fn test_disable_peer() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 1);

    let validator_0 = validators[0].clone();

    // Verify single validator in peers
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(!request_manager.no_available_peers());

    // Disable validator 0
    request_manager.disable_peer(&validator_0, ConnectionOrigin::Outbound);

    // Verify validator 0 is still known, but no longer available
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(request_manager.no_available_peers());

    // Add validator 0 and verify it's now enabled
    request_manager.enable_peer(validator_0, ConnectionOrigin::Outbound);
    assert!(!request_manager.no_available_peers());
}

#[test]
fn test_score_chunk_success() {
    let num_validators = 4;
    let (mut request_manager, validators) =
        generate_request_manager_and_test_validators(0, num_validators);

    // Process empty chunk responses from all validators
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        for validator_index in 0..num_validators {
            request_manager.process_empty_chunk(&validators[validator_index as usize]);
        }
    }

    // Process successful chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_success_response(&validators[0]);
    }

    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(&mut request_manager, NUM_PICKS_TO_MAKE);

    // Verify validator 0 is chosen more often than the other validators
    let validator_0_count = pick_counts.get(&validators[0]).unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != 0 {
            assert!(validator_0_count > pick_counts.get(&validator).unwrap());
        }
    }
}

#[test]
fn test_score_chunk_timeout() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = vec![validators[0].clone()];

    // Process multiple request timeouts from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(1, validator_0.clone());
        assert!(request_manager.check_timeout(1).unwrap());
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_chunk_version_mismatch() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = validators[0].clone();

    // Process multiple chunk version mismatches from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(100, vec![validator_0.clone()]);
        request_manager
            .process_chunk_version_mismatch(&validator_0, 100, 0)
            .unwrap_err();
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_empty_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple empty chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_empty_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_invalid_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple invalid chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_invalid_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_remove_requests() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    let validator_0 = vec![validators[0].clone()];
    let validator_1 = vec![validators[1].clone()];

    // Add version requests to request manager
    request_manager.add_request(1, validator_0.clone());
    request_manager.add_request(3, validator_1.clone());
    request_manager.add_request(5, validator_0.clone());
    request_manager.add_request(10, validator_0);
    request_manager.add_request(12, validator_1);

    // Remove all request versions below 5
    request_manager.remove_requests(5);

    // Verify versions updated correctly
    assert!(request_manager.get_last_request_time(1).is_none());
    assert!(request_manager.get_last_request_time(3).is_none());
    assert!(request_manager.get_last_request_time(5).is_some());
    assert!(request_manager.get_last_request_time(10).is_some());
    assert!(request_manager.get_last_request_time(12).is_some());
}

#[test]
fn test_request_times() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    // Verify first request time doesn't exist for missing request
    assert!(request_manager.get_first_request_time(1).is_none());

    // Add version requests to request manager
    request_manager.add_request(1, vec![validators[0].clone()]);
    request_manager.add_request(1, vec![validators[1].clone()]);

    // Verify first request time is less than last request time
    assert!(
        request_manager.get_first_request_time(1).unwrap()
            < request_manager.get_last_request_time(1).unwrap()
    );
}

/// Verify that the bad validator is chosen less often than the other validators (due to
/// having a lower peer score internally).
fn verify_bad_validator_picked_less_often(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    bad_validator_index: usize,
) {
    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(request_manager, NUM_PICKS_TO_MAKE);

    // Verify bad validator is chosen less often than the other validators
    let bad_validator_count = pick_counts
        .get(&validators[bad_validator_index])
        .unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != bad_validator_index {
            assert!(bad_validator_count < pick_counts.get(&validator).unwrap());
        }
    }
}

/// Picks a peer to send a chunk request to (multiple times) and constructs a pick count
/// for each of the chosen peers.
fn calculate_pick_counts_for_validators(
    request_manager: &mut RequestManager,
    number_of_picks_to_execute: u64,
) -> HashMap<PeerNetworkId, u64> {
    let mut pick_counts = HashMap::new();

    for _ in 0..number_of_picks_to_execute {
        let picked_peers = request_manager.pick_peers();
        assert_eq!(1, picked_peers.len()); // Ensure only one validator per multicast level

        let picked_peer = picked_peers[0].clone();
        let counter = pick_counts.entry(picked_peer).or_insert(0);
        *counter += 1;
    }

    pick_counts
}

/// Generates a new request manager with a specified number of validator peers enabled.
fn generate_request_manager_and_test_validators(
    request_timeout: u64,
    num_validators: u64,
) -> (RequestManager, Vec<PeerNetworkId>) {
    let mut request_manager = RequestManager::new(
        UpstreamConfig::default(),
        Duration::from_secs(request_timeout),
        Duration::from_secs(30),
        HashMap::new(),
    );

    let mut validators = Vec::new();
    for _ in 0..num_validators {
        let validator = PeerNetworkId::random_validator();
        request_manager.enable_peer(validator.clone(), ConnectionOrigin::Outbound);
        validators.push(validator);
    }

    (request_manager, validators)
}
<commit_msg>[State Sync] Add missing test for non-penalization of multicast chunk version mismatches<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::request_manager::RequestManager;
use diem_config::config::{PeerNetworkId, UpstreamConfig};
use netcore::transport::ConnectionOrigin;
use std::{collections::HashMap, time::Duration};

const NUM_CHUNKS_TO_PROCESS: u64 = 50;
const NUM_PICKS_TO_MAKE: u64 = 1000;

#[test]
fn test_disable_peer() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(0, 1);

    let validator_0 = validators[0].clone();

    // Verify single validator in peers
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(!request_manager.no_available_peers());

    // Disable validator 0
    request_manager.disable_peer(&validator_0, ConnectionOrigin::Outbound);

    // Verify validator 0 is still known, but no longer available
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(request_manager.no_available_peers());

    // Add validator 0 and verify it's now enabled
    request_manager.enable_peer(validator_0, ConnectionOrigin::Outbound);
    assert!(!request_manager.no_available_peers());
}

#[test]
fn test_score_chunk_success() {
    let num_validators = 4;
    let (mut request_manager, validators) =
        generate_request_manager_and_validators(0, num_validators);

    // Process empty chunk responses from all validators
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        for validator_index in 0..num_validators {
            request_manager.process_empty_chunk(&validators[validator_index as usize]);
        }
    }

    // Process successful chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_success_response(&validators[0]);
    }

    // Verify validator 0 is chosen more often than the other validators
    verify_validator_picked_most_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_chunk_timeout() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(0, 4);

    let validator_0 = vec![validators[0].clone()];

    // Process multiple request timeouts from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(1, validator_0.clone());
        assert!(request_manager.check_timeout(1).unwrap());
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_validator_picked_least_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_chunk_version_mismatch() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(0, 4);

    let validator_0 = validators[0].clone();

    // Process multiple chunk version mismatches from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(100, vec![validator_0.clone()]);
        request_manager
            .process_chunk_version_mismatch(&validator_0, 100, 0)
            .unwrap_err();
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_validator_picked_least_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_chunk_version_mismatch_multicast() {
    let num_validators = 4;
    let (mut request_manager, validators) =
        generate_request_manager_and_validators(0, num_validators);

    let validator_0 = validators[0].clone();
    let validator_1 = validators[1].clone();

    // Process empty chunk responses from all validators except validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        for validator_index in 1..num_validators {
            request_manager.process_empty_chunk(&validators[validator_index as usize]);
        }
    }

    // Process multiple multi-cast chunk version mismatches from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(100, vec![validator_0.clone(), validator_1.clone()]);
        request_manager
            .process_chunk_version_mismatch(&validator_0, 100, 0)
            .unwrap_err();
    }

    // Verify validator 0 is chosen more often than the other validators
    verify_validator_picked_most_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_empty_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(10, 4);

    // Process multiple empty chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_empty_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_validator_picked_least_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_invalid_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(10, 4);

    // Process multiple invalid chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_invalid_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_validator_picked_least_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_remove_requests() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(0, 2);

    let validator_0 = vec![validators[0].clone()];
    let validator_1 = vec![validators[1].clone()];

    // Add version requests to request manager
    request_manager.add_request(1, validator_0.clone());
    request_manager.add_request(3, validator_1.clone());
    request_manager.add_request(5, validator_0.clone());
    request_manager.add_request(10, validator_0);
    request_manager.add_request(12, validator_1);

    // Remove all request versions below 5
    request_manager.remove_requests(5);

    // Verify versions updated correctly
    assert!(request_manager.get_last_request_time(1).is_none());
    assert!(request_manager.get_last_request_time(3).is_none());
    assert!(request_manager.get_last_request_time(5).is_some());
    assert!(request_manager.get_last_request_time(10).is_some());
    assert!(request_manager.get_last_request_time(12).is_some());
}

#[test]
fn test_request_times() {
    let (mut request_manager, validators) = generate_request_manager_and_validators(0, 2);

    // Verify first request time doesn't exist for missing request
    assert!(request_manager.get_first_request_time(1).is_none());

    // Add version requests to request manager
    request_manager.add_request(1, vec![validators[0].clone()]);
    request_manager.add_request(1, vec![validators[1].clone()]);

    // Verify first request time is less than last request time
    assert!(
        request_manager.get_first_request_time(1).unwrap()
            < request_manager.get_last_request_time(1).unwrap()
    );
}

/// Verify that the specified validator is chosen most often (due to having a
/// higher peer score internally).
fn verify_validator_picked_most_often(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    validator_index: usize,
) {
    verify_validator_pick_frequency(request_manager, validators, validator_index, true)
}

/// Verify that the specified validator is chosen least often (due to having a
/// lower peer score internally).
fn verify_validator_picked_least_often(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    validator_index: usize,
) {
    verify_validator_pick_frequency(request_manager, validators, validator_index, false)
}

/// Verifies the picking frequency of the specified validator: if `check_highest_frequency`
/// is true we verify the validator is chosen most often, otherwise, we verify it is
/// chosen least often.
fn verify_validator_pick_frequency(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    validator_index: usize,
    check_highest_frequency: bool,
) {
    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(request_manager, NUM_PICKS_TO_MAKE);

    // Verify validator frequency
    let validator_count = pick_counts.get(&validators[validator_index]).unwrap_or(&0);
    for (index, validator) in validators.iter().enumerate() {
        if validator_index != index {
            if check_highest_frequency {
                assert!(validator_count > pick_counts.get(&validator).unwrap());
            } else {
                assert!(validator_count < pick_counts.get(&validator).unwrap());
            }
        }
    }
}

/// Picks a peer to send a chunk request to (multiple times) and constructs a pick count
/// for each of the chosen peers.
fn calculate_pick_counts_for_validators(
    request_manager: &mut RequestManager,
    number_of_picks_to_execute: u64,
) -> HashMap<PeerNetworkId, u64> {
    let mut pick_counts = HashMap::new();

    for _ in 0..number_of_picks_to_execute {
        let picked_peers = request_manager.pick_peers();
        assert_eq!(1, picked_peers.len()); // Ensure only one validator per multicast level

        let picked_peer = picked_peers[0].clone();
        let counter = pick_counts.entry(picked_peer).or_insert(0);
        *counter += 1;
    }

    pick_counts
}

/// Generates a new request manager with a specified number of validator peers enabled.
fn generate_request_manager_and_validators(
    request_timeout: u64,
    num_validators: u64,
) -> (RequestManager, Vec<PeerNetworkId>) {
    let mut request_manager = RequestManager::new(
        UpstreamConfig::default(),
        Duration::from_secs(request_timeout),
        Duration::from_secs(30),
        HashMap::new(),
    );

    let mut validators = Vec::new();
    for _ in 0..num_validators {
        let validator = PeerNetworkId::random_validator();
        request_manager.enable_peer(validator.clone(), ConnectionOrigin::Outbound);
        validators.push(validator);
    }

    (request_manager, validators)
}
<|endoftext|>