<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Token {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d*\]$").unwrap();
        }

        match RE.captures(s) {
            None => {
                Token::Identifier { ident: String::from(s), next: None }
            },
            Some(captures) => {
                match captures.get(0) {
                    None => Token::Identifier { ident: String::from(s), next: None },
                    Some(mtch) => {
                        let mtch = mtch.as_str().replace("[","").replace("]","");
                        println!("{}", mtch);
                        let i : i64 = FromStr::from_str(&mtch).unwrap();
                        Token::Index {
                            idx: i,
                            next: None,
                        }
                    }
                }
            }
        }
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = mk_token_object(token);
                build_token_tree(split, &mut token);
                last.set_next(token);
            }
        }
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            let mut tok = mk_token_object(token);
            build_token_tree(&mut tokens, &mut tok);
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Add more test cases<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Token {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d*\]$").unwrap();
        }

        match RE.captures(s) {
            None => {
                Token::Identifier { ident: String::from(s), next: None }
            },
            Some(captures) => {
                match captures.get(0) {
                    None => Token::Identifier { ident: String::from(s), next: None },
                    Some(mtch) => {
                        let mtch = mtch.as_str().replace("[","").replace("]","");
                        println!("{}", mtch);
                        let i : i64 = FromStr::from_str(&mtch).unwrap();
                        Token::Index {
                            idx: i,
                            next: None,
                        }
                    }
                }
            }
        }
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = mk_token_object(token);
                build_token_tree(split, &mut token);
                last.set_next(token);
            }
        }
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            let mut tok = mk_token_object(token);
            build_token_tree(&mut tokens, &mut tok);
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    SymbolCannotBeNumber,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }

    fn next_symbol(&mut self, start: char) -> TokenResult {
        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else {
                try!(self.scanner.unget_char()
                    .map_err(|e| TokenError::ScannerError(e.to_owned())));
                break;
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol.chars().next().map_or(false, |c| c.is_digit(10)) {
            symbol.parse::<f64>()
                .map(|value| Token::Number(value))
                .map_err(|_| TokenError::SymbolCannotBeNumber)
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            '\\' => Token::Backslash,
            '(' => Token::ParenOpen,
            ')' => Token::ParenClose,
            '"' => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.next_symbol(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);

        test_number("1233.14", 1233.14f64);

        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('1'));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        // Ignore first token
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<commit_msg>Renamed SymbolCannotBeNumber to a much better name<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    UnexpectedEndOfInput,
    InvalidNumericLiteral,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let c = self.scanner.get_char();
            if c.is_none() {
                break;
            }

            let c = c.unwrap();
            if !c.is_whitespace() || c == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }

    fn next_symbol(&mut self, start: char) -> TokenResult {
        let mut symbol = String::new();

        let mut next_char = Some(start);
        while !next_char.is_none() {
            let c = next_char.unwrap();
            if SYMBOL_CHARS.contains(&c) {
                symbol.push(c)
            }
            else {
                try!(self.scanner.unget_char()
                    .map_err(|e| TokenError::ScannerError(e.to_owned())));
                break;
            }

            next_char = self.scanner.get_char();
        }

        self.validate_symbol(symbol)
    }

    fn validate_symbol(&mut self, symbol: String) -> TokenResult {
        if symbol.len() == 0 {
            Err(match self.scanner.get_char() {
                Some(c) => TokenError::UnrecognizedCharacter(c),
                None => TokenError::UnexpectedEndOfInput,
            })
        }
        else if symbol.chars().next().map_or(false, |c| c.is_digit(10)) {
            symbol.parse::<f64>()
                .map(|value| Token::Number(value))
                .map_err(|_| TokenError::InvalidNumericLiteral)
        }
        else if symbol == "=" {
            Ok(Token::Equals)
        }
        else {
            Ok(Token::Symbol(symbol))
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        let c = self.scanner.get_char();
        if c.is_none() {
            self.reached_eof = true;
            return None;
        }

        let c = c.unwrap();
        Some(Ok(match c {
            '\\' => Token::Backslash,
            '(' => Token::ParenOpen,
            ')' => Token::ParenClose,
            '"' => Token::StringBoundary,
            EOL_CHAR => Token::EOL,
            _ => return Some(self.next_symbol(c)),
        }))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert_eq!(tokenizer.next().unwrap().unwrap(), Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert_eq!(tokenizer.next().unwrap().unwrap(), ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert_eq!(tokenizer.next().unwrap().unwrap(), StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn number() {
        test_number("-5394", -5394f64);

        test_number("1233.14", 1233.14f64);

        test_number("3.14159e-3", 3.14159e-3f64);
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert_eq!(tokenizer.next().unwrap().unwrap(), EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn allows_symbols_starting_with_minus() {
        let symbol = "-abqq$$^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::InvalidNumericLiteral);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers_and_minus() {
        let symbol = "-123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('1'));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "a[$abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);

        // Ignore first token
        assert_eq!(tokenizer.next().unwrap().unwrap_err(), TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert_eq!(tokenizer.next().unwrap().unwrap(), *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh$*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert_eq!(tokenizer.next().unwrap().unwrap(), Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }

    fn test_number(literal: &str, expected_value: f64) {
        const ERROR_MARGIN: f64 = 0.0;
        let mut tokenizer = tokenizer_for(literal);

        let matched_token = tokenizer.next().unwrap().unwrap();
        if let Number(value) = matched_token {
            assert!(value - expected_value <= ERROR_MARGIN,
                format!("The literal '{}' did not match the value {}, got {} instead", literal, expected_value, value));
        }
        else {
            panic!("Did not get Number when expecting {}, got {:?} instead", expected_value, matched_token);
        }

        assert!(tokenizer.next().is_none());
    }
}
<|endoftext|>


<commit_before>use ritual::config::Config;
use ritual_common::errors::Result;

/// QtWidgets specific configuration.
pub fn widgets_config(config: &mut Config) -> Result<()> {
    config.set_cpp_parser_path_hook(|path| {
        let string = path.to_templateless_string();
        let blocked = &["QWidgetData", "QWidgetItemV2"];
        if blocked.contains(&string.as_str()) {
            return Ok(false);
        }
        Ok(true)
    });

    Ok(())
}
<commit_msg>blacklist QWidgetItemData<commit_after>use ritual::config::Config;
use ritual_common::errors::Result;

/// QtWidgets specific configuration.
pub fn widgets_config(config: &mut Config) -> Result<()> {
    config.set_cpp_parser_path_hook(|path| {
        let string = path.to_templateless_string();
        let blocked = &[
            //internal
            "QWidgetData",
            "QWidgetItemV2",
            "QWidgetItemData",
        ];
        if blocked.contains(&string.as_str()) {
            return Ok(false);
        }
        Ok(true)
    });

    Ok(())
}
<|endoftext|>


<commit_before>use super::{InlineAsmArch, InlineAsmType};
use rustc_macros::HashStable_Generic;
use std::fmt;

def_reg_class! {
    S390x S390xInlineAsmRegClass {
        reg,
        freg,
    }
}

impl S390xInlineAsmRegClass {
    pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] {
        &[]
    }

    pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option<Self> {
        None
    }

    pub fn suggest_modifier(
        self,
        _arch: InlineAsmArch,
        _ty: InlineAsmType,
    ) -> Option<(char, &'static str)> {
        None
    }

    pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<(char, &'static str)> {
        None
    }

    pub fn supported_types(
        self,
        arch: InlineAsmArch,
    ) -> &'static [(InlineAsmType, Option<&'static str>)] {
        match (self, arch) {
            (Self::reg, _) => types! { _: I8, I16, I32, I64; },
            (Self::freg, _) => types! { _: F32, F64; },
        }
    }
}

def_regs! {
    S390x S390xInlineAsmReg S390xInlineAsmRegClass {
        r0: reg = ["r0"],
        r1: reg = ["r1"],
        r2: reg = ["r2"],
        r3: reg = ["r3"],
        r4: reg = ["r4"],
        r5: reg = ["r5"],
        r6: reg = ["r6"],
        r7: reg = ["r7"],
        r8: reg = ["r8"],
        r9: reg = ["r9"],
        r10: reg = ["r10"],
        r12: reg = ["r12"],
        r13: reg = ["r13"],
        r14: reg = ["r14"],
        f0: freg = ["f0"],
        f1: freg = ["f1"],
        f2: freg = ["f2"],
        f3: freg = ["f3"],
        f4: freg = ["f4"],
        f5: freg = ["f5"],
        f6: freg = ["f6"],
        f7: freg = ["f7"],
        f8: freg = ["f8"],
        f9: freg = ["f9"],
        f10: freg = ["f10"],
        f11: freg = ["f11"],
        f12: freg = ["f12"],
        f13: freg = ["f13"],
        f14: freg = ["f14"],
        f15: freg = ["f15"],
        #error = ["r11"] =>
            "The frame pointer cannot be used as an operand for inline asm",
        #error = ["r15"] =>
            "The stack pointer cannot be used as an operand for inline asm",
        #error = [
            "c0", "c1", "c2", "c3",
            "c4", "c5", "c6", "c7",
            "c8", "c9", "c10", "c11",
            "c12", "c13", "c14", "c15"
        ] =>
            "control registers are reserved by the kernel and cannot be used as operands for inline asm",
        #error = [
            "a0", "a1", "a2", "a3",
            "a4", "a5", "a6", "a7",
            "a8", "a9", "a10", "a11",
            "a12", "a13", "a14", "a15"
        ] =>
            "access registers are not supported and cannot be used as operands for inline asm",
    }
}

impl S390xInlineAsmReg {
    pub fn emit(
        self,
        out: &mut dyn fmt::Write,
        _arch: InlineAsmArch,
        _modifier: Option<char>,
    ) -> fmt::Result {
        out.write_str(&format!("%{}", self.name()))
    }
}
<commit_msg>Fix: made suggested change<commit_after>use super::{InlineAsmArch, InlineAsmType};
use rustc_macros::HashStable_Generic;
use std::fmt;

def_reg_class! {
    S390x S390xInlineAsmRegClass {
        reg,
        freg,
    }
}

impl S390xInlineAsmRegClass {
    pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] {
        &[]
    }

    pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option<Self> {
        None
    }

    pub fn suggest_modifier(
        self,
        _arch: InlineAsmArch,
        _ty: InlineAsmType,
    ) -> Option<(char, &'static str)> {
        None
    }

    pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<(char, &'static str)> {
        None
    }

    pub fn supported_types(
        self,
        arch: InlineAsmArch,
    ) -> &'static [(InlineAsmType, Option<&'static str>)] {
        match (self, arch) {
            (Self::reg, _) => types! { _: I8, I16, I32, I64; },
            (Self::freg, _) => types! { _: F32, F64; },
        }
    }
}

def_regs! {
    S390x S390xInlineAsmReg S390xInlineAsmRegClass {
        r0: reg = ["r0"],
        r1: reg = ["r1"],
        r2: reg = ["r2"],
        r3: reg = ["r3"],
        r4: reg = ["r4"],
        r5: reg = ["r5"],
        r6: reg = ["r6"],
        r7: reg = ["r7"],
        r8: reg = ["r8"],
        r9: reg = ["r9"],
        r10: reg = ["r10"],
        r12: reg = ["r12"],
        r13: reg = ["r13"],
        r14: reg = ["r14"],
        f0: freg = ["f0"],
        f1: freg = ["f1"],
        f2: freg = ["f2"],
        f3: freg = ["f3"],
        f4: freg = ["f4"],
        f5: freg = ["f5"],
        f6: freg = ["f6"],
        f7: freg = ["f7"],
        f8: freg = ["f8"],
        f9: freg = ["f9"],
        f10: freg = ["f10"],
        f11: freg = ["f11"],
        f12: freg = ["f12"],
        f13: freg = ["f13"],
        f14: freg = ["f14"],
        f15: freg = ["f15"],
        #error = ["r11"] =>
            "The frame pointer cannot be used as an operand for inline asm",
        #error = ["r15"] =>
            "The stack pointer cannot be used as an operand for inline asm",
        #error = [
            "c0", "c1", "c2", "c3",
            "c4", "c5", "c6", "c7",
            "c8", "c9", "c10", "c11",
            "c12", "c13", "c14", "c15"
        ] =>
            "control registers are reserved by the kernel and cannot be used as operands for inline asm",
        #error = [
            "a0", "a1", "a2", "a3",
            "a4", "a5", "a6", "a7",
            "a8", "a9", "a10", "a11",
            "a12", "a13", "a14", "a15"
        ] =>
            "access registers are not supported and cannot be used as operands for inline asm",
    }
}

impl S390xInlineAsmReg {
    pub fn emit(
        self,
        out: &mut dyn fmt::Write,
        _arch: InlineAsmArch,
        _modifier: Option<char>,
    ) -> fmt::Result {
        write!(out, "%{}", self.name())
    }
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This is the backtracking matching engine. It has the same exact capability
// as the full NFA simulation, except it is artificially restricted to small
// regexes on small inputs because of its memory requirements.
//
// In particular, this is a *bounded* backtracking engine. It retains worst
// case linear time by keeping track of the states that is has visited (using a
// bitmap). Namely, once a state is visited, it is never visited again. Since a
// state is keyed by `(instruction index, input index)`, we have that its time
// complexity is `O(mn)`.
//
// The backtracking engine can beat out the NFA simulation on small
// regexes/inputs because it doesn't have to keep track of multiple copies of
// the capture groups. In benchmarks, the backtracking engine is roughly twice
// as fast as the full NFA simulation.

use input::{Input, InputAt, CharInput};
use program::{InstIdx, Program};
use re::CaptureIdxs;

type Bits = u32;
const BIT_SIZE: usize = 32;
const MAX_PROG_SIZE: usize = 100;
const MAX_INPUT_SIZE: usize = 256 * (1 << 10);

// Total memory usage in bytes is determined by:
//
//   ((len(insts) * (len(input) + 1) + bits - 1) / bits) / (bits / 8)
//
// With the above settings, this comes out to ~3.2MB. Mostly these numbers
// were picked empirically with suspicious benchmarks.

/// A backtracking matching engine.
#[derive(Debug)]
pub struct Backtrack<'a, 'r, 't, 'c> {
    prog: &'r Program,
    input: CharInput<'t>,
    caps: &'c mut CaptureIdxs,
    m: &'a mut BackMachine,
}

/// Shared cached state between multiple invocations of a backtracking engine
/// in the same thread.
///
/// It is exported so that it can be cached by `program::Program`.
#[derive(Debug)]
pub struct BackMachine {
    jobs: Vec<Job>,
    visited: Vec<Bits>,
}

impl BackMachine {
    /// Create new empty state for the backtracking engine.
    pub fn new() -> BackMachine {
        BackMachine {
            jobs: vec![],
            visited: vec![],
        }
    }
}

/// A job is an explicit unit of stack space in the backtracking engine.
///
/// The "normal" representation is a single state transition, which corresponds
/// to an NFA state and a character in the input. However, the backtracking
/// engine must keep track of old capture group values. We use the explicit
/// stack to do it.
#[derive(Clone, Copy, Debug)]
enum Job {
    Inst { pc: InstIdx, at: InputAt },
    SaveRestore { slot: usize, old_pos: Option<usize> },
}

impl<'a, 'r, 't, 'c> Backtrack<'a, 'r, 't, 'c> {
    /// Execute the backtracking matching engine.
    ///
    /// If there's a match, `exec` returns `true` and populates the given
    /// captures accordingly.
    pub fn exec(
        prog: &'r Program,
        mut caps: &mut CaptureIdxs,
        text: &'t str,
        start: usize,
    ) -> bool {
        let input = CharInput::new(text);
        let start = input.at(start);
        let mut m = prog.backtrack.get();
        let mut b = Backtrack {
            prog: prog,
            input: input,
            caps: caps,
            m: &mut m,
        };
        b.exec_(start)
    }

    /// Returns true iff the given regex and input can be executed by this
    /// engine with reasonable memory usage.
    pub fn should_exec(prog: &'r Program, input: &str) -> bool {
        prog.insts.len() <= MAX_PROG_SIZE && input.len() <= MAX_INPUT_SIZE
    }

    fn clear(&mut self) {
        // Reset the job memory so that we start fresh.
        self.m.jobs.truncate(0);

        // Now we need to clear the bit state set.
        // We do this by figuring out how much space we need to keep track
        // of the states we've visited.
        // Then we reset all existing allocated space to 0.
        // Finally, we request more space if we need it.
        //
        // This is all a little circuitous, but doing this unsafely
        // doesn't seem to have a measurable impact on performance.
        // (Probably because backtracking is limited to such small
        // inputs/regexes in the first place.)
        let visited_len =
            (self.prog.insts.len() * (self.input.len() + 1) + BIT_SIZE - 1)
            /
            BIT_SIZE;
        for v in &mut self.m.visited {
            *v = 0;
        }
        let cur_visited_cap = self.m.visited.capacity();
        if visited_len > cur_visited_cap {
            self.m.visited.reserve_exact(visited_len - cur_visited_cap);
            for _ in 0..(visited_len - cur_visited_cap) {
                self.m.visited.push(0);
            }
        }
    }

    fn exec_(&mut self, mut at: InputAt) -> bool {
        self.clear();
        if self.prog.anchored_begin {
            return if !at.is_beginning() {
                false
            } else {
                match self.input.prefix_at(&self.prog.prefixes, at) {
                    None => false,
                    Some(at) => self.backtrack(at),
                }
            };
        }
        loop {
            if !self.prog.prefixes.is_empty() {
                at = match self.input.prefix_at(&self.prog.prefixes, at) {
                    None => return false,
                    Some(at) => at,
                };
            }
            if self.backtrack(at) {
                return true;
            }
            if at.char().is_none() {
                return false;
            }
            at = self.input.at(at.next_pos());
        }
    }

    // This `inline(always)` seems to result in about a 10-15% increase in
    // throughput on the `hard` benchmarks (over a standard `inline`). ---AG
    #[inline(always)]
    fn backtrack(&mut self, start: InputAt) -> bool {
        self.push(0, start);
        while let Some(job) = self.m.jobs.pop() {
            match job {
                Job::Inst { pc, at } => {
                    if self.step(pc, at) {
                        return true;
                    }
                }
                Job::SaveRestore { slot, old_pos } => {
                    self.caps[slot] = old_pos;
                }
            }
        }
        false
    }

    fn step(&mut self, mut pc: InstIdx, mut at: InputAt) -> bool {
        use program::Inst::*;
        loop {
            // This loop is an optimization to avoid constantly pushing/popping
            // from the stack. Namely, if we're pushing a job only to run it
            // next, avoid the push and just mutate `pc` (and possibly `at`)
            // in place.
            match self.prog.insts[pc] {
                Match => return true,
                Save(slot) => {
                    if slot < self.caps.len() {
                        // If this path doesn't work out, then we save the old
                        // capture index (if one exists) in an alternate
                        // job. If the next path fails, then the alternate
                        // job is popped and the old capture index is restored.
                        let old_pos = self.caps[slot];
                        self.push_save_restore(slot, old_pos);
                        self.caps[slot] = Some(at.pos());
                    }
                    pc += 1;
                }
                Jump(pc2) => pc = pc2,
                Split(x, y) => {
                    self.push(y, at);
                    pc = x;
                }
                EmptyLook(ref inst) => {
                    let prev = self.input.previous_at(at.pos());
                    if inst.matches(prev.char(), at.char()) {
                        pc += 1;
                    } else {
                        return false;
                    }
                }
                Char(c) => {
                    if c == at.char() {
                        pc += 1;
                        at = self.input.at(at.next_pos());
                    } else {
                        return false;
                    }
                }
                Ranges(ref inst) => {
                    if inst.matches(at.char()) {
                        pc += 1;
                        at = self.input.at(at.next_pos());
                    } else {
                        return false;
                    }
                }
            }
            if self.has_visited(pc, at) {
                return false;
            }
        }
    }

    fn push(&mut self, pc: InstIdx, at: InputAt) {
        self.m.jobs.push(Job::Inst { pc: pc, at: at });
    }

    fn push_save_restore(&mut self, slot: usize, old_pos: Option<usize>) {
        self.m.jobs.push(Job::SaveRestore { slot: slot, old_pos: old_pos });
    }

    fn has_visited(&mut self, pc: InstIdx, at: InputAt) -> bool {
        let k = pc * (self.input.len() + 1) + at.pos();
        let k1 = k / BIT_SIZE;
        let k2 = (1 << (k & (BIT_SIZE - 1))) as Bits;
        if self.m.visited[k1] & k2 == 0 {
            self.m.visited[k1] |= k2;
            false
        } else {
            true
        }
    }
}
<commit_msg>Fix perf bug.<commit_after>// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This is the backtracking matching engine. It has the same exact capability
// as the full NFA simulation, except it is artificially restricted to small
// regexes on small inputs because of its memory requirements.
//
// In particular, this is a *bounded* backtracking engine. It retains worst
// case linear time by keeping track of the states that is has visited (using a
// bitmap). Namely, once a state is visited, it is never visited again. Since a
// state is keyed by `(instruction index, input index)`, we have that its time
// complexity is `O(mn)`.
//
// The backtracking engine can beat out the NFA simulation on small
// regexes/inputs because it doesn't have to keep track of multiple copies of
// the capture groups. In benchmarks, the backtracking engine is roughly twice
// as fast as the full NFA simulation.

use input::{Input, InputAt, CharInput};
use program::{InstIdx, Program};
use re::CaptureIdxs;

type Bits = u32;
const BIT_SIZE: usize = 32;
const MAX_PROG_SIZE: usize = 100;
const MAX_INPUT_SIZE: usize = 256 * (1 << 10);

// Total memory usage in bytes is determined by:
//
//   ((len(insts) * (len(input) + 1) + bits - 1) / bits) / (bits / 8)
//
// With the above settings, this comes out to ~3.2MB. Mostly these numbers
// were picked empirically with suspicious benchmarks.

/// A backtracking matching engine.
#[derive(Debug)]
pub struct Backtrack<'a, 'r, 't, 'c> {
    prog: &'r Program,
    input: CharInput<'t>,
    caps: &'c mut CaptureIdxs,
    m: &'a mut BackMachine,
}

/// Shared cached state between multiple invocations of a backtracking engine
/// in the same thread.
///
/// It is exported so that it can be cached by `program::Program`.
#[derive(Debug)]
pub struct BackMachine {
    jobs: Vec<Job>,
    visited: Vec<Bits>,
}

impl BackMachine {
    /// Create new empty state for the backtracking engine.
    pub fn new() -> BackMachine {
        BackMachine {
            jobs: vec![],
            visited: vec![],
        }
    }
}

/// A job is an explicit unit of stack space in the backtracking engine.
///
/// The "normal" representation is a single state transition, which corresponds
/// to an NFA state and a character in the input. However, the backtracking
/// engine must keep track of old capture group values. We use the explicit
/// stack to do it.
#[derive(Clone, Copy, Debug)]
enum Job {
    Inst { pc: InstIdx, at: InputAt },
    SaveRestore { slot: usize, old_pos: Option<usize> },
}

impl<'a, 'r, 't, 'c> Backtrack<'a, 'r, 't, 'c> {
    /// Execute the backtracking matching engine.
    ///
    /// If there's a match, `exec` returns `true` and populates the given
    /// captures accordingly.
    pub fn exec(
        prog: &'r Program,
        mut caps: &mut CaptureIdxs,
        text: &'t str,
        start: usize,
    ) -> bool {
        let input = CharInput::new(text);
        let start = input.at(start);
        let mut m = prog.backtrack.get();
        let mut b = Backtrack {
            prog: prog,
            input: input,
            caps: caps,
            m: &mut m,
        };
        b.exec_(start)
    }

    /// Returns true iff the given regex and input can be executed by this
    /// engine with reasonable memory usage.
    pub fn should_exec(prog: &'r Program, input: &str) -> bool {
        prog.insts.len() <= MAX_PROG_SIZE && input.len() <= MAX_INPUT_SIZE
    }

    fn clear(&mut self) {
        // Reset the job memory so that we start fresh.
        self.m.jobs.truncate(0);

        // Now we need to clear the bit state set.
        // We do this by figuring out how much space we need to keep track
        // of the states we've visited.
        // Then we reset all existing allocated space to 0.
        // Finally, we request more space if we need it.
        //
        // This is all a little circuitous, but doing this unsafely
        // doesn't seem to have a measurable impact on performance.
        // (Probably because backtracking is limited to such small
        // inputs/regexes in the first place.)
        let visited_len =
            (self.prog.insts.len() * (self.input.len() + 1) + BIT_SIZE - 1)
            /
            BIT_SIZE;
        self.m.visited.truncate(visited_len);
        for v in &mut self.m.visited {
            *v = 0;
        }
        if visited_len > self.m.visited.len() {
            let len = self.m.visited.len();
            self.m.visited.reserve_exact(visited_len - len);
            for _ in 0..(visited_len - len) {
                self.m.visited.push(0);
            }
        }
    }

    fn exec_(&mut self, mut at: InputAt) -> bool {
        self.clear();
        if self.prog.anchored_begin {
            return if !at.is_beginning() {
                false
            } else {
                match self.input.prefix_at(&self.prog.prefixes, at) {
                    None => false,
                    Some(at) => self.backtrack(at),
                }
            };
        }
        loop {
            if !self.prog.prefixes.is_empty() {
                at = match self.input.prefix_at(&self.prog.prefixes, at) {
                    None => return false,
                    Some(at) => at,
                };
            }
            if self.backtrack(at) {
                return true;
            }
            if at.char().is_none() {
                return false;
            }
            at = self.input.at(at.next_pos());
        }
    }

    // This `inline(always)` seems to result in about a 10-15% increase in
    // throughput on the `hard` benchmarks (over a standard `inline`). ---AG
    #[inline(always)]
    fn backtrack(&mut self, start: InputAt) -> bool {
        self.push(0, start);
        while let Some(job) = self.m.jobs.pop() {
            match job {
                Job::Inst { pc, at } => {
                    if self.step(pc, at) {
                        return true;
                    }
                }
                Job::SaveRestore { slot, old_pos } => {
                    self.caps[slot] = old_pos;
                }
            }
        }
        false
    }

    fn step(&mut self, mut pc: InstIdx, mut at: InputAt) -> bool {
        use program::Inst::*;
        loop {
            // This loop is an optimization to avoid constantly pushing/popping
            // from the stack. Namely, if we're pushing a job only to run it
            // next, avoid the push and just mutate `pc` (and possibly `at`)
            // in place.
            match self.prog.insts[pc] {
                Match => return true,
                Save(slot) => {
                    if slot < self.caps.len() {
                        // If this path doesn't work out, then we save the old
                        // capture index (if one exists) in an alternate
                        // job. If the next path fails, then the alternate
                        // job is popped and the old capture index is restored.
                        let old_pos = self.caps[slot];
                        self.push_save_restore(slot, old_pos);
                        self.caps[slot] = Some(at.pos());
                    }
                    pc += 1;
                }
                Jump(pc2) => pc = pc2,
                Split(x, y) => {
                    self.push(y, at);
                    pc = x;
                }
                EmptyLook(ref inst) => {
                    let prev = self.input.previous_at(at.pos());
                    if inst.matches(prev.char(), at.char()) {
                        pc += 1;
                    } else {
                        return false;
                    }
                }
                Char(c) => {
                    if c == at.char() {
                        pc += 1;
                        at = self.input.at(at.next_pos());
                    } else {
                        return false;
                    }
                }
                Ranges(ref inst) => {
                    if inst.matches(at.char()) {
                        pc += 1;
                        at = self.input.at(at.next_pos());
                    } else {
                        return false;
                    }
                }
            }
            if self.has_visited(pc, at) {
                return false;
            }
        }
    }

    fn push(&mut self, pc: InstIdx, at: InputAt) {
        self.m.jobs.push(Job::Inst { pc: pc, at: at });
    }

    fn push_save_restore(&mut self, slot: usize, old_pos: Option<usize>) {
        self.m.jobs.push(Job::SaveRestore { slot: slot, old_pos: old_pos });
    }

    fn has_visited(&mut self, pc: InstIdx, at: InputAt) -> bool {
        let k = pc * (self.input.len() + 1) + at.pos();
        let k1 = k / BIT_SIZE;
        let k2 = (1 << (k & (BIT_SIZE - 1))) as Bits;
        if self.m.visited[k1] & k2 == 0 {
            self.m.visited[k1] |= k2;
            false
        } else {
            true
        }
    }
}
<|endoftext|>


<commit_before>//! An implementation of disjoint set forests for union find.

/// Data structure for efficient union find.
pub struct DisjointSetForest {
    /// Number of forest elements.
    count: usize,
    /// parent[i] is the index of the parent
    /// of the element with index i. If parent[i] == i
    /// then i is a root.
    parent: Vec<usize>,
    /// tree_size[i] is the size of the tree rooted at i.
    tree_size: Vec<usize>,
}

impl DisjointSetForest {

    /// Constructs forest of singletons with count elements.
    pub fn new(count: usize) -> DisjointSetForest {
        let mut parent = vec![0 as usize; count];
        for i in 0..count {
            parent[i] = i;
        }
        let tree_size = vec![1 as usize; count];
        DisjointSetForest {
            count: count,
            parent: parent,
            tree_size: tree_size}
    }

    /// Returns the number of trees in the forest.
    pub fn num_trees(&self) -> usize {
        self.parent
            .iter()
            .enumerate()
            .fold(0, |acc, (i, p)| acc + if i == *p { 1 } else { 0 })
    }

    /// Returns index of the root of the tree containing i.
    /// Needs mutable reference to self for path compression.
    pub fn root(&mut self, i: usize) -> usize {
        let mut j = i;
        loop {
            unsafe {
                let p = *self.parent.get_unchecked(j);
                *self.parent.get_unchecked_mut(j) = *self.parent.get_unchecked(p);
                if j == p {
                    break;
                }
                j = p;
            }
        }
        j
    }

    /// Returns true if i and j are in the same tree.
    /// Need mutable reference to self for path compression.
    pub fn find(&mut self, i: usize, j: usize) -> bool {
        self.root(i) == self.root(j)
    }

    /// Unions the trees containing i and j.
    pub fn union(&mut self, i: usize, j: usize) {
        let p = self.root(i);
        let q = self.root(j);
        if p == q {
            return;
        }
        unsafe {
            let p_size = *self.tree_size.get_unchecked(p);
            let q_size = *self.tree_size.get_unchecked(q);
            if p_size < q_size {
                *self.parent.get_unchecked_mut(p) = q;
                *self.tree_size.get_unchecked_mut(q) = p_size + q_size;
            }
            else {
                *self.parent.get_unchecked_mut(q) = p;
                *self.tree_size.get_unchecked_mut(p) = p_size + q_size;
            }
        }
    }

    /// Returns the elements of each tree.
    pub fn trees(&mut self) -> Vec<Vec<usize>> {
        use std::collections::HashMap;

        // Maps a tree root to the index of the set
        // containing its children
        let mut root_sets: HashMap<usize, usize> = HashMap::new();

        let mut sets: Vec<Vec<usize>> = vec![];
        for i in 0..self.count {
            let root = self.root(i);
            match root_sets.get(&root).cloned() {
                Some(set_idx) => {
                    sets[set_idx].push(i);
                },
                None => {
                    let idx = sets.len();
                    let set = vec![i];
                    sets.push(set);
                    root_sets.insert(root, idx);
                }
            }
        }
        sets
    }
}

#[cfg(test)]
mod test {

    use super::DisjointSetForest;
    use test;
    use rand::{SeedableRng, StdRng};
    use rand::distributions::{IndependentSample, Range};

    #[test]
    fn test_trees() {
        //    3         4
        //    |        /  \
        //    1       5    7
        //   /  \     |
        //  0    2    6
        let mut forest = DisjointSetForest {
            count: 8,
            // element:     0, 1, 2, 3, 4, 5, 6, 7
            parent:    vec![1, 3, 1, 3, 4, 4, 5, 4],
            tree_size: vec![1, 3, 1, 4, 4, 2, 1, 1]
        };

        assert_eq!(forest.trees(),
            vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7]]);
    }

    #[test]
    fn test_union_find_sequence() {

        let mut forest = DisjointSetForest::new(6);
        // 0  1  2  3  4  5

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 3, 4, 5]);
        assert_eq!(forest.num_trees(), 6);

        forest.union(0, 4);
        // 0  1  2  3  5
        // |
        // 4

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 3, 0, 5]);
        assert_eq!(forest.num_trees(), 5);

        forest.union(1, 3);
        // 0  1  2  5
        // |  |
        // 4  3

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 4);

        forest.union(3, 2);
        // 0    1     5
        // |   / \
        // 4  3   2

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 1, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 3);

        forest.union(2, 4);
        //    1     5
        //  / | \
        // 0  3  2
        // |
        // 4

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![1, 1, 1, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 2);
    }

    #[bench]
    fn bench_disjoint_set_forest(b: &mut test::Bencher) {
        let num_nodes = 500;
        let num_edges = 20 * num_nodes;

        let seed_array: &[_] = &[1usize];
        let mut rng: StdRng = SeedableRng::from_seed(seed_array);
        let uniform = Range::new(0, num_nodes);

        let mut forest = DisjointSetForest::new(num_nodes);
        b.iter(|| {
            let mut count = 0;
            while count < num_edges {
                let u = uniform.ind_sample(&mut rng);
                let v = uniform.ind_sample(&mut rng);
                forest.union(u, v);
                count += 1;
            }
            test::black_box(forest.num_trees());
        });
    }
}
<commit_msg>Unionfind uses unsafe code internally, so assert that user input is within bounds<commit_after>//! An implementation of disjoint set forests for union find.

/// Data structure for efficient union find.
pub struct DisjointSetForest {
    /// Number of forest elements.
    count: usize,
    /// parent[i] is the index of the parent
    /// of the element with index i. If parent[i] == i
    /// then i is a root.
    parent: Vec<usize>,
    /// tree_size[i] is the size of the tree rooted at i.
    tree_size: Vec<usize>,
}

impl DisjointSetForest {

    /// Constructs forest of singletons with count elements.
    pub fn new(count: usize) -> DisjointSetForest {
        let mut parent = vec![0 as usize; count];
        for i in 0..count {
            parent[i] = i;
        }
        let tree_size = vec![1 as usize; count];
        DisjointSetForest {
            count: count,
            parent: parent,
            tree_size: tree_size}
    }

    /// Returns the number of trees in the forest.
    pub fn num_trees(&self) -> usize {
        self.parent
            .iter()
            .enumerate()
            .fold(0, |acc, (i, p)| acc + if i == *p { 1 } else { 0 })
    }

    /// Returns index of the root of the tree containing i.
    /// Needs mutable reference to self for path compression.
    pub fn root(&mut self, i: usize) -> usize {
        assert!(i < self.count);
        let mut j = i;
        loop {
            unsafe {
                let p = *self.parent.get_unchecked(j);
                *self.parent.get_unchecked_mut(j) = *self.parent.get_unchecked(p);
                if j == p {
                    break;
                }
                j = p;
            }
        }
        j
    }

    /// Returns true if i and j are in the same tree.
    /// Need mutable reference to self for path compression.
    pub fn find(&mut self, i: usize, j: usize) -> bool {
        assert!(i < self.count && j < self.count);
        self.root(i) == self.root(j)
    }

    /// Unions the trees containing i and j.
    pub fn union(&mut self, i: usize, j: usize) {
        assert!(i < self.count && j < self.count);
        let p = self.root(i);
        let q = self.root(j);
        if p == q {
            return;
        }
        unsafe {
            let p_size = *self.tree_size.get_unchecked(p);
            let q_size = *self.tree_size.get_unchecked(q);
            if p_size < q_size {
                *self.parent.get_unchecked_mut(p) = q;
                *self.tree_size.get_unchecked_mut(q) = p_size + q_size;
            }
            else {
                *self.parent.get_unchecked_mut(q) = p;
                *self.tree_size.get_unchecked_mut(p) = p_size + q_size;
            }
        }
    }

    /// Returns the elements of each tree.
    pub fn trees(&mut self) -> Vec<Vec<usize>> {
        use std::collections::HashMap;

        // Maps a tree root to the index of the set
        // containing its children
        let mut root_sets: HashMap<usize, usize> = HashMap::new();

        let mut sets: Vec<Vec<usize>> = vec![];
        for i in 0..self.count {
            let root = self.root(i);
            match root_sets.get(&root).cloned() {
                Some(set_idx) => {
                    sets[set_idx].push(i);
                },
                None => {
                    let idx = sets.len();
                    let set = vec![i];
                    sets.push(set);
                    root_sets.insert(root, idx);
                }
            }
        }
        sets
    }
}

#[cfg(test)]
mod test {

    use super::DisjointSetForest;
    use test;
    use rand::{SeedableRng, StdRng};
    use rand::distributions::{IndependentSample, Range};

    #[test]
    fn test_trees() {
        //    3         4
        //    |        /  \
        //    1       5    7
        //   /  \     |
        //  0    2    6
        let mut forest = DisjointSetForest {
            count: 8,
            // element:     0, 1, 2, 3, 4, 5, 6, 7
            parent:    vec![1, 3, 1, 3, 4, 4, 5, 4],
            tree_size: vec![1, 3, 1, 4, 4, 2, 1, 1]
        };

        assert_eq!(forest.trees(),
            vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7]]);
    }

    #[test]
    fn test_union_find_sequence() {

        let mut forest = DisjointSetForest::new(6);
        // 0  1  2  3  4  5

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 3, 4, 5]);
        assert_eq!(forest.num_trees(), 6);

        forest.union(0, 4);
        // 0  1  2  3  5
        // |
        // 4

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 3, 0, 5]);
        assert_eq!(forest.num_trees(), 5);

        forest.union(1, 3);
        // 0  1  2  5
        // |  |
        // 4  3

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 2, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 4);

        forest.union(3, 2);
        // 0    1     5
        // |   / \
        // 4  3   2

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![0, 1, 1, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 3);

        forest.union(2, 4);
        //    1     5
        //  / | \
        // 0  3  2
        // |
        // 4

        //                             0, 1, 2, 3, 4, 5
        assert_eq!(forest.parent, vec![1, 1, 1, 1, 0, 5]);
        assert_eq!(forest.num_trees(), 2);
    }

    #[bench]
    fn bench_disjoint_set_forest(b: &mut test::Bencher) {
        let num_nodes = 500;
        let num_edges = 20 * num_nodes;

        let seed_array: &[_] = &[1usize];
        let mut rng: StdRng = SeedableRng::from_seed(seed_array);
        let uniform = Range::new(0, num_nodes);

        let mut forest = DisjointSetForest::new(num_nodes);
        b.iter(|| {
            let mut count = 0;
            while count < num_edges {
                let u = uniform.ind_sample(&mut rng);
                let v = uniform.ind_sample(&mut rng);
                forest.union(u, v);
                count += 1;
            }
            test::black_box(forest.num_trees());
        });
    }
}
<|endoftext|>


<commit_before>use std::io::prelude::*;
use std::io;

use cargo::ops;
use cargo::core::{SourceId, Source};
use cargo::sources::RegistrySource;
use cargo::util::{CargoError, CliResult, CargoResultExt, Config};

#[derive(Deserialize)]
pub struct Options {
    flag_host: Option<String>,
    arg_token: Option<String>,
    flag_verbose: u32,
    flag_quiet: Option<bool>,
    flag_color: Option<String>,
    flag_frozen: bool,
    flag_locked: bool,
    #[serde(rename = "flag_Z")]
    flag_z: Vec<String>,
    flag_registry: Option<String>,
}

pub const USAGE: &'static str = "
Save an api token from the registry locally

Usage:
    cargo login [options] [<token>]

Options:
    -h, --help               Print this message
    --host HOST              Host to set the token for
    -v, --verbose ...        Use verbose output (-vv very verbose/build.rs output)
    -q, --quiet              No output printed to stdout
    --color WHEN             Coloring: auto, always, never
    --frozen                 Require Cargo.lock and cache are up to date
    --locked                 Require Cargo.lock is up to date
    -Z FLAG ...              Unstable (nightly-only) flags to Cargo
    --registry REGISTRY      Registry to use

";

pub fn execute(options: Options, config: &mut Config) -> CliResult {
    config.configure(options.flag_verbose,
                     options.flag_quiet,
                     &options.flag_color,
                     options.flag_frozen,
                     options.flag_locked,
                     &options.flag_z)?;

    if options.flag_registry.is_some() && !config.cli_unstable().unstable_options {
        return Err(CargoError::from("registry option is an unstable feature and requires -Zunstable-options to use.").into());
    }

    let token = match options.arg_token {
        Some(token) => token,
        None => {
            let host = match options.flag_registry {
                Some(ref _registry) => {
                    return Err(CargoError::from("token must be provided when --registry is provided.").into());
                }
                None => {
                    let src = SourceId::crates_io(config)?;
                    let mut src = RegistrySource::remote(&src, config);
                    src.update()?;
                    let config = src.config()?.unwrap();
                    options.flag_host.clone().unwrap_or(config.api.unwrap())
                }
            };
            println!("please visit {}me and paste the API Token below", host);
            let mut line = String::new();
            let input = io::stdin();
            input.lock().read_line(&mut line).chain_err(|| {
                "failed to read stdin"
            })?;
            line.trim().to_string()
        }
    };

    ops::registry_login(config, token, options.flag_registry)?;
    Ok(())
}
<commit_msg>Auto merge of #4696 - paulkernfeld:master, r=matklad<commit_after>use std::io::prelude::*;
use std::io;

use cargo::ops;
use cargo::core::{SourceId, Source};
use cargo::sources::RegistrySource;
use cargo::util::{CargoError, CliResult, CargoResultExt, Config};

#[derive(Deserialize)]
pub struct Options {
    flag_host: Option<String>,
    arg_token: Option<String>,
    flag_verbose: u32,
    flag_quiet: Option<bool>,
    flag_color: Option<String>,
    flag_frozen: bool,
    flag_locked: bool,
    #[serde(rename = "flag_Z")]
    flag_z: Vec<String>,
    flag_registry: Option<String>,
}

pub const USAGE: &'static str = "
Save an api token from the registry locally. If token is not specified, it will be read from stdin.

Usage:
    cargo login [options] [<token>]

Options:
    -h, --help               Print this message
    --host HOST              Host to set the token for
    -v, --verbose ...        Use verbose output (-vv very verbose/build.rs output)
    -q, --quiet              No output printed to stdout
    --color WHEN             Coloring: auto, always, never
    --frozen                 Require Cargo.lock and cache are up to date
    --locked                 Require Cargo.lock is up to date
    -Z FLAG ...              Unstable (nightly-only) flags to Cargo
    --registry REGISTRY      Registry to use

";

pub fn execute(options: Options, config: &mut Config) -> CliResult {
    config.configure(options.flag_verbose,
                     options.flag_quiet,
                     &options.flag_color,
                     options.flag_frozen,
                     options.flag_locked,
                     &options.flag_z)?;

    if options.flag_registry.is_some() && !config.cli_unstable().unstable_options {
        return Err(CargoError::from("registry option is an unstable feature and requires -Zunstable-options to use.").into());
    }

    let token = match options.arg_token {
        Some(token) => token,
        None => {
            let host = match options.flag_registry {
                Some(ref _registry) => {
                    return Err(CargoError::from("token must be provided when --registry is provided.").into());
                }
                None => {
                    let src = SourceId::crates_io(config)?;
                    let mut src = RegistrySource::remote(&src, config);
                    src.update()?;
                    let config = src.config()?.unwrap();
                    options.flag_host.clone().unwrap_or(config.api.unwrap())
                }
            };
            println!("please visit {}me and paste the API Token below", host);
            let mut line = String::new();
            let input = io::stdin();
            input.lock().read_line(&mut line).chain_err(|| {
                "failed to read stdin"
            })?;
            line.trim().to_string()
        }
    };

    ops::registry_login(config, token, options.flag_registry)?;
    Ok(())
}
<|endoftext|>