<commit_before>use io::key::{Cmd, Key};
use io::redraw::RedrawTask;
use state::editor::Editor;
use state::mode::Mode;

#[cfg(feature = "orbital")]
use orbclient::{EventOption, Event};

#[derive(Copy, Clone)]
/// An instruction, i.e. a command and a numeral parameter
pub struct Inst(pub Parameter, pub Cmd);

/// A numeral parameter, i.e. a number (or nothing) given before a command (toghether making an
/// instruction)
#[derive(Copy, Clone)]
pub enum Parameter {
    /// An integer as parameter
    Int(usize),
    /// Not given (the user have not defined any numeral parameter to this command)
    Null,
}
impl Parameter {
    /// Either unwrap the Int(n) to n or fallback to a given value
    #[inline]
    pub fn or(self, fallback: usize) -> usize {
        if let Parameter::Int(n) = self {
            n
        } else {
            fallback
        }
    }
    /// Fallback to one (default)
    #[inline]
    pub fn d(self) -> usize {
        self.or(1)
    }
}

impl Editor {
    /// Get the next character input. Useful for commands taking a character as post-parameter,
    /// such as r (replace).
    pub fn get_char(&mut self) -> char {
        #[cfg(feature = "orbital")]
        loop {
            match self.window
                      .events().next()
                      .unwrap_or(Event::new())
                      .to_option() {
                EventOption::Key(k) => {
                    if let Some(Key::Char(c)) = self.key_state.feed(k) {
                        self.status_bar.cmd.push(c);
                        self.redraw_task = RedrawTask::StatusBar;
                        return c;
                    }
                },
                _ => {},
            }
        }
    }

    /// Get the next instruction, i.e. the next input of a command together with a numeral
    /// parameter.
    pub fn get_inst(&mut self) -> Inst {
        let mut n = 0;
        let mut unset = true;

        let mut key = Key::Null;
        self.status_bar.cmd = String::new();

        // self.status_bar.cmd = String::new();
        #[cfg(feature = "orbital")]
        loop {
             match self.window.events().next()
                       .unwrap_or(Event::new())
                       .to_option() {
                EventOption::Key(key_event) => {
                    if let Some(k) = self.key_state.feed(key_event) {
                        let c = k.to_char();
                        self.status_bar.cmd.push(c);
                        self.redraw_status_bar();

                        match self.cursor().mode {
                            Mode::Primitive(_) => {
                                key = k;
                            }
                            Mode::Command(_) => {
                                n = match c {
                                    '0' ... '9' => {
                                        unset = false;
                                        n * 10 + ((c as u8) - b'0') as usize
                                    }
                                    _ => {

                                        key = k;
                                        n
                                    }
                                };
                            }

                        }
                    }
                    match key {
                        Key::Null => {},
                        _ => {
                            return Inst(
                                if unset {
                                    Parameter::Null
                                } else {
                                    Parameter::Int(n)
                                },
                                Cmd { key: key }
                            );
                        },
                    }
                },
                EventOption::Quit(_) => {
                    return Inst(Parameter::Null, Cmd { key: Key::Quit });
                },
                _ => {},
            }
        }

    }
}
<commit_msg>Fix missing orbital events<commit_after>use io::key::{Cmd, Key};
use io::redraw::RedrawTask;
use state::editor::Editor;
use state::mode::Mode;

#[cfg(feature = "orbital")]
use orbclient::EventOption;

#[derive(Copy, Clone)]
/// An instruction, i.e. a command and a numeral parameter
pub struct Inst(pub Parameter, pub Cmd);

/// A numeral parameter, i.e. a number (or nothing) given before a command (toghether making an
/// instruction)
#[derive(Copy, Clone)]
pub enum Parameter {
    /// An integer as parameter
    Int(usize),
    /// Not given (the user have not defined any numeral parameter to this command)
    Null,
}
impl Parameter {
    /// Either unwrap the Int(n) to n or fallback to a given value
    #[inline]
    pub fn or(self, fallback: usize) -> usize {
        if let Parameter::Int(n) = self {
            n
        } else {
            fallback
        }
    }
    /// Fallback to one (default)
    #[inline]
    pub fn d(self) -> usize {
        self.or(1)
    }
}

impl Editor {
    /// Get the next character input. Useful for commands taking a character as post-parameter,
    /// such as r (replace).
    pub fn get_char(&mut self) -> char {
        #[cfg(feature = "orbital")]
        loop {
            for event in  self.window.events() {
                match event.to_option(){
                    EventOption::Key(k) => {
                        if let Some(Key::Char(c)) = self.key_state.feed(k) {
                            self.status_bar.cmd.push(c);
                            self.redraw_task = RedrawTask::StatusBar;
                            return c;
                        }
                    },
                    _ => {},
                }
            }
        }
    }

    /// Get the next instruction, i.e. the next input of a command together with a numeral
    /// parameter.
    pub fn get_inst(&mut self) -> Inst {
        let mut n = 0;
        let mut unset = true;

        let mut key = Key::Null;
        self.status_bar.cmd = String::new();

        // self.status_bar.cmd = String::new();
        #[cfg(feature = "orbital")]
        loop {
            for event in self.window.events() {
                match event.to_option() {
                    EventOption::Key(key_event) => {
                        if let Some(k) = self.key_state.feed(key_event) {
                            let c = k.to_char();
                            self.status_bar.cmd.push(c);
                            self.redraw_status_bar();

                            match self.cursor().mode {
                                Mode::Primitive(_) => {
                                    key = k;
                                }
                                Mode::Command(_) => {
                                    n = match c {
                                        '0' ... '9' => {
                                            unset = false;
                                            n * 10 + ((c as u8) - b'0') as usize
                                        }
                                        _ => {

                                            key = k;
                                            n
                                        }
                                    };
                                }

                            }
                        }
                        match key {
                            Key::Null => {},
                            _ => {
                                return Inst(
                                    if unset {
                                        Parameter::Null
                                    } else {
                                        Parameter::Int(n)
                                    },
                                    Cmd { key: key }
                                );
                            },
                        }
                    },
                    EventOption::Quit(_) => {
                        return Inst(Parameter::Null, Cmd { key: Key::Quit });
                    },
                    _ => {},
                }
            }
        }

    }
}
<|endoftext|>


<commit_before>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! let tokens = tokenize("word");
//! assert_eq!(tokens.len(), 1);
//!
//! let tokens = tokenize("show interface");
//! assert_eq!(tokens.len(), 3);
//!
//! let tokens = tokenize("echo -n \"a b c\"");
//! assert_eq!(tokens.len(), 5);
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Special => {
                    panic!("Special not yet implemented.");
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => panic!("Escaping backslash at end of input"),
            State::Doublequote => panic!("Unclosed double quote at end of input"),
            State::DoublequoteBackslash => panic!("Escaped doublequote at end of input"),
            State::Special => panic!("Special not yet implemented"),
        }
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Vec<Token> {
    let mut tokenizer = Tokenizer::new(text);
    tokenizer.tokenize();
    tokenizer.tokens
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        let ts = tokenize("");
        assert_eq!(ts.len(), 0);
    }

    #[test]
    fn single_word() {
        let ts = tokenize("a");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
    }

    #[test]
    fn multiple_words() {
        let ts = tokenize(" aa bb  ccc ");
        assert_eq!(ts.len(), 7);
        assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
        assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
        assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
        assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
        assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
        assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
        assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
    }

    #[test]
    fn quoted_text() {
        let ts = tokenize("a \"b c\"");
        assert_eq!(ts.len(), 3);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
        assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
        assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
    }

    #[test]
    fn escaped_whitespace_in_word() {
        let ts = tokenize("a\\ b");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
    }
}
<commit_msg>tokenizer: Add doc test for \\ handling in words.<commit_after>//! # Command Tokenization
//!
//! The command parser needs to be able to tokenize commands
//! into their constituent words and whitespace.
//!
//! The `tokenizer` breaks source text into a vector of tokens
//! which can be either whitespace or a word. The tokenizer
//! handles using double quotes to provide a single token
//! which may include whitespace.
//!
//! Tokens also track their source location within the source
//! text. This allows the parser using the tokenizer to provide
//! better error highlighting and other functionality.
//!
//! # Examples
//!
//! ```
//! use commands::tokenizer::tokenize;
//!
//! let tokens = tokenize("word");
//! assert_eq!(tokens.len(), 1);
//!
//! let tokens = tokenize("show interface");
//! assert_eq!(tokens.len(), 3);
//!
//! let tokens = tokenize("echo -n \"a b c\"");
//! assert_eq!(tokens.len(), 5);
//!
//! let tokens = tokenize("ls My\\ Documents");
//! assert_eq!(tokens.len(), 3);
//! assert_eq!(tokens[2].text, "My\\ Documents");
//! ```

/// A position within a string.
///
/// The `SourceOffset` tracks 2 different ways of locating the
/// position:
///
/// * The index of the character within the string.
/// * The column and line number of the character.
///
/// The `SourceOffset` is typically used as a pair of offsets
/// indicating the start and end of a range of text as used
/// by the `SourceLocation`.
#[derive(Debug,PartialEq)]
pub struct SourceOffset {
    /// The index of this character within the string.
    pub char: usize,
    /// The line number on which this character may be found.
    pub line: usize,
    /// The column on which this character may be found.
    pub column: usize,
}

impl SourceOffset {
    /// Construct a `SourceOffset`.
    pub fn new(char: usize, line: usize, column: usize) -> SourceOffset {
        SourceOffset {
            char: char,
            line: line,
            column: column,
        }
    }
}

/// A range within a string.
#[derive(Debug,PartialEq)]
pub struct SourceLocation {
    /// The start of the range.
    pub start: SourceOffset,
    /// The end of the range.
    pub end: SourceOffset,
}

impl SourceLocation {
    /// Construct a `SourceLocation`.
    pub fn new(start: SourceOffset, end: SourceOffset) -> SourceLocation {
        SourceLocation {
            start: start,
            end: end,
        }
    }
}

/// The role that a token plays: `Whitespace` or `Word`.
#[derive(Clone,Copy,Debug,PartialEq)]
pub enum TokenType {
    /// Internal usage only.
    Invalid,
    /// The token represents whitespace and not a word.
    Whitespace,
    /// The token represents a word within the string. This
    /// takes double quoted strings into account.
    Word,
}

/// A token from a body of text.
///
/// The lifetime parameter `'t` refers to the lifetime
/// of the body of text that was tokenized, creating this token.
#[derive(Debug,PartialEq)]
pub struct Token<'t> {
    /// The text of the token.
    pub text: &'t str,
    /// The type of the token (`Whitespace` or `Word`).
    pub token_type: TokenType,
    /// The location of the token in the source body of text.
    pub location: SourceLocation,
}

impl<'t> Token<'t> {
    /// Construct a `Token`. The lifetime parameter `'t` refers to the lifetime of the
    /// text being tokenized.
    pub fn new(text: &'t str, token_type: TokenType, location: SourceLocation) -> Token {
        Token {
            text: text,
            token_type: token_type,
            location: location,
        }
    }
}

#[derive(Clone,Copy,PartialEq)]
enum State {
    Initial,
    Special,
    Whitespace,
    Doublequote,
    DoublequoteBackslash,
    Word,
    WordBackslash,
}

struct Tokenizer<'t> {
    text: &'t str,
    state: State,
    token_type: TokenType,
    token_start: usize,
    token_end: usize,
    tokens: Vec<Token<'t>>,
}

impl<'t> Tokenizer<'t> {
    fn new(text: &'t str) -> Tokenizer {
        Tokenizer {
            text: text,
            state: State::Initial,
            token_type: TokenType::Invalid,
            token_start: 0,
            token_end: 0,
            tokens: vec![],
        }
    }

    fn reset(&mut self) {
        self.state = State::Initial;
        self.token_type = TokenType::Invalid;
        self.token_start = 0;
        self.token_end = 0;
    }

    fn reduce(&mut self) {
        let token_text = &self.text[self.token_start..self.token_end + 1];
        let loc = SourceLocation::new(SourceOffset::new(self.token_start, 0, self.token_start),
                                      SourceOffset::new(self.token_end, 0, self.token_end));
        self.tokens.push(Token::new(token_text, self.token_type, loc));
        self.reset();
    }

    fn shift(&mut self, offset: usize, next_state: State) {
        self.recognize(offset, next_state);
        self.token_end = offset;
        self.state = next_state;
    }

    fn recognize(&mut self, offset: usize, next_state: State) {
        if self.token_type == TokenType::Invalid {
            self.token_type = if next_state == State::Whitespace {
                TokenType::Whitespace
            } else {
                TokenType::Word
            };
            self.token_start = offset;
        }
    }

    fn special(&mut self, offset: usize) {
        self.shift(offset, State::Special);
        self.reduce();
    }

    fn initial(&mut self, offset: usize, c: char) {
        if c.is_whitespace() {
            self.shift(offset, State::Whitespace);
        } else if c == ';' {
            self.special(offset);
        } else if c == '?' {
            self.special(offset);
        } else if c == '|' {
            self.special(offset);
        } else if c == '"' {
            self.shift(offset, State::Doublequote);
        } else if c == '\\' {
            self.recognize(offset, State::Word);
            self.shift(offset, State::WordBackslash);
        } else {
            self.shift(offset, State::Word);
        }
    }

    fn tokenize(&mut self) {
        for (offset, c) in self.text.chars().enumerate() {
            match self.state {
                State::Initial => self.initial(offset, c),
                State::Whitespace => {
                    if c.is_whitespace() {
                        self.shift(offset, State::Whitespace);
                    } else {
                        self.reduce();
                        self.initial(offset, c);
                    };
                }
                State::Word => {
                    if c.is_whitespace() {
                        self.reduce();
                        self.shift(offset, State::Whitespace);
                    } else if c == ';' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '|' {
                        self.reduce();
                        self.special(offset);
                    } else if c == '"' {
                        self.reduce();
                        self.shift(offset, State::Doublequote);
                    } else if c == '\\' {
                        self.shift(offset, State::WordBackslash);
                    } else {
                        self.shift(offset, State::Word);
                    }
                }
                State::WordBackslash => {
                    // XXX: This should be if !c.is_control() perhaps?
                    if c.is_alphanumeric() || c.is_whitespace() {
                        self.shift(offset, State::Word);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Doublequote => {
                    if c == '"' {
                        self.shift(offset, State::Doublequote);
                        self.reduce();
                    } else if c == '\\' {
                        self.shift(offset, State::DoublequoteBackslash);
                    } else {
                        self.shift(offset, State::Doublequote);
                    };
                }
                State::DoublequoteBackslash => {
                    if !c.is_whitespace() {
                        self.shift(offset, State::Doublequote);
                    } else {
                        panic!("Character not allowed here.");
                    };
                }
                State::Special => {
                    panic!("Special not yet implemented.");
                }
            }
        }

        // Now for the end of the text...
        match self.state {
            State::Initial => {}
            State::Word => self.reduce(),
            State::Whitespace => self.reduce(),
            State::WordBackslash => panic!("Escaping backslash at end of input"),
            State::Doublequote => panic!("Unclosed double quote at end of input"),
            State::DoublequoteBackslash => panic!("Escaped doublequote at end of input"),
            State::Special => panic!("Special not yet implemented"),
        }
    }
}

/// Tokenize a body of text.
pub fn tokenize(text: &str) -> Vec<Token> {
    let mut tokenizer = Tokenizer::new(text);
    tokenizer.tokenize();
    tokenizer.tokens
}

#[cfg(test)]
mod test {
    use super::*;

    fn mk_token(text: &str, token_type: TokenType, start: usize, end: usize) -> Token {
        Token::new(text,
                   token_type,
                   SourceLocation::new(SourceOffset::new(start, 0, start),
                                       SourceOffset::new(end, 0, end)))
    }

    #[test]
    fn empty_test() {
        let ts = tokenize("");
        assert_eq!(ts.len(), 0);
    }

    #[test]
    fn single_word() {
        let ts = tokenize("a");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
    }

    #[test]
    fn multiple_words() {
        let ts = tokenize(" aa bb  ccc ");
        assert_eq!(ts.len(), 7);
        assert_eq!(ts[0], mk_token(" ", TokenType::Whitespace, 0, 0));
        assert_eq!(ts[1], mk_token("aa", TokenType::Word, 1, 2));
        assert_eq!(ts[2], mk_token(" ", TokenType::Whitespace, 3, 3));
        assert_eq!(ts[3], mk_token("bb", TokenType::Word, 4, 5));
        assert_eq!(ts[4], mk_token("  ", TokenType::Whitespace, 6, 7));
        assert_eq!(ts[5], mk_token("ccc", TokenType::Word, 8, 10));
        assert_eq!(ts[6], mk_token(" ", TokenType::Whitespace, 11, 11));
    }

    #[test]
    fn quoted_text() {
        let ts = tokenize("a \"b c\"");
        assert_eq!(ts.len(), 3);
        assert_eq!(ts[0], mk_token("a", TokenType::Word, 0, 0));
        assert_eq!(ts[1], mk_token(" ", TokenType::Whitespace, 1, 1));
        assert_eq!(ts[2], mk_token("\"b c\"", TokenType::Word, 2, 6));
    }

    #[test]
    fn escaped_whitespace_in_word() {
        let ts = tokenize("a\\ b");
        assert_eq!(ts.len(), 1);
        assert_eq!(ts[0], mk_token("a\\ b", TokenType::Word, 0, 3));
    }
}
<|endoftext|>


<commit_before>//! Transports for communicating with the docker daemon

extern crate hyper;

use self::super::{Error, Result};
use hyper::Body;
use hyper::client::{Client, HttpConnector};
use hyper::{Method, Request, Response, StatusCode};
use hyper::header;
use hyper_openssl::HttpsConnector;
use hyper::rt::{Future, Stream};
use hyperlocal::Uri as DomainUri;
use hyperlocal::UnixConnector;
use mime::Mime;
use rustc_serialize::json;
use std::fmt;
use std::cell::{RefMut, RefCell};
use std::io::{BufReader, Cursor};
use std::io::Read;
use tokio::runtime::Runtime;

pub fn tar() -> Mime {
    "application/tar".parse().unwrap()
}

/// Transports are types which define the means of communication
/// with the docker daemon
pub enum Transport {
    /// A network tcp interface
    Tcp {
        client: Client<HttpConnector>,
        runtime: RefCell<Runtime>,
        host: String,
    },
    /// TCP/TLS.w
    EncryptedTcp {
        client: Client<HttpsConnector<HttpConnector>>,
        runtime: RefCell<Runtime>,
        host: String,
    },
    /// A Unix domain socket
    Unix {
        client: Client<UnixConnector>,
        runtime: RefCell<Runtime>,
        path: String,
    },
}

impl fmt::Debug for Transport {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Transport::Tcp { ref host, .. } => write!(f, "Tcp({})", host),
            Transport::EncryptedTcp { ref host, .. } => write!(f, "EncryptedTcp({})", host),
            Transport::Unix { ref path, .. } => write!(f, "Unix({})", path),
        }
    }
}

impl Transport {
    pub fn request<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<String>
    where
        B: Into<Body>,
    {
        let mut res = self.stream(method, endpoint, body)?;
        let mut body = String::new();
        res.read_to_string(&mut body)?;
        debug!("{} raw response: {}", endpoint, body);
        Ok(body)
    }

    /// Builds an HTTP request.
    fn build_request<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<Request<Body>>
    where
        B: Into<Body>,
    {
        let mut builder = Request::builder();
        let req = match *self {
            Transport::Tcp { ref host, .. } => {
                builder.method(method).uri(&format!("{}{}", host, endpoint))
            }
            Transport::EncryptedTcp { ref host, .. } => {
                builder.method(method).uri(&format!("{}{}", host, endpoint))
            }
            Transport::Unix { ref path, .. } => {
                let uri: hyper::Uri = DomainUri::new(&path, endpoint).into();
                builder.method(method).uri(&uri.to_string())
            }
        };
        let req = req.header(header::HOST, "");

        match body {
            Some((b, c)) => Ok(req
                .header(header::CONTENT_TYPE, &c.to_string()[..])
                .body(b.into())?),
            _ => Ok(req.body(Body::empty())?),
        }
    }

    pub fn stream<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<Box<Read>>
    where
        B: Into<Body>,
    {
        let req = self.build_request(method, endpoint, body)?;
        let res = self.send_request(req)?;

        match res.status() {
            StatusCode::OK |
            StatusCode::CREATED |
            StatusCode::SWITCHING_PROTOCOLS => {
                let chunk = self.runtime().block_on(res.into_body().concat2())?;
                Ok(Box::new(Cursor::new(chunk.into_iter().collect::<Vec<u8>>())))
            },
            StatusCode::NO_CONTENT => Ok(
                Box::new(BufReader::new("".as_bytes())),
            ),
            // todo: constantize these
            StatusCode::BAD_REQUEST => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "bad parameter"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::NOT_FOUND => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "not found".to_owned(),
                    ),
                })
            }
            StatusCode::NOT_MODIFIED => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "not modified"
                            .to_owned(),
                    ),
                })
            },
            StatusCode::NOT_ACCEPTABLE => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "not acceptable"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::CONFLICT => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "conflict found"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::INTERNAL_SERVER_ERROR => {
                Err(Error::Fault {
                    code: res.status(),
                    message: get_error_message(res).unwrap_or(
                        "internal server error"
                            .to_owned(),
                    ),
                })
            }
            _ => unreachable!(),
        }
    }

    fn send_request(&self, req: Request<hyper::Body>) -> Result<hyper::Response<Body>> {
        let req = match self {
            Transport::Tcp { ref client, .. } => client.request(req),
            Transport::EncryptedTcp { ref client, .. } => client.request(req),
            Transport::Unix { ref client, .. } => client.request(req),
        };

        self.runtime().block_on(req).map_err(Error::Hyper)
    }

    fn runtime(&self) -> RefMut<Runtime> {
        match self {
            Transport::Tcp { ref runtime, .. } => runtime.borrow_mut(),
            Transport::EncryptedTcp { ref runtime, .. } => runtime.borrow_mut(),
            Transport::Unix { ref runtime, .. } => runtime.borrow_mut(),
        }
    }
}

/// Extract the error message content from an HTTP response that
/// contains a Docker JSON error structure.
fn get_error_message(res: Response<Body>) -> Option<String> {
    let chunk = match res.into_body().concat2().wait() {
        Ok(c) => c,
        Err(..) => return None,
    };

    match String::from_utf8(chunk.into_iter().collect()) {
        Ok(output) => {
            let json_response = json::Json::from_str(output.as_str()).ok();
            let message = json_response
                .as_ref()
                .and_then(|x| x.as_object())
                .and_then(|x| x.get("message"))
                .and_then(|x| x.as_string())
                .map(|x| x.to_owned());

            message
        }
        Err(..) => {
            None
        },
    }
}
<commit_msg>Remove use of wait() in get_error_message()<commit_after>//! Transports for communicating with the docker daemon

extern crate hyper;

use self::super::{Error, Result};
use hyper::Body;
use hyper::client::{Client, HttpConnector};
use hyper::{Method, Request, Response, StatusCode};
use hyper::header;
use hyper_openssl::HttpsConnector;
use hyper::rt::Stream;
use hyperlocal::Uri as DomainUri;
use hyperlocal::UnixConnector;
use mime::Mime;
use rustc_serialize::json;
use std::fmt;
use std::cell::{RefMut, RefCell};
use std::io::{BufReader, Cursor};
use std::io::Read;
use tokio::runtime::Runtime;

pub fn tar() -> Mime {
    "application/tar".parse().unwrap()
}

/// Transports are types which define the means of communication
/// with the docker daemon
pub enum Transport {
    /// A network tcp interface
    Tcp {
        client: Client<HttpConnector>,
        runtime: RefCell<Runtime>,
        host: String,
    },
    /// TCP/TLS.w
    EncryptedTcp {
        client: Client<HttpsConnector<HttpConnector>>,
        runtime: RefCell<Runtime>,
        host: String,
    },
    /// A Unix domain socket
    Unix {
        client: Client<UnixConnector>,
        runtime: RefCell<Runtime>,
        path: String,
    },
}

impl fmt::Debug for Transport {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Transport::Tcp { ref host, .. } => write!(f, "Tcp({})", host),
            Transport::EncryptedTcp { ref host, .. } => write!(f, "EncryptedTcp({})", host),
            Transport::Unix { ref path, .. } => write!(f, "Unix({})", path),
        }
    }
}

impl Transport {
    pub fn request<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<String>
    where
        B: Into<Body>,
    {
        let mut res = self.stream(method, endpoint, body)?;
        let mut body = String::new();
        res.read_to_string(&mut body)?;
        debug!("{} raw response: {}", endpoint, body);
        Ok(body)
    }

    /// Builds an HTTP request.
    fn build_request<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<Request<Body>>
    where
        B: Into<Body>,
    {
        let mut builder = Request::builder();
        let req = match *self {
            Transport::Tcp { ref host, .. } => {
                builder.method(method).uri(&format!("{}{}", host, endpoint))
            }
            Transport::EncryptedTcp { ref host, .. } => {
                builder.method(method).uri(&format!("{}{}", host, endpoint))
            }
            Transport::Unix { ref path, .. } => {
                let uri: hyper::Uri = DomainUri::new(&path, endpoint).into();
                builder.method(method).uri(&uri.to_string())
            }
        };
        let req = req.header(header::HOST, "");

        match body {
            Some((b, c)) => Ok(req
                .header(header::CONTENT_TYPE, &c.to_string()[..])
                .body(b.into())?),
            _ => Ok(req.body(Body::empty())?),
        }
    }

    pub fn stream<B>(
        &self,
        method: Method,
        endpoint: &str,
        body: Option<(B, Mime)>,
    ) -> Result<Box<Read>>
    where
        B: Into<Body>,
    {
        let req = self.build_request(method, endpoint, body)?;
        let res = self.send_request(req)?;

        match res.status() {
            StatusCode::OK |
            StatusCode::CREATED |
            StatusCode::SWITCHING_PROTOCOLS => {
                let chunk = self.runtime().block_on(res.into_body().concat2())?;
                Ok(Box::new(Cursor::new(chunk.into_iter().collect::<Vec<u8>>())))
            },
            StatusCode::NO_CONTENT => Ok(
                Box::new(BufReader::new("".as_bytes())),
            ),
            // todo: constantize these
            StatusCode::BAD_REQUEST => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "bad parameter"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::NOT_FOUND => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "not found".to_owned(),
                    ),
                })
            }
            StatusCode::NOT_MODIFIED => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "not modified"
                            .to_owned(),
                    ),
                })
            },
            StatusCode::NOT_ACCEPTABLE => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "not acceptable"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::CONFLICT => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "conflict found"
                            .to_owned(),
                    ),
                })
            }
            StatusCode::INTERNAL_SERVER_ERROR => {
                Err(Error::Fault {
                    code: res.status(),
                    message: self.get_error_message(res).unwrap_or(
                        "internal server error"
                            .to_owned(),
                    ),
                })
            }
            _ => unreachable!(),
        }
    }

    fn send_request(&self, req: Request<hyper::Body>) -> Result<hyper::Response<Body>> {
        let req = match self {
            Transport::Tcp { ref client, .. } => client.request(req),
            Transport::EncryptedTcp { ref client, .. } => client.request(req),
            Transport::Unix { ref client, .. } => client.request(req),
        };

        self.runtime().block_on(req).map_err(Error::Hyper)
    }

    fn runtime(&self) -> RefMut<Runtime> {
        match self {
            Transport::Tcp { ref runtime, .. } => runtime.borrow_mut(),
            Transport::EncryptedTcp { ref runtime, .. } => runtime.borrow_mut(),
            Transport::Unix { ref runtime, .. } => runtime.borrow_mut(),
        }
    }

    /// Extract the error message content from an HTTP response that
    /// contains a Docker JSON error structure.
    fn get_error_message(&self, res: Response<Body>) -> Option<String> {
        let chunk = match self.runtime().block_on(res.into_body().concat2()) {
            Ok(c) => c,
            Err(..) => return None,
        };

        match String::from_utf8(chunk.into_iter().collect()) {
            Ok(output) => {
                let json_response = json::Json::from_str(output.as_str()).ok();
                let message = json_response
                    .as_ref()
                    .and_then(|x| x.as_object())
                    .and_then(|x| x.get("message"))
                    .and_then(|x| x.as_string())
                    .map(|x| x.to_owned());

                message
            }
            Err(..) => {
                None
            },
        }
    }
}

<|endoftext|>


<commit_before>
use vertex::{RawVertex, Vertex};
use edge::{Edge, EdgeProxy};
use graph::TraversableToVertex;
use path::{Path, Element};

// use std::iter::Iterator;

// define a traversal as a series of stages

pub struct GraphQuery {
    // many paths
    pub paths: Vec<Path>
}

impl GraphQuery {
    pub fn new(vertices: Vec<Vertex>) -> GraphQuery {
        // create a new path for each proxy, add the proxy
        let mut paths = Vec::new();
        for p in vertices {
            let mut path = Path::new(p);
            paths.push(path)
        }
        GraphQuery{paths:paths}
    }

    pub fn empty() -> GraphQuery {
        // return an empty graph query
        GraphQuery{paths:Vec::new()}
    }

    pub fn get_paths(&self) -> &Vec<Path> {
        &self.paths
    }
    pub fn push(&mut self, path:Path) {
        self.paths.push(path)
    }

    /*
    generic mapper for our queries
    future this may use worker threads to perform
    traversals
     */
    fn map<F: Fn(&Path) -> Vec<Element>>(&self, closure: F) -> GraphQuery  {
        let mut result = GraphQuery::empty(); // result
        for path in self.paths.iter() {
            let mut tmp = closure(path);
            let mut tmp2 = tmp.as_slice();
            // currently gets back a Vec<Path> but what if it gets Elements?
            let new_elements = path.permute(tmp2);
            result.paths.push_all(&new_elements);
        }
        result
    }

    pub fn outV(&self) -> GraphQuery {
        // needs to be modified to return a Vec of elements or a slice?
        // map can figure out the rest
        let f = |path: &Path| {
            println!("applying outV");
            //take the final element in the path
            let element = path.last().unwrap();
            let result : Vec<Element> = Vec::new();
            // let ref result : Vec<Element> = match element {
            //     &Element::Vertex(ref v) =>
            //         v.outV().iter().map(|&v| Element::Vertex(v)).collect(),
            //     &Element::Edge(ref e) =>
            //         Vec::new()
            // };
            // apply outV
            result
        };
        self.map(f)
    }

//    pub fn inV(&self) -> GraphQuery {
//        let g = GraphQuery::empty();
//        let f = |element: &Element| {
//            element.inV()
//        };
//        self.map(f)
//    }
}
<commit_msg>fixed push_all issue<commit_after>
use vertex::{RawVertex, Vertex};
use edge::{Edge, EdgeProxy};
use graph::TraversableToVertex;
use path::{Path, Element};
use std::iter::Extend;

// use std::iter::Iterator;

// define a traversal as a series of stages

pub struct GraphQuery {
    // many paths
    pub paths: Vec<Path>
}

impl GraphQuery {
    pub fn new(vertices: Vec<Vertex>) -> GraphQuery {
        // create a new path for each proxy, add the proxy
        let mut paths = Vec::new();
        for p in vertices {
            let mut path = Path::new(p);
            paths.push(path)
        }
        GraphQuery{paths:paths}
    }

    pub fn empty() -> GraphQuery {
        // return an empty graph query
        GraphQuery{paths:Vec::new()}
    }

    pub fn get_paths(&self) -> &Vec<Path> {
        &self.paths
    }
    pub fn push(&mut self, path:Path) {
        self.paths.push(path)
    }

    /*
    generic mapper for our queries
    future this may use worker threads to perform
    traversals
     */
    fn map<F: Fn(&Path) -> Vec<Element>>(&self, closure: F) -> GraphQuery  {
        let mut result = GraphQuery::empty(); // result
        for path in self.paths.iter() {
            let mut tmp = closure(path);
            let mut tmp2 = tmp.as_slice();
            // currently gets back a Vec<Path> but what if it gets Elements?
            let new_elements = path.permute(tmp2);
            result.paths.extend(new_elements);
        }
        result
    }

    pub fn outV(&self) -> GraphQuery {
        // needs to be modified to return a Vec of elements or a slice?
        // map can figure out the rest
        let f = |path: &Path| {
            println!("applying outV");
            //take the final element in the path
            let element = path.last().unwrap();
            let result : Vec<Element> = Vec::new();
            // let ref result : Vec<Element> = match element {
            //     &Element::Vertex(ref v) =>
            //         v.outV().iter().map(|&v| Element::Vertex(v)).collect(),
            //     &Element::Edge(ref e) =>
            //         Vec::new()
            // };
            // apply outV
            result
        };
        self.map(f)
    }

//    pub fn inV(&self) -> GraphQuery {
//        let g = GraphQuery::empty();
//        let f = |element: &Element| {
//            element.inV()
//        };
//        self.map(f)
//    }
}
<|endoftext|>


<commit_before>use super::token_conflicts::TokenConflictMap;
use crate::generate::dedup::split_state_id_groups;
use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar, VariableType};
use crate::generate::rules::{AliasMap, Symbol, TokenSet};
use crate::generate::tables::{ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry};
use log::info;
use std::collections::{HashMap, HashSet};
use std::mem;

pub(crate) fn minimize_parse_table(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let mut minimizer = Minimizer {
        parse_table,
        syntax_grammar,
        lexical_grammar,
        token_conflict_map,
        keywords,
        simple_aliases,
    };
    minimizer.merge_compatible_states();
    minimizer.remove_unit_reductions();
    minimizer.remove_unused_states();
}

struct Minimizer<'a> {
    parse_table: &'a mut ParseTable,
    syntax_grammar: &'a SyntaxGrammar,
    lexical_grammar: &'a LexicalGrammar,
    token_conflict_map: &'a TokenConflictMap<'a>,
    keywords: &'a TokenSet,
    simple_aliases: &'a AliasMap,
}

impl<'a> Minimizer<'a> {
    fn remove_unit_reductions(&mut self) {
        let mut aliased_symbols = HashSet::new();
        for variable in &self.syntax_grammar.variables {
            for production in &variable.productions {
                for step in &production.steps {
                    if step.alias.is_some() {
                        aliased_symbols.insert(step.symbol);
                    }
                }
            }
        }

        let mut unit_reduction_symbols_by_state = HashMap::new();
        for (i, state) in self.parse_table.states.iter().enumerate() {
            let mut only_unit_reductions = true;
            let mut unit_reduction_symbol = None;
            for (_, entry) in &state.terminal_entries {
                for action in &entry.actions {
                    match action {
                        ParseAction::ShiftExtra => continue,
                        ParseAction::Reduce {
                            child_count: 1,
                            production_id: 0,
                            symbol,
                            ..
                        } => {
                            if !self.simple_aliases.contains_key(&symbol)
                                && !aliased_symbols.contains(&symbol)
                                && self.syntax_grammar.variables[symbol.index].kind
                                    != VariableType::Named
                                && (unit_reduction_symbol.is_none()
                                    || unit_reduction_symbol == Some(symbol))
                            {
                                unit_reduction_symbol = Some(symbol);
                                continue;
                            }
                        }
                        _ => {}
                    }
                    only_unit_reductions = false;
                    break;
                }

                if !only_unit_reductions {
                    break;
                }
            }

            if let Some(symbol) = unit_reduction_symbol {
                if only_unit_reductions {
                    unit_reduction_symbols_by_state.insert(i, *symbol);
                }
            }
        }

        for state in self.parse_table.states.iter_mut() {
            let mut done = false;
            while !done {
                done = true;
                state.update_referenced_states(|other_state_id, state| {
                    if let Some(symbol) = unit_reduction_symbols_by_state.get(&other_state_id) {
                        done = false;
                        state.nonterminal_entries[symbol]
                    } else {
                        other_state_id
                    }
                })
            }
        }
    }

    fn merge_compatible_states(&mut self) {
        let core_count = 1 + self
            .parse_table
            .states
            .iter()
            .map(|state| state.core_id)
            .max()
            .unwrap();

        // Initially group the states by their parse item set core.
        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
        for (i, state) in self.parse_table.states.iter().enumerate() {
            state_ids_by_group_id[state.core_id].push(i);
            group_ids_by_state_id.push(state.core_id);
        }

        split_state_id_groups(
            &self.parse_table.states,
            &mut state_ids_by_group_id,
            &mut group_ids_by_state_id,
            0,
            |left, right, groups| self.states_conflict(left, right, groups),
        );

        while split_state_id_groups(
            &self.parse_table.states,
            &mut state_ids_by_group_id,
            &mut group_ids_by_state_id,
            0,
            |left, right, groups| self.state_successors_differ(left, right, groups),
        ) {
            continue;
        }

        let error_group_index = state_ids_by_group_id
            .iter()
            .position(|g| g.contains(&0))
            .unwrap();
        let start_group_index = state_ids_by_group_id
            .iter()
            .position(|g| g.contains(&1))
            .unwrap();
        state_ids_by_group_id.swap(error_group_index, 0);
        state_ids_by_group_id.swap(start_group_index, 1);

        // Create a list of new parse states: one state for each group of old states.
        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
        for state_ids in &state_ids_by_group_id {
            // Initialize the new state based on the first old state in the group.
            let mut parse_state = ParseState::default();
            mem::swap(&mut parse_state, &mut self.parse_table.states[state_ids[0]]);

            // Extend the new state with all of the actions from the other old states
            // in the group.
            for state_id in &state_ids[1..] {
                let mut other_parse_state = ParseState::default();
                mem::swap(
                    &mut other_parse_state,
                    &mut self.parse_table.states[*state_id],
                );

                parse_state
                    .terminal_entries
                    .extend(other_parse_state.terminal_entries);
                parse_state
                    .nonterminal_entries
                    .extend(other_parse_state.nonterminal_entries);
            }

            // Update the new state's outgoing references using the new grouping.
            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
            new_states.push(parse_state);
        }

        self.parse_table.states = new_states;
    }

    fn states_conflict(
        &self,
        left_state: &ParseState,
        right_state: &ParseState,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        for (token, left_entry) in &left_state.terminal_entries {
            if let Some(right_entry) = right_state.terminal_entries.get(token) {
                if self.entries_conflict(
                    left_state.id,
                    right_state.id,
                    token,
                    left_entry,
                    right_entry,
                    group_ids_by_state_id,
                ) {
                    return true;
                }
            } else if self.token_conflicts(
                left_state.id,
                right_state.id,
                right_state.terminal_entries.keys(),
                *token,
            ) {
                return true;
            }
        }

        for token in right_state.terminal_entries.keys() {
            if !left_state.terminal_entries.contains_key(token) {
                if self.token_conflicts(
                    left_state.id,
                    right_state.id,
                    left_state.terminal_entries.keys(),
                    *token,
                ) {
                    return true;
                }
            }
        }

        false
    }

    fn state_successors_differ(
        &self,
        state1: &ParseState,
        state2: &ParseState,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        for (token, entry1) in &state1.terminal_entries {
            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
                if let Some(entry2) = state2.terminal_entries.get(token) {
                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
                        let group1 = group_ids_by_state_id[*s1];
                        let group2 = group_ids_by_state_id[*s2];
                        if group1 != group2 {
                            info!(
                                "split states {} {} - successors for {} are split: {} {}",
                                state1.id,
                                state2.id,
                                self.symbol_name(token),
                                s1,
                                s2,
                            );
                            return true;
                        }
                    }
                }
            }
        }

        for (symbol, s1) in &state1.nonterminal_entries {
            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
                let group1 = group_ids_by_state_id[*s1];
                let group2 = group_ids_by_state_id[*s2];
                if group1 != group2 {
                    info!(
                        "split states {} {} - successors for {} are split: {} {}",
                        state1.id,
                        state2.id,
                        self.symbol_name(symbol),
                        s1,
                        s2,
                    );
                    return true;
                }
            }
        }

        false
    }

    fn entries_conflict(
        &self,
        state_id1: ParseStateId,
        state_id2: ParseStateId,
        token: &Symbol,
        entry1: &ParseTableEntry,
        entry2: &ParseTableEntry,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        // To be compatible, entries need to have the same actions.
        let actions1 = &entry1.actions;
        let actions2 = &entry2.actions;
        if actions1.len() != actions2.len() {
            info!(
                "split states {} {} - differing action counts for token {}",
                state_id1,
                state_id2,
                self.symbol_name(token)
            );
            return true;
        }

        for (i, action1) in actions1.iter().enumerate() {
            let action2 = &actions2[i];

            // Two shift actions are equivalent if their destinations are in the same group.
            if let (
                ParseAction::Shift {
                    state: s1,
                    is_repetition: is_repetition1,
                },
                ParseAction::Shift {
                    state: s2,
                    is_repetition: is_repetition2,
                },
            ) = (action1, action2)
            {
                let group1 = group_ids_by_state_id[*s1];
                let group2 = group_ids_by_state_id[*s2];
                if group1 == group2 && is_repetition1 == is_repetition2 {
                    continue;
                } else {
                    info!(
                        "split states {} {} - successors for {} are split: {} {}",
                        state_id1,
                        state_id2,
                        self.symbol_name(token),
                        s1,
                        s2,
                    );
                    return true;
                }
            } else if action1 != action2 {
                info!(
                    "split states {} {} - unequal actions for {}",
                    state_id1,
                    state_id2,
                    self.symbol_name(token),
                );
                return true;
            }
        }

        false
    }

    fn token_conflicts<'b>(
        &self,
        left_id: ParseStateId,
        right_id: ParseStateId,
        existing_tokens: impl Iterator<Item = &'b Symbol>,
        new_token: Symbol,
    ) -> bool {
        // Do not add external tokens; they could conflict lexically with any of the state's
        // existing lookahead tokens.
        if new_token.is_external() {
            info!(
                "split states {} {} - external token {}",
                left_id,
                right_id,
                self.symbol_name(&new_token),
            );
            return true;
        }

        // Do not add tokens which are both internal and external. Their validity could
        // influence the behavior of the external scanner.
        if self
            .syntax_grammar
            .external_tokens
            .iter()
            .any(|external| external.corresponding_internal_token == Some(new_token))
        {
            info!(
                "split states {} {} - internal/external token {}",
                left_id,
                right_id,
                self.symbol_name(&new_token),
            );
            return true;
        }

        // Do not add a token if it conflicts with an existing token.
        for token in existing_tokens {
            if token.is_terminal() {
                if !(self.syntax_grammar.word_token == Some(*token)
                    && self.keywords.contains(&new_token))
                    && !(self.syntax_grammar.word_token == Some(new_token)
                        && self.keywords.contains(token))
                    && (self
                        .token_conflict_map
                        .does_conflict(new_token.index, token.index)
                        || self
                            .token_conflict_map
                            .does_match_same_string(new_token.index, token.index))
                {
                    info!(
                        "split states {} {} - token {} conflicts with {}",
                        left_id,
                        right_id,
                        self.symbol_name(&new_token),
                        self.symbol_name(token),
                    );
                    return true;
                }
            }
        }

        false
    }

    fn symbol_name(&self, symbol: &Symbol) -> &String {
        if symbol.is_non_terminal() {
            &self.syntax_grammar.variables[symbol.index].name
        } else if symbol.is_external() {
            &self.syntax_grammar.external_tokens[symbol.index].name
        } else {
            &self.lexical_grammar.variables[symbol.index].name
        }
    }

    fn remove_unused_states(&mut self) {
        let mut state_usage_map = vec![false; self.parse_table.states.len()];

        state_usage_map[0] = true;
        state_usage_map[1] = true;

        for state in &self.parse_table.states {
            for referenced_state in state.referenced_states() {
                state_usage_map[referenced_state] = true;
            }
        }
        let mut removed_predecessor_count = 0;
        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
        for state_id in 0..self.parse_table.states.len() {
            state_replacement_map[state_id] = state_id - removed_predecessor_count;
            if !state_usage_map[state_id] {
                removed_predecessor_count += 1;
            }
        }
        let mut state_id = 0;
        let mut original_state_id = 0;
        while state_id < self.parse_table.states.len() {
            if state_usage_map[original_state_id] {
                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
                    state_replacement_map[other_state_id]
                });
                state_id += 1;
            } else {
                self.parse_table.states.remove(state_id);
            }
            original_state_id += 1;
        }
    }
}
<commit_msg>Reorder parse states by descending symbol count<commit_after>use super::token_conflicts::TokenConflictMap;
use crate::generate::dedup::split_state_id_groups;
use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar, VariableType};
use crate::generate::rules::{AliasMap, Symbol, TokenSet};
use crate::generate::tables::{ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry};
use log::info;
use std::collections::{HashMap, HashSet};
use std::mem;

pub(crate) fn minimize_parse_table(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let mut minimizer = Minimizer {
        parse_table,
        syntax_grammar,
        lexical_grammar,
        token_conflict_map,
        keywords,
        simple_aliases,
    };
    minimizer.merge_compatible_states();
    minimizer.remove_unit_reductions();
    minimizer.remove_unused_states();
    minimizer.reorder_states_by_descending_size();
}

struct Minimizer<'a> {
    parse_table: &'a mut ParseTable,
    syntax_grammar: &'a SyntaxGrammar,
    lexical_grammar: &'a LexicalGrammar,
    token_conflict_map: &'a TokenConflictMap<'a>,
    keywords: &'a TokenSet,
    simple_aliases: &'a AliasMap,
}

impl<'a> Minimizer<'a> {
    fn remove_unit_reductions(&mut self) {
        let mut aliased_symbols = HashSet::new();
        for variable in &self.syntax_grammar.variables {
            for production in &variable.productions {
                for step in &production.steps {
                    if step.alias.is_some() {
                        aliased_symbols.insert(step.symbol);
                    }
                }
            }
        }

        let mut unit_reduction_symbols_by_state = HashMap::new();
        for (i, state) in self.parse_table.states.iter().enumerate() {
            let mut only_unit_reductions = true;
            let mut unit_reduction_symbol = None;
            for (_, entry) in &state.terminal_entries {
                for action in &entry.actions {
                    match action {
                        ParseAction::ShiftExtra => continue,
                        ParseAction::Reduce {
                            child_count: 1,
                            production_id: 0,
                            symbol,
                            ..
                        } => {
                            if !self.simple_aliases.contains_key(&symbol)
                                && !aliased_symbols.contains(&symbol)
                                && self.syntax_grammar.variables[symbol.index].kind
                                    != VariableType::Named
                                && (unit_reduction_symbol.is_none()
                                    || unit_reduction_symbol == Some(symbol))
                            {
                                unit_reduction_symbol = Some(symbol);
                                continue;
                            }
                        }
                        _ => {}
                    }
                    only_unit_reductions = false;
                    break;
                }

                if !only_unit_reductions {
                    break;
                }
            }

            if let Some(symbol) = unit_reduction_symbol {
                if only_unit_reductions {
                    unit_reduction_symbols_by_state.insert(i, *symbol);
                }
            }
        }

        for state in self.parse_table.states.iter_mut() {
            let mut done = false;
            while !done {
                done = true;
                state.update_referenced_states(|other_state_id, state| {
                    if let Some(symbol) = unit_reduction_symbols_by_state.get(&other_state_id) {
                        done = false;
                        state.nonterminal_entries[symbol]
                    } else {
                        other_state_id
                    }
                })
            }
        }
    }

    fn merge_compatible_states(&mut self) {
        let core_count = 1 + self
            .parse_table
            .states
            .iter()
            .map(|state| state.core_id)
            .max()
            .unwrap();

        // Initially group the states by their parse item set core.
        let mut group_ids_by_state_id = Vec::with_capacity(self.parse_table.states.len());
        let mut state_ids_by_group_id = vec![Vec::<ParseStateId>::new(); core_count];
        for (i, state) in self.parse_table.states.iter().enumerate() {
            state_ids_by_group_id[state.core_id].push(i);
            group_ids_by_state_id.push(state.core_id);
        }

        split_state_id_groups(
            &self.parse_table.states,
            &mut state_ids_by_group_id,
            &mut group_ids_by_state_id,
            0,
            |left, right, groups| self.states_conflict(left, right, groups),
        );

        while split_state_id_groups(
            &self.parse_table.states,
            &mut state_ids_by_group_id,
            &mut group_ids_by_state_id,
            0,
            |left, right, groups| self.state_successors_differ(left, right, groups),
        ) {
            continue;
        }

        let error_group_index = state_ids_by_group_id
            .iter()
            .position(|g| g.contains(&0))
            .unwrap();
        let start_group_index = state_ids_by_group_id
            .iter()
            .position(|g| g.contains(&1))
            .unwrap();
        state_ids_by_group_id.swap(error_group_index, 0);
        state_ids_by_group_id.swap(start_group_index, 1);

        // Create a list of new parse states: one state for each group of old states.
        let mut new_states = Vec::with_capacity(state_ids_by_group_id.len());
        for state_ids in &state_ids_by_group_id {
            // Initialize the new state based on the first old state in the group.
            let mut parse_state = ParseState::default();
            mem::swap(&mut parse_state, &mut self.parse_table.states[state_ids[0]]);

            // Extend the new state with all of the actions from the other old states
            // in the group.
            for state_id in &state_ids[1..] {
                let mut other_parse_state = ParseState::default();
                mem::swap(
                    &mut other_parse_state,
                    &mut self.parse_table.states[*state_id],
                );

                parse_state
                    .terminal_entries
                    .extend(other_parse_state.terminal_entries);
                parse_state
                    .nonterminal_entries
                    .extend(other_parse_state.nonterminal_entries);
            }

            // Update the new state's outgoing references using the new grouping.
            parse_state.update_referenced_states(|state_id, _| group_ids_by_state_id[state_id]);
            new_states.push(parse_state);
        }

        self.parse_table.states = new_states;
    }

    fn states_conflict(
        &self,
        left_state: &ParseState,
        right_state: &ParseState,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        for (token, left_entry) in &left_state.terminal_entries {
            if let Some(right_entry) = right_state.terminal_entries.get(token) {
                if self.entries_conflict(
                    left_state.id,
                    right_state.id,
                    token,
                    left_entry,
                    right_entry,
                    group_ids_by_state_id,
                ) {
                    return true;
                }
            } else if self.token_conflicts(
                left_state.id,
                right_state.id,
                right_state.terminal_entries.keys(),
                *token,
            ) {
                return true;
            }
        }

        for token in right_state.terminal_entries.keys() {
            if !left_state.terminal_entries.contains_key(token) {
                if self.token_conflicts(
                    left_state.id,
                    right_state.id,
                    left_state.terminal_entries.keys(),
                    *token,
                ) {
                    return true;
                }
            }
        }

        false
    }

    fn state_successors_differ(
        &self,
        state1: &ParseState,
        state2: &ParseState,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        for (token, entry1) in &state1.terminal_entries {
            if let ParseAction::Shift { state: s1, .. } = entry1.actions.last().unwrap() {
                if let Some(entry2) = state2.terminal_entries.get(token) {
                    if let ParseAction::Shift { state: s2, .. } = entry2.actions.last().unwrap() {
                        let group1 = group_ids_by_state_id[*s1];
                        let group2 = group_ids_by_state_id[*s2];
                        if group1 != group2 {
                            info!(
                                "split states {} {} - successors for {} are split: {} {}",
                                state1.id,
                                state2.id,
                                self.symbol_name(token),
                                s1,
                                s2,
                            );
                            return true;
                        }
                    }
                }
            }
        }

        for (symbol, s1) in &state1.nonterminal_entries {
            if let Some(s2) = state2.nonterminal_entries.get(symbol) {
                let group1 = group_ids_by_state_id[*s1];
                let group2 = group_ids_by_state_id[*s2];
                if group1 != group2 {
                    info!(
                        "split states {} {} - successors for {} are split: {} {}",
                        state1.id,
                        state2.id,
                        self.symbol_name(symbol),
                        s1,
                        s2,
                    );
                    return true;
                }
            }
        }

        false
    }

    fn entries_conflict(
        &self,
        state_id1: ParseStateId,
        state_id2: ParseStateId,
        token: &Symbol,
        entry1: &ParseTableEntry,
        entry2: &ParseTableEntry,
        group_ids_by_state_id: &Vec<ParseStateId>,
    ) -> bool {
        // To be compatible, entries need to have the same actions.
        let actions1 = &entry1.actions;
        let actions2 = &entry2.actions;
        if actions1.len() != actions2.len() {
            info!(
                "split states {} {} - differing action counts for token {}",
                state_id1,
                state_id2,
                self.symbol_name(token)
            );
            return true;
        }

        for (i, action1) in actions1.iter().enumerate() {
            let action2 = &actions2[i];

            // Two shift actions are equivalent if their destinations are in the same group.
            if let (
                ParseAction::Shift {
                    state: s1,
                    is_repetition: is_repetition1,
                },
                ParseAction::Shift {
                    state: s2,
                    is_repetition: is_repetition2,
                },
            ) = (action1, action2)
            {
                let group1 = group_ids_by_state_id[*s1];
                let group2 = group_ids_by_state_id[*s2];
                if group1 == group2 && is_repetition1 == is_repetition2 {
                    continue;
                } else {
                    info!(
                        "split states {} {} - successors for {} are split: {} {}",
                        state_id1,
                        state_id2,
                        self.symbol_name(token),
                        s1,
                        s2,
                    );
                    return true;
                }
            } else if action1 != action2 {
                info!(
                    "split states {} {} - unequal actions for {}",
                    state_id1,
                    state_id2,
                    self.symbol_name(token),
                );
                return true;
            }
        }

        false
    }

    fn token_conflicts<'b>(
        &self,
        left_id: ParseStateId,
        right_id: ParseStateId,
        existing_tokens: impl Iterator<Item = &'b Symbol>,
        new_token: Symbol,
    ) -> bool {
        // Do not add external tokens; they could conflict lexically with any of the state's
        // existing lookahead tokens.
        if new_token.is_external() {
            info!(
                "split states {} {} - external token {}",
                left_id,
                right_id,
                self.symbol_name(&new_token),
            );
            return true;
        }

        // Do not add tokens which are both internal and external. Their validity could
        // influence the behavior of the external scanner.
        if self
            .syntax_grammar
            .external_tokens
            .iter()
            .any(|external| external.corresponding_internal_token == Some(new_token))
        {
            info!(
                "split states {} {} - internal/external token {}",
                left_id,
                right_id,
                self.symbol_name(&new_token),
            );
            return true;
        }

        // Do not add a token if it conflicts with an existing token.
        for token in existing_tokens {
            if token.is_terminal() {
                if !(self.syntax_grammar.word_token == Some(*token)
                    && self.keywords.contains(&new_token))
                    && !(self.syntax_grammar.word_token == Some(new_token)
                        && self.keywords.contains(token))
                    && (self
                        .token_conflict_map
                        .does_conflict(new_token.index, token.index)
                        || self
                            .token_conflict_map
                            .does_match_same_string(new_token.index, token.index))
                {
                    info!(
                        "split states {} {} - token {} conflicts with {}",
                        left_id,
                        right_id,
                        self.symbol_name(&new_token),
                        self.symbol_name(token),
                    );
                    return true;
                }
            }
        }

        false
    }

    fn symbol_name(&self, symbol: &Symbol) -> &String {
        if symbol.is_non_terminal() {
            &self.syntax_grammar.variables[symbol.index].name
        } else if symbol.is_external() {
            &self.syntax_grammar.external_tokens[symbol.index].name
        } else {
            &self.lexical_grammar.variables[symbol.index].name
        }
    }

    fn remove_unused_states(&mut self) {
        let mut state_usage_map = vec![false; self.parse_table.states.len()];

        state_usage_map[0] = true;
        state_usage_map[1] = true;

        for state in &self.parse_table.states {
            for referenced_state in state.referenced_states() {
                state_usage_map[referenced_state] = true;
            }
        }
        let mut removed_predecessor_count = 0;
        let mut state_replacement_map = vec![0; self.parse_table.states.len()];
        for state_id in 0..self.parse_table.states.len() {
            state_replacement_map[state_id] = state_id - removed_predecessor_count;
            if !state_usage_map[state_id] {
                removed_predecessor_count += 1;
            }
        }
        let mut state_id = 0;
        let mut original_state_id = 0;
        while state_id < self.parse_table.states.len() {
            if state_usage_map[original_state_id] {
                self.parse_table.states[state_id].update_referenced_states(|other_state_id, _| {
                    state_replacement_map[other_state_id]
                });
                state_id += 1;
            } else {
                self.parse_table.states.remove(state_id);
            }
            original_state_id += 1;
        }
    }

    fn reorder_states_by_descending_size(&mut self) {
        // Get a mapping of old state index -> new_state_index
        let mut old_ids_by_new_id = (0..self.parse_table.states.len()).collect::<Vec<_>>();
        &old_ids_by_new_id.sort_unstable_by_key(|i| {
            // Don't changes states 0 (the error state) or 1 (the start state).
            if *i <= 1 {
                return *i as i64 - 1_000_000;
            }

            // Reorder all the other states by descending symbol count.
            let state = &self.parse_table.states[*i];
            -((state.terminal_entries.len() + state.nonterminal_entries.len()) as i64)
        });

        // Get the inverse mapping
        let mut new_ids_by_old_id = vec![0; old_ids_by_new_id.len()];
        for (id, old_id) in old_ids_by_new_id.iter().enumerate() {
            new_ids_by_old_id[*old_id] = id;
        }

        // Reorder the parse states and update their references to reflect
        // the new ordering.
        self.parse_table.states = old_ids_by_new_id
            .iter()
            .map(|old_id| {
                let mut state = ParseState::default();
                mem::swap(&mut state, &mut self.parse_table.states[*old_id]);
                state.update_referenced_states(|id, _| new_ids_by_old_id[id]);
                state
            })
            .collect();
    }
}
<|endoftext|>