<commit_before>use token::{Lexeme, Token};
use error::{Result, SassError, ErrorKind};

use std::str::CharIndices;
use std::iter::Peekable;

pub struct Tokenizer<'a> {
    chars: Peekable<CharIndices<'a>>,
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<Lexeme>;

    fn next(&mut self) -> Option<Result<Lexeme>> {
        match self.parse() {
            Ok(Some(t)) => Some(Ok(t)),
            Ok(None) => None,
            Err(e) => Some(Err(e)),
        }
    }
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &str) -> Tokenizer {
        Tokenizer {
            chars: text.char_indices().peekable(),
        }
    }

    // In the string parsing sense, not the parsing-out-semantics part yet.
    // Well, there's a tiny bit of semantics, depending on how you define "meaning".
    pub fn parse(&mut self) -> Result<Option<Lexeme>> {
        while let Some((char_offset, curr_char)) = self.chars.next() {
            // Skip leading whitespace
            if curr_char.is_whitespace() {
                continue;
            } else {
                let single_char_token = Token::from_char(curr_char);
                if single_char_token.is_some()
                   && !self.hyphen_starting_shit(curr_char)
                   && !self.comment_starting(curr_char) {
                    // We already tested that single_char_token was Some.
                    return Ok(Some(Lexeme {
                        token: single_char_token.unwrap(),
                        offset: Some(char_offset),
                    }))
                } else {
                    if self.comment_starting(curr_char) {
                        return self.comment(curr_char, char_offset)
                    } else if curr_char == '"' {
                        return self.string_literal(curr_char, char_offset)
                    } else if curr_char.is_numeric() || self.hyphen_starting_number(curr_char) {
                        return self.number(curr_char, char_offset)
                    } else {
                        return self.ident(curr_char, char_offset)
                    }
                }
            }
        }
        return Ok(None)
    }

    fn hyphen_starting_shit(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && !peek_char.unwrap().is_whitespace()
    }

    fn comment_starting(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '/' && peek_char.is_some() && peek_char.unwrap() == '*'
    }

    fn hyphen_starting_number(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && peek_char.unwrap().is_numeric()
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.chars.peek() {
            Some(&(_, peek_char)) => Some(peek_char),
            None => None,
        }
    }

    fn ident(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-ident char (hyphens are special)
            if peek_char.is_whitespace() || (
                is_single_char_token(peek_char) && peek_char != '-'
            ) {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }
        Ok(Some(Lexeme { token: Token::String(value), offset: Some(start) }))
    }

    fn number(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-numeric char
            // TODO: disallow two `.`s in one number
            if !peek_char.is_numeric() && peek_char != '.' {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let value = match value.parse() {
            Ok(v) => v,
            Err(_) => return Err(SassError {
                offset: start,
                kind: ErrorKind::TokenizerError,
                message: format!(
                    "Tried to parse `{}` into a f32 but failed.",
                    value,
                ),
            })
        };

        let mut unit = String::new();
        while let Some(peek_char) = self.peek_char() {
            // Get units; stop when we reach a space or non-percent operator
            if peek_char == '%' || (
                !peek_char.is_whitespace() && !is_single_char_token(peek_char)
            ) {
                unit.push(peek_char);
                self.chars.next();
            } else {
                break;
            }
        }

        let token = if unit.len() > 0 {
            Token::Number(value, Some(unit))
        } else {
            Token::Number(value, None)
        };

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }

    fn string_literal(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            if peek_char == '"' && !value.ends_with("\\") {
                value.push(peek_char);
                self.chars.next();
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let token = Token::StringLiteral(value);

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }

    fn comment(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        if self.peek_char() == Some('*') {
            value.push(self.peek_char().unwrap());
            self.chars.next();
        }
        // TODO: else line comment, else error

        while let Some(peek_char) = self.peek_char() {
            if peek_char == '/' && value.ends_with("*") {
                value.push(peek_char);
                self.chars.next();
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let token = Token::Comment(value);

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }
}

fn is_single_char_token(ch: char) -> bool {
    Token::from_char(ch).is_some()
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::{Lexeme, Token};
    use error::Result;

    fn expected_lexeme(expected_token: Token, expected_offset: usize) -> Option<Result<Lexeme>> {
        Some(Ok(Lexeme {
            token: expected_token,
            offset: Some(expected_offset),
        }))
    }

    fn expected_ident(expected_value: &str, expected_offset: usize) -> Option<Result<Lexeme>> {
        expected_lexeme(Token::String(expected_value.into()), expected_offset)
    }

    #[test]
    fn it_returns_none_for_empty_string() {
        let mut tokenizer = Tokenizer::new("");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_none_for_only_whitespace() {
        let mut tokenizer = Tokenizer::new("     ");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_words() {
        // Without regard to Sass word validity
        let mut tokenizer = Tokenizer::new(" \n  div   aoeu  ");
        assert_eq!(tokenizer.next(), expected_ident("div", 4));
        assert_eq!(tokenizer.next(), expected_ident("aoeu", 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_curly_braces() {
        // Without regard to matching
        let mut tokenizer = Tokenizer::new("{}}a{ blah}");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 2));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 4));
        assert_eq!(tokenizer.next(), expected_ident("blah", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_colon() {
        let mut tokenizer = Tokenizer::new(":invalid: property::");
        // This might be wrong. I have a note somewhere else that colons can start idents.
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 0));
        assert_eq!(tokenizer.next(), expected_ident("invalid", 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 8));
        assert_eq!(tokenizer.next(), expected_ident("property", 10));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 18));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 19));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_semicolon() {
        let mut tokenizer = Tokenizer::new(";;\na;\nb\n;");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 1));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 4));
        assert_eq!(tokenizer.next(), expected_ident("b", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 8));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_numbers() {
        let mut tokenizer = Tokenizer::new("border: 0px 1.5 11em;");
        assert_eq!(tokenizer.next(), expected_ident("border", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(
            Token::Number(0.0, Some("px".into())), 8
        ));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(1.5, None), 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(11.0, Some("em".into())), 16));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 20));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_hyphen() {
        let mut tokenizer = Tokenizer::new("font-weight -webkit -3 - 4-5 a-1 -");
        assert_eq!(tokenizer.next(), expected_ident("font-weight", 0));
        assert_eq!(tokenizer.next(), expected_ident("-webkit", 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-3.0, None), 20));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 23));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 25));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-5.0, None), 26));
        assert_eq!(tokenizer.next(), expected_ident("a-1", 29));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 33));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_parens() {
        let mut tokenizer = Tokenizer::new("() rgb)()(");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 1));
        assert_eq!(tokenizer.next(), expected_ident("rgb", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 7));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 9));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_slash() {
        let mut tokenizer = Tokenizer::new("/ / 3/4 / 8");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 2));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(3.0, None), 4));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 5));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(8.0, None), 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_comma() {
        let mut tokenizer = Tokenizer::new("a, b {");
        assert_eq!(tokenizer.next(), expected_ident("a", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comma, 1));
        assert_eq!(tokenizer.next(), expected_ident("b", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 5));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_double_quoted_string_literals() {
        let mut tokenizer = Tokenizer::new("what: \"hey \\\"ya\";");
        assert_eq!(tokenizer.next(), expected_ident("what", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 4));
        assert_eq!(
            tokenizer.next(),
            expected_lexeme(Token::StringLiteral("\"hey \\\"ya\"".into()), 6)
        );
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 16));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_multiline_comments() {
        let mut tokenizer = Tokenizer::new("a /* foo\nbar */ no");
        assert_eq!(tokenizer.next(), expected_ident("a", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comment("/* foo\nbar */".into()), 2));
        assert_eq!(tokenizer.next(), expected_ident("no", 16));
        assert_eq!(tokenizer.next(), None);
    }
}
<commit_msg>Tokenize single line comments as one big token<commit_after>use token::{Lexeme, Token};
use error::{Result, SassError, ErrorKind};

use std::str::CharIndices;
use std::iter::Peekable;

pub struct Tokenizer<'a> {
    chars: Peekable<CharIndices<'a>>,
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<Lexeme>;

    fn next(&mut self) -> Option<Result<Lexeme>> {
        match self.parse() {
            Ok(Some(t)) => Some(Ok(t)),
            Ok(None) => None,
            Err(e) => Some(Err(e)),
        }
    }
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &str) -> Tokenizer {
        Tokenizer {
            chars: text.char_indices().peekable(),
        }
    }

    // In the string parsing sense, not the parsing-out-semantics part yet.
    // Well, there's a tiny bit of semantics, depending on how you define "meaning".
    pub fn parse(&mut self) -> Result<Option<Lexeme>> {
        while let Some((char_offset, curr_char)) = self.chars.next() {
            // Skip leading whitespace
            if curr_char.is_whitespace() {
                continue;
            } else {
                let single_char_token = Token::from_char(curr_char);
                if single_char_token.is_some()
                   && !self.hyphen_starting_shit(curr_char)
                   && !self.comment_starting(curr_char) {
                    // We already tested that single_char_token was Some.
                    return Ok(Some(Lexeme {
                        token: single_char_token.unwrap(),
                        offset: Some(char_offset),
                    }))
                } else {
                    if self.comment_starting(curr_char) {
                        return self.comment(curr_char, char_offset)
                    } else if curr_char == '"' {
                        return self.string_literal(curr_char, char_offset)
                    } else if curr_char.is_numeric() || self.hyphen_starting_number(curr_char) {
                        return self.number(curr_char, char_offset)
                    } else {
                        return self.ident(curr_char, char_offset)
                    }
                }
            }
        }
        return Ok(None)
    }

    fn hyphen_starting_shit(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && !peek_char.unwrap().is_whitespace()
    }

    fn comment_starting(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '/'
          && peek_char.is_some()
          && (peek_char.unwrap() == '*' || peek_char.unwrap() == '/')
    }

    fn hyphen_starting_number(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && peek_char.unwrap().is_numeric()
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.chars.peek() {
            Some(&(_, peek_char)) => Some(peek_char),
            None => None,
        }
    }

    fn ident(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-ident char (hyphens are special)
            if peek_char.is_whitespace() || (
                is_single_char_token(peek_char) && peek_char != '-'
            ) {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }
        Ok(Some(Lexeme { token: Token::String(value), offset: Some(start) }))
    }

    fn number(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-numeric char
            // TODO: disallow two `.`s in one number
            if !peek_char.is_numeric() && peek_char != '.' {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let value = match value.parse() {
            Ok(v) => v,
            Err(_) => return Err(SassError {
                offset: start,
                kind: ErrorKind::TokenizerError,
                message: format!(
                    "Tried to parse `{}` into a f32 but failed.",
                    value,
                ),
            })
        };

        let mut unit = String::new();
        while let Some(peek_char) = self.peek_char() {
            // Get units; stop when we reach a space or non-percent operator
            if peek_char == '%' || (
                !peek_char.is_whitespace() && !is_single_char_token(peek_char)
            ) {
                unit.push(peek_char);
                self.chars.next();
            } else {
                break;
            }
        }

        let token = if unit.len() > 0 {
            Token::Number(value, Some(unit))
        } else {
            Token::Number(value, None)
        };

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }

    fn string_literal(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            if peek_char == '"' && !value.ends_with("\\") {
                value.push(peek_char);
                self.chars.next();
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let token = Token::StringLiteral(value);

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }

    fn comment(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        let multiline = match self.peek_char() {
            Some('*') => true,
            Some('/') => false,
            _ => unreachable!("You expected that you would test for a comment before calling this"),
        };

        value.push(self.peek_char().unwrap());
        self.chars.next();

        while let Some(peek_char) = self.peek_char() {
            if multiline && peek_char == '/' && value.ends_with("*") {
                value.push(peek_char);
                self.chars.next();
                break;
            } else if !multiline && peek_char == '\n' {
                self.chars.next();
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let token = Token::Comment(value);

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }
}

fn is_single_char_token(ch: char) -> bool {
    Token::from_char(ch).is_some()
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::{Lexeme, Token};
    use error::Result;

    fn expected_lexeme(expected_token: Token, expected_offset: usize) -> Option<Result<Lexeme>> {
        Some(Ok(Lexeme {
            token: expected_token,
            offset: Some(expected_offset),
        }))
    }

    fn expected_ident(expected_value: &str, expected_offset: usize) -> Option<Result<Lexeme>> {
        expected_lexeme(Token::String(expected_value.into()), expected_offset)
    }

    #[test]
    fn it_returns_none_for_empty_string() {
        let mut tokenizer = Tokenizer::new("");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_none_for_only_whitespace() {
        let mut tokenizer = Tokenizer::new("     ");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_words() {
        // Without regard to Sass word validity
        let mut tokenizer = Tokenizer::new(" \n  div   aoeu  ");
        assert_eq!(tokenizer.next(), expected_ident("div", 4));
        assert_eq!(tokenizer.next(), expected_ident("aoeu", 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_curly_braces() {
        // Without regard to matching
        let mut tokenizer = Tokenizer::new("{}}a{ blah}");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 2));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 4));
        assert_eq!(tokenizer.next(), expected_ident("blah", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_colon() {
        let mut tokenizer = Tokenizer::new(":invalid: property::");
        // This might be wrong. I have a note somewhere else that colons can start idents.
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 0));
        assert_eq!(tokenizer.next(), expected_ident("invalid", 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 8));
        assert_eq!(tokenizer.next(), expected_ident("property", 10));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 18));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 19));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_semicolon() {
        let mut tokenizer = Tokenizer::new(";;\na;\nb\n;");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 1));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 4));
        assert_eq!(tokenizer.next(), expected_ident("b", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 8));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_numbers() {
        let mut tokenizer = Tokenizer::new("border: 0px 1.5 11em;");
        assert_eq!(tokenizer.next(), expected_ident("border", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(
            Token::Number(0.0, Some("px".into())), 8
        ));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(1.5, None), 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(11.0, Some("em".into())), 16));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 20));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_hyphen() {
        let mut tokenizer = Tokenizer::new("font-weight -webkit -3 - 4-5 a-1 -");
        assert_eq!(tokenizer.next(), expected_ident("font-weight", 0));
        assert_eq!(tokenizer.next(), expected_ident("-webkit", 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-3.0, None), 20));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 23));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 25));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-5.0, None), 26));
        assert_eq!(tokenizer.next(), expected_ident("a-1", 29));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 33));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_parens() {
        let mut tokenizer = Tokenizer::new("() rgb)()(");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 1));
        assert_eq!(tokenizer.next(), expected_ident("rgb", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 7));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 9));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_slash() {
        let mut tokenizer = Tokenizer::new("/ / 3/4 / 8");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 2));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(3.0, None), 4));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 5));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(8.0, None), 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_comma() {
        let mut tokenizer = Tokenizer::new("a, b {");
        assert_eq!(tokenizer.next(), expected_ident("a", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comma, 1));
        assert_eq!(tokenizer.next(), expected_ident("b", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 5));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_double_quoted_string_literals() {
        let mut tokenizer = Tokenizer::new("what: \"hey \\\"ya\";");
        assert_eq!(tokenizer.next(), expected_ident("what", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 4));
        assert_eq!(
            tokenizer.next(),
            expected_lexeme(Token::StringLiteral("\"hey \\\"ya\"".into()), 6)
        );
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 16));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_multiline_comments() {
        let mut tokenizer = Tokenizer::new("a /* foo\nbar */ no");
        assert_eq!(tokenizer.next(), expected_ident("a", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comment("/* foo\nbar */".into()), 2));
        assert_eq!(tokenizer.next(), expected_ident("no", 16));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_single_line_comments() {
        let mut tokenizer = Tokenizer::new("b // because; hi \nc");
        assert_eq!(tokenizer.next(), expected_ident("b", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comment("// because; hi ".into()), 2));
        assert_eq!(tokenizer.next(), expected_ident("c", 18));
        assert_eq!(tokenizer.next(), None);
    }
}
<|endoftext|>


<commit_before>use super::super::RW;
use super::super::thread::{Handler, Message};
use super::super::evented::{EventSourceTrait, RcEventSource, Evented, EventedImpl};
use mio_orig::{EventLoop, Token, EventSet};
use std::sync::Arc;
use spin::Mutex;
use super::super::thread::MioSender;
use super::super::{sender_retry, in_coroutine};

use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::mpsc;
type ArcChannelShared = Arc<Mutex<ChannelShared>>;
type ArcCounter = Arc<AtomicUsize>;

/// State shared between senders and receiver.
struct ChannelShared {
    token: Option<Token>,
    sender: Option<MioSender>,
}

/// Channel receiving end
///
/// Create with `channel()`
pub struct Receiver<T>(RcEventSource<ReceiverCore<T>>);

struct ReceiverCore<T> {
    receiver: mpsc::Receiver<T>,
    shared: ArcChannelShared,
    counter: ArcCounter,
}

impl<T> EventedImpl for Receiver<T>
    where T: 'static
{
    type Raw = ReceiverCore<T>;

    fn shared(&self) -> &RcEventSource<ReceiverCore<T>> {
        &self.0
    }
}

impl<T> EventSourceTrait for ReceiverCore<T> {
    fn register(&mut self, event_loop: &mut EventLoop<Handler>, token: Token, interest: EventSet) -> bool {
        debug_assert!(interest.is_readable());
        trace!("Receiver({}): register", token.as_usize());
        let mut lock = self.shared.lock();

        lock.token = Some(token);
        lock.sender = Some(event_loop.channel());

        if self.counter.load(Ordering::SeqCst) != 0 {
            trace!("Receiver({}): not empty; self notify", token.as_usize());
            lock.token = None;
            true
        } else {
            false
        }
    }

    fn reregister(&mut self,
                  _event_loop: &mut EventLoop<Handler>,
                  token: Token,
                  interest: EventSet) -> bool {
        debug_assert!(interest.is_readable());
        trace!("Receiver({}): reregister", token.as_usize());
        let mut lock = self.shared.lock();

        debug_assert!(lock.token.is_some());
        debug_assert!(lock.sender.is_some());

        if self.counter.load(Ordering::SeqCst) != 0 {
            lock.token = None;
            true
        } else {
            false
        }
    }

    fn deregister(&mut self, _event_loop: &mut EventLoop<Handler>, token: Token) {
        trace!("Receiver({}): dereregister", token.as_usize());
        let mut lock = self.shared.lock();
        lock.token = None;
        lock.sender = None;
    }
}

impl<T> Receiver<T> {
    fn new(shared: ArcChannelShared, counter: ArcCounter, receiver: mpsc::Receiver<T>) -> Self {
        let core = ReceiverCore {
            shared: shared,
            counter: counter,
            receiver: receiver,
        };

        Receiver(RcEventSource::new(core))
    }
}

impl<T> Receiver<T>
    where T: 'static
{
    /// Receive `T` sent using corresponding `Sender::send()`.
    ///
    /// Will block coroutine if no elements are available.
    pub fn recv(&self) -> Result<T, mpsc::RecvError> {
        if in_coroutine() {
            loop {
                let recv = self.try_recv();

                match recv {
                    Err(mpsc::TryRecvError::Empty) => self.block_on(RW::read()),
                    Err(mpsc::TryRecvError::Disconnected) => return Err(mpsc::RecvError),
                    Ok(t) => return Ok(t),
                }
                debug_assert!(self.empty_shared_token());
            }
        } else {
            self.shared().io_ref().receiver.recv()
        }
    }

    /// Try reading data from the queue.
    ///
    /// This will not block.
    pub fn try_recv(&self) -> Result<T, mpsc::TryRecvError> {
        let shared = self.shared();
        let io_ref = shared.io_ref();
        let res = io_ref.receiver.try_recv();
        if res.is_ok() {
            let _prev_counter = self.shared().io_ref().counter.fetch_sub(1, Ordering::SeqCst);
            // since send() first sends, then increases the counter, it is
            // possible for _prev_counter to be zero
        }
        res
    }

    fn empty_shared_token(&self) -> bool {
        let shared = self.shared();
        let io_ref = shared.io_ref();
        let lock = io_ref.shared.lock();
        lock.token.is_none()
    }
}

struct SenderShared {
    shared: ArcChannelShared,
    counter: ArcCounter,
}

impl Drop for SenderShared {
    fn drop(&mut self) {
        let prev_counter = self.counter.fetch_add(1, Ordering::SeqCst);
        if prev_counter == 0 {
            maybe_notify_receiver(&self.shared);
        }
    }
}

/// Channel sending end
///
/// Use this inside mioco coroutines or outside of mioco itself to send data
/// asynchronously to the receiving end.
///
/// Create with `channel()`
pub struct Sender<T> {
    shared: Arc<SenderShared>,
    sender: mpsc::Sender<T>,
}

impl<T> Clone for Sender<T> {
    fn clone(&self) -> Self {
        Sender {
            shared: self.shared.clone(),
            sender: self.sender.clone(),
        }
    }
}

impl<T> Sender<T> {
    fn new(shared: ArcChannelShared, counter: ArcCounter, sender: mpsc::Sender<T>) -> Self {
        Sender {
            shared: Arc::new(SenderShared {
                shared: shared,
                counter: counter,
            }),
            sender: sender,
        }
    }
}

fn maybe_notify_receiver(shared: &ArcChannelShared) {
    let mut lock = shared.lock();
    let ChannelShared { ref mut sender, ref mut token } = *lock;

    if let Some(token) = *token {
        trace!("Sender: notifying {:?}", token);
        let sender = sender.as_ref().unwrap();
        sender_retry(&sender, Message::ChannelMsg(token))
    }
}

impl<T> Sender<T> {
    /// Deliver `T` to the other end of the channel.
    ///
    /// Channel behaves like a queue.
    ///
    /// This is non-blocking operation.
    pub fn send(&self, t: T) -> Result<(), mpsc::SendError<T>> {
        try!(self.sender.send(t));
        let prev_counter = self.shared.counter.fetch_add(1, Ordering::SeqCst);

        if prev_counter == 0 {
            maybe_notify_receiver(&self.shared.shared);
        }
        Ok(())
    }
}

/// Create a channel
///
/// Channel can be used to deliver data via MPSC queue.
///
/// Channel is modeled after `std::sync::mpsc::channel()`, only
/// supporting mioco-aware sending and receiving.
///
/// When receiving end is outside of coroutine, channel will behave just
/// like `std::sync::mpsc::channel()`.
pub fn channel<T>() -> (Sender<T>, Receiver<T>) {
    let shared = ChannelShared {
        token: None,
        sender: None,
    };

    let shared = Arc::new(Mutex::new(shared));

    let counter = Arc::new(AtomicUsize::new(0));
    let (sender, receiver) = mpsc::channel();
    (Sender::new(shared.clone(), counter.clone(), sender), Receiver::new(shared, counter, receiver))
}

unsafe impl<T : Send> Send for Receiver<T> {}
unsafe impl<T : Send> Send for Sender<T> {}
<commit_msg>adding SyncSender<commit_after>use super::super::RW;
use super::super::thread::{Handler, Message};
use super::super::evented::{EventSourceTrait, RcEventSource, Evented, EventedImpl};
use mio_orig::{EventLoop, Token, EventSet};
use std::sync::Arc;
use spin::Mutex;
use super::super::thread::MioSender;
use super::super::{sender_retry, in_coroutine};

use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::mpsc;
type ArcChannelShared = Arc<Mutex<ChannelShared>>;
type ArcCounter = Arc<AtomicUsize>;

/// State shared between senders and receiver.
struct ChannelShared {
    token: Option<Token>,
    sender: Option<MioSender>,
}

/// Channel receiving end
///
/// Create with `channel()`
pub struct Receiver<T>(RcEventSource<ReceiverCore<T>>);

struct ReceiverCore<T> {
    receiver: mpsc::Receiver<T>,
    shared: ArcChannelShared,
    counter: ArcCounter,
}

impl<T> EventedImpl for Receiver<T>
    where T: 'static
{
    type Raw = ReceiverCore<T>;

    fn shared(&self) -> &RcEventSource<ReceiverCore<T>> {
        &self.0
    }
}

impl<T> EventSourceTrait for ReceiverCore<T> {
    fn register(&mut self, event_loop: &mut EventLoop<Handler>, token: Token, interest: EventSet) -> bool {
        debug_assert!(interest.is_readable());
        trace!("Receiver({}): register", token.as_usize());
        let mut lock = self.shared.lock();

        lock.token = Some(token);
        lock.sender = Some(event_loop.channel());

        if self.counter.load(Ordering::SeqCst) != 0 {
            trace!("Receiver({}): not empty; self notify", token.as_usize());
            lock.token = None;
            true
        } else {
            false
        }
    }

    fn reregister(&mut self,
                  _event_loop: &mut EventLoop<Handler>,
                  token: Token,
                  interest: EventSet) -> bool {
        debug_assert!(interest.is_readable());
        trace!("Receiver({}): reregister", token.as_usize());
        let mut lock = self.shared.lock();

        debug_assert!(lock.token.is_some());
        debug_assert!(lock.sender.is_some());

        if self.counter.load(Ordering::SeqCst) != 0 {
            lock.token = None;
            true
        } else {
            false
        }
    }

    fn deregister(&mut self, _event_loop: &mut EventLoop<Handler>, token: Token) {
        trace!("Receiver({}): dereregister", token.as_usize());
        let mut lock = self.shared.lock();
        lock.token = None;
        lock.sender = None;
    }
}

impl<T> Receiver<T> {
    fn new(shared: ArcChannelShared, counter: ArcCounter, receiver: mpsc::Receiver<T>) -> Self {
        let core = ReceiverCore {
            shared: shared,
            counter: counter,
            receiver: receiver,
        };

        Receiver(RcEventSource::new(core))
    }
}

impl<T> Receiver<T>
    where T: 'static
{
    /// Receive `T` sent using corresponding `Sender::send()`.
    ///
    /// Will block coroutine if no elements are available.
    pub fn recv(&self) -> Result<T, mpsc::RecvError> {
        if in_coroutine() {
            loop {
                let recv = self.try_recv();

                match recv {
                    Err(mpsc::TryRecvError::Empty) => self.block_on(RW::read()),
                    Err(mpsc::TryRecvError::Disconnected) => return Err(mpsc::RecvError),
                    Ok(t) => return Ok(t),
                }
                debug_assert!(self.empty_shared_token());
            }
        } else {
            self.shared().io_ref().receiver.recv()
        }
    }

    /// Try reading data from the queue.
    ///
    /// This will not block.
    pub fn try_recv(&self) -> Result<T, mpsc::TryRecvError> {
        let shared = self.shared();
        let io_ref = shared.io_ref();
        let res = io_ref.receiver.try_recv();
        if res.is_ok() {
            let _prev_counter = self.shared().io_ref().counter.fetch_sub(1, Ordering::SeqCst);
            // since send() first sends, then increases the counter, it is
            // possible for _prev_counter to be zero
        }
        res
    }

    fn empty_shared_token(&self) -> bool {
        let shared = self.shared();
        let io_ref = shared.io_ref();
        let lock = io_ref.shared.lock();
        lock.token.is_none()
    }
}

struct SenderShared {
    shared: ArcChannelShared,
    counter: ArcCounter,
}

impl Drop for SenderShared {
    fn drop(&mut self) {
        let prev_counter = self.counter.fetch_add(1, Ordering::SeqCst);
        if prev_counter == 0 {
            maybe_notify_receiver(&self.shared);
        }
    }
}

/// Channel sending end
///
/// Use this inside mioco coroutines or outside of mioco itself to send data
/// asynchronously to the receiving end.
///
/// Create with `channel()`
pub struct Sender<T> {
    shared: Arc<SenderShared>,
    sender: mpsc::Sender<T>,
}

pub struct SyncSender<T> {
    shared: Arc<SenderShared>,
    sender: mpsc::SyncSender<T>,
}

impl<T> Clone for Sender<T> {
    fn clone(&self) -> Self {
        Sender {
            shared: self.shared.clone(),
            sender: self.sender.clone(),
        }
    }
}

impl<T> Sender<T> {
    fn new(shared: ArcChannelShared, counter: ArcCounter, sender: mpsc::Sender<T>) -> Self {
        Sender {
            shared: Arc::new(SenderShared {
                shared: shared,
                counter: counter,
            }),
            sender: sender,
        }
    }
}

impl<T> SyncSender<T> {
    fn new(shared: ArcChannelShared, counter: ArcCounter, sender: mpsc::SyncSender<T>) -> Self {
        SyncSender {
            shared: Arc::new(SenderShared {
                shared: shared,
                counter: counter,
            }),
            sender: sender,
        }
    }

    /// Deliver `T` to the other end of the channel.
    ///
    /// Channel behaves like a queue.
    ///
    /// This is non-blocking operation.
    pub fn send(&self, t: T) -> Result<(), mpsc::SendError<T>> {
        try!(self.sender.send(t));
        let prev_counter = self.shared.counter.fetch_add(1, Ordering::SeqCst);

        if prev_counter == 0 {
            maybe_notify_receiver(&self.shared.shared);
        }
        Ok(())
    }
}

fn maybe_notify_receiver(shared: &ArcChannelShared) {
    let mut lock = shared.lock();
    let ChannelShared { ref mut sender, ref mut token } = *lock;

    if let Some(token) = *token {
        trace!("Sender: notifying {:?}", token);
        let sender = sender.as_ref().unwrap();
        sender_retry(&sender, Message::ChannelMsg(token))
    }
}

impl<T> Sender<T> {
    /// Deliver `T` to the other end of the channel.
    ///
    /// Channel behaves like a queue.
    ///
    /// This is non-blocking operation.
    pub fn send(&self, t: T) -> Result<(), mpsc::SendError<T>> {
        try!(self.sender.send(t));
        let prev_counter = self.shared.counter.fetch_add(1, Ordering::SeqCst);

        if prev_counter == 0 {
            maybe_notify_receiver(&self.shared.shared);
        }
        Ok(())
    }
}

/// Create a channel
///
/// Channel can be used to deliver data via MPSC queue.
///
/// Channel is modeled after `std::sync::mpsc::channel()`, only
/// supporting mioco-aware sending and receiving.
///
/// When receiving end is outside of coroutine, channel will behave just
/// like `std::sync::mpsc::channel()`.
pub fn channel<T>() -> (Sender<T>, Receiver<T>) {
    let shared = ChannelShared {
        token: None,
        sender: None,
    };

    let shared = Arc::new(Mutex::new(shared));

    let counter = Arc::new(AtomicUsize::new(0));
    let (sender, receiver) = mpsc::channel();
    (Sender::new(shared.clone(), counter.clone(), sender), Receiver::new(shared, counter, receiver))
}

/// Create a channel
///
/// Channel can be used to deliver data via MPSC queue.
///
/// Channel is modeled after `std::sync::mpsc::channel()`, only
/// supporting mioco-aware sending and receiving.
///
/// When receiving end is outside of coroutine, channel will behave just
/// like `std::sync::mpsc::channel()`.
pub fn sync_channel<T>(bound: usize) -> (SyncSender<T>, Receiver<T>) {
    let shared = ChannelShared {
        token: None,
        sender: None,
    };

    let shared = Arc::new(Mutex::new(shared));

    let counter = Arc::new(AtomicUsize::new(0));
    let (sender, receiver) = mpsc::sync_channel(bound);
    (SyncSender::new(shared.clone(), counter.clone(), sender), Receiver::new(shared, counter, receiver))
}

unsafe impl<T : Send> Send for Receiver<T> {}
unsafe impl<T : Send> Send for Sender<T> {}
unsafe impl<T : Send> Send for SyncSender<T> {}
<|endoftext|>


<commit_before>use {Errno, Result};
use libc::c_int;
use std::os::unix::io::RawFd;

mod ffi {
    use libc::{c_int};
    use super::EpollEvent;

    extern {
        pub fn epoll_create(size: c_int) -> c_int;
        pub fn epoll_ctl(epfd: c_int, op: c_int, fd: c_int, event: *const EpollEvent) -> c_int;
        pub fn epoll_wait(epfd: c_int, events: *mut EpollEvent, max_events: c_int, timeout: c_int) -> c_int;
    }
}

bitflags!(
    #[repr(C)]
    flags EpollEventKind: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

#[cfg(not(target_arch = "x86_64"))]
#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(target_arch = "x86_64")]
#[derive(Clone, Copy)]
#[repr(C, packed)]
pub struct EpollEvent {
    pub events: EpollEventKind,
    pub data: u64
}

#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 12);
}

#[cfg(target_arch = "arm")]
#[test]
fn test_epoll_event_size() {
    use std::mem::size_of;
    assert_eq!(size_of::<EpollEvent>(), 16);
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { ffi::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl(epfd: RawFd, op: EpollOp, fd: RawFd, event: &EpollEvent) -> Result<()> {
    let res = unsafe { ffi::epoll_ctl(epfd, op as c_int, fd, event as *const EpollEvent) };

    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        ffi::epoll_wait(epfd, events.as_mut_ptr(), events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<commit_msg>Auto merge of #410 - fiveop:epoll_create1, r=posborne<commit_after>use {Errno, Result};
use libc::{self, c_int};
use std::os::unix::io::RawFd;

bitflags!(
    #[repr(C)]
    flags EpollFlags: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

libc_bitflags!{
    flags EpollCreateFlags: c_int {
        EPOLL_CLOEXEC,
    }
}

#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    event: libc::epoll_event,
}

impl EpollEvent {
    pub fn new(events: EpollFlags, data: u64) -> EpollEvent {
        EpollEvent { event: libc::epoll_event { events: events.bits(), u64: data } }
    }

    pub fn events(&self) -> EpollFlags {
        EpollFlags::from_bits(self.event.events).unwrap()
    }

    pub fn data(&self) -> u64 {
        self.event.u64
    }
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { libc::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_create1(flags: EpollCreateFlags) -> Result<RawFd> {
    let res = unsafe { libc::epoll_create1(flags.bits()) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl(epfd: RawFd, op: EpollOp, fd: RawFd, event: &mut EpollEvent) -> Result<()> {
    let res = unsafe { libc::epoll_ctl(epfd, op as c_int, fd, &mut event.event) };

    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        libc::epoll_wait(epfd, events.as_mut_ptr() as *mut libc::epoll_event, events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<|endoftext|>


<commit_before>//! Toolbox to make text layout easier.

use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;

mod lines_iterator;
mod reader;

pub use self::lines_iterator::{LinesIterator, Row};
pub use self::reader::ProgressReader;

/// The length and width of a part of a string.
pub struct Prefix {
    /// The length (in bytes) of the string.
    pub length: usize,
    /// The unicode-width of the string.
    pub width: usize
}

/// Computes the length (number of bytes) and width of a prefix that fits in the given `width`.
///
/// Takes non-breakable elements from `iter`, while keeping the
/// string width under `width` (and adding the length of `delimiter`
/// between each element).
///
/// Given `total_text = iter.collect().join(delimiter)`, the result
/// is the length of the longest prefix of `width` or less cells,
/// without breaking inside an element.
///
/// Example:
///
/// ```
/// # extern crate cursive;
/// extern crate unicode_segmentation;
/// use unicode_segmentation::UnicodeSegmentation;
///
/// # use cursive::utils::prefix;
/// # fn main() {
/// let my_text = "blah...";
/// // This returns the number of bytes for a prefix of `my_text` that
/// // fits within 5 cells.
/// prefix(my_text.graphemes(true), 5, "");
/// # }
/// ```
pub fn prefix<'a, I>(iter: I, available_width: usize, delimiter: &str) -> Prefix
    where I: Iterator<Item = &'a str>
{
    let delimiter_width = delimiter.width();
    let delimiter_len = delimiter.len();

    let mut current_width = 0;
    let sum = iter.take_while(|token| {
            let width = token.width();
            if current_width + width > available_width {
                false
            } else {
                if current_width != 0 {
                    current_width += delimiter_width;
                }
                current_width += width;
                true
            }
        })
        .map(|token| token.len() + delimiter_len)
        .fold(0, |a, b| a + b);

    // We counted delimiter once too many times,
    // but only if the iterator was non empty.
    let length = if sum == 0 { sum } else { sum - delimiter_len };

    Prefix {
        length: length,
        width: current_width
    }
}

/// Computes the length (number of bytes) and width of a suffix that fits in the given `width`.
///
/// Doesn't break inside elements returned by `iter`.
///
/// Returns the number of bytes of the longest
/// suffix from `text` that fits in `width`.
///
/// This is a shortcut for `prefix_length(iter.rev(), width, delimiter)`
pub fn suffix<'a, I>(iter: I, width: usize, delimiter: &str) -> Prefix
    where I: DoubleEndedIterator<Item = &'a str>
{
    prefix(iter.rev(), width, delimiter)
}

/// Computes the length (number of bytes) and width of a suffix that fits in the given `width`.
///
/// Breaks between any two graphemes.
pub fn simple_suffix(text: &str, width: usize) -> Prefix {
    suffix(text.graphemes(true), width, "")
}
<commit_msg>fix #113<commit_after>//! Toolbox to make text layout easier.

use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;

mod lines_iterator;
mod reader;

pub use self::lines_iterator::{LinesIterator, Row};
pub use self::reader::ProgressReader;

/// The length and width of a part of a string.
pub struct Prefix {
    /// The length (in bytes) of the string.
    pub length: usize,
    /// The unicode-width of the string.
    pub width: usize
}

/// Computes the length (number of bytes) and width of a prefix that fits in the given `width`.
///
/// Takes non-breakable elements from `iter`, while keeping the
/// string width under `width` (and adding the length of `delimiter`
/// between each element).
///
/// Given `total_text = iter.collect().join(delimiter)`, the result
/// is the length of the longest prefix of `width` or less cells,
/// without breaking inside an element.
///
/// Example:
///
/// ```
/// # extern crate cursive;
/// extern crate unicode_segmentation;
/// use unicode_segmentation::UnicodeSegmentation;
///
/// # use cursive::utils::prefix;
/// # fn main() {
/// let my_text = "blah...";
/// // This returns the number of bytes for a prefix of `my_text` that
/// // fits within 5 cells.
/// prefix(my_text.graphemes(true), 5, "");
/// # }
/// ```
pub fn prefix<'a, I>(iter: I, available_width: usize, delimiter: &str) -> Prefix
    where I: Iterator<Item = &'a str>
{
    let delimiter_width = delimiter.width();
    let delimiter_len = delimiter.len();

    let mut current_width = 0;
    let sum = iter.take_while(|token| {
            let needed_width = if current_width == 0 {
                token.width()
            } else {
                // We also need to insert the delimiter, if not at the beginning.
                token.width() + delimiter_width
            };
            if current_width + needed_width > available_width {
                false
            } else {
                current_width += needed_width;
                true
            }
        })
        .map(|token| token.len() + delimiter_len)
        .fold(0, |a, b| a + b);

    // We counted delimiter once too many times,
    // but only if the iterator was non empty.
    let length = sum.saturating_sub(delimiter_len);

    debug_assert!(current_width <= available_width);

    Prefix {
        length: length,
        width: current_width
    }
}

/// Computes the length (number of bytes) and width of a suffix that fits in the given `width`.
///
/// Doesn't break inside elements returned by `iter`.
///
/// Returns the number of bytes of the longest
/// suffix from `text` that fits in `width`.
///
/// This is a shortcut for `prefix_length(iter.rev(), width, delimiter)`
pub fn suffix<'a, I>(iter: I, width: usize, delimiter: &str) -> Prefix
    where I: DoubleEndedIterator<Item = &'a str>
{
    prefix(iter.rev(), width, delimiter)
}

/// Computes the length (number of bytes) and width of a suffix that fits in the given `width`.
///
/// Breaks between any two graphemes.
pub fn simple_suffix(text: &str, width: usize) -> Prefix {
    suffix(text.graphemes(true), width, "")
}
<|endoftext|>


<commit_before>//! Types encapsulating _handles_ to managed JavaScript memory.

use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::error::Error;
use std::fmt::{self, Debug, Display};
use neon_runtime;
use neon_runtime::raw;
use value::{JsResult, Value};
use value::internal::SuperType;
use context::Context;
use result::ResultExt;

/// The trait of data that is managed by the JS garbage collector and can only be accessed via handles.
pub trait Managed: Copy {
    fn to_raw(self) -> raw::Local;

    fn from_raw(h: raw::Local) -> Self;
}

/// A safely rooted _handle_ to a JS value in memory that is managed by the garbage collector.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct Handle<'a, T: Managed + 'a> {
    value: T,
    phantom: PhantomData<&'a T>
}

impl<'a, T: Managed + 'a> PartialEq for Handle<'a, T> {
    fn eq(&self, other: &Self) -> bool {
        unsafe { neon_runtime::mem::same_handle(self.to_raw(), other.to_raw()) }
    }
}

impl<'a, T: Managed + 'a> Eq for Handle<'a, T> { }

impl<'a, T: Managed + 'a> Handle<'a, T> {
    pub(crate) fn new_internal(value: T) -> Handle<'a, T> {
        Handle {
            value: value,
            phantom: PhantomData
        }
    }
}

/// An error representing a failed downcast.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)]
pub struct DowncastError<F: Value, T: Value> {
    phantom_from: PhantomData<F>,
    phantom_to: PhantomData<T>,
    description: String
}

impl<F: Value, T: Value> Debug for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "DowncastError")
    }
}

impl<F: Value, T: Value> DowncastError<F, T> {
    fn new() -> Self {
        DowncastError {
            phantom_from: PhantomData,
            phantom_to: PhantomData,
            description: format!("failed downcast to {}", T::name())
        }
    }
}

impl<F: Value, T: Value> Display for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.description())
    }
}

impl<F: Value, T: Value> Error for DowncastError<F, T> {
    fn description(&self) -> &str {
        &self.description
    }
}

/// The result of a call to `Handle::downcast()`.
pub type DowncastResult<'a, F, T> = Result<Handle<'a, T>, DowncastError<F, T>>;

impl<'a, F: Value, T: Value> ResultExt<'a, T> for DowncastResult<'a, F, T> {
    fn unwrap_or_throw<'b, C: Context<'b>>(self, cx: &mut C) -> JsResult<'a, T> {
        match self {
            Ok(v) => Ok(v),
            Err(e) => cx.throw_type_error(&e.description)
        }
    }
}

impl<'a, T: Value> Handle<'a, T> {

    /// Safely upcast a handle to a supertype.
    ///
    /// This method does not require an execution context because it only copies a handle.
    pub fn upcast<U: Value + SuperType<T>>(&self) -> Handle<'a, U> {
        Handle::new_internal(SuperType::upcast_internal(self.value))
    }

    /// Tests whether this value is an instance of the given type.
    ///
    /// # Example:
    ///
    /// ```no_run
    /// # use neon::prelude::*;
    /// # fn my_neon_function(mut cx: FunctionContext) -> JsResult<JsUndefined> {
    /// let v: Handle<JsValue> = cx.number(17).upcast();
    /// v.is_a::<JsString>(); // false
    /// v.is_a::<JsNumber>(); // true
    /// v.is_a::<JsValue>();  // true
    /// # Ok(cx.undefined())
    /// # }
    /// ```
    pub fn is_a<U: Value>(&self) -> bool {
        U::is_typeof(self.value)
    }

    /// Attempts to downcast a handle to another type, which may fail.
    pub fn downcast<U: Value>(&self) -> DowncastResult<'a, T, U> {
        match U::downcast(self.value) {
            Some(v) => Ok(Handle::new_internal(v)),
            None => Err(DowncastError::new())
        }
    }

}

impl<'a, T: Managed> Deref for Handle<'a, T> {
    type Target = T;
    fn deref<'b>(&'b self) -> &'b T {
        &self.value
    }
}

impl<'a, T: Managed> DerefMut for Handle<'a, T> {
    fn deref_mut<'b>(&'b mut self) -> &'b mut T {
        &mut self.value
    }
}
<commit_msg>Add `downcast_or_throw` shorthand method.<commit_after>//! Types encapsulating _handles_ to managed JavaScript memory.

use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::error::Error;
use std::fmt::{self, Debug, Display};
use neon_runtime;
use neon_runtime::raw;
use value::{JsResult, Value};
use value::internal::SuperType;
use context::Context;
use result::ResultExt;

/// The trait of data that is managed by the JS garbage collector and can only be accessed via handles.
pub trait Managed: Copy {
    fn to_raw(self) -> raw::Local;

    fn from_raw(h: raw::Local) -> Self;
}

/// A safely rooted _handle_ to a JS value in memory that is managed by the garbage collector.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct Handle<'a, T: Managed + 'a> {
    value: T,
    phantom: PhantomData<&'a T>
}

impl<'a, T: Managed + 'a> PartialEq for Handle<'a, T> {
    fn eq(&self, other: &Self) -> bool {
        unsafe { neon_runtime::mem::same_handle(self.to_raw(), other.to_raw()) }
    }
}

impl<'a, T: Managed + 'a> Eq for Handle<'a, T> { }

impl<'a, T: Managed + 'a> Handle<'a, T> {
    pub(crate) fn new_internal(value: T) -> Handle<'a, T> {
        Handle {
            value: value,
            phantom: PhantomData
        }
    }
}

/// An error representing a failed downcast.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)]
pub struct DowncastError<F: Value, T: Value> {
    phantom_from: PhantomData<F>,
    phantom_to: PhantomData<T>,
    description: String
}

impl<F: Value, T: Value> Debug for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "DowncastError")
    }
}

impl<F: Value, T: Value> DowncastError<F, T> {
    fn new() -> Self {
        DowncastError {
            phantom_from: PhantomData,
            phantom_to: PhantomData,
            description: format!("failed downcast to {}", T::name())
        }
    }
}

impl<F: Value, T: Value> Display for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.description())
    }
}

impl<F: Value, T: Value> Error for DowncastError<F, T> {
    fn description(&self) -> &str {
        &self.description
    }
}

/// The result of a call to `Handle::downcast()`.
pub type DowncastResult<'a, F, T> = Result<Handle<'a, T>, DowncastError<F, T>>;

impl<'a, F: Value, T: Value> ResultExt<'a, T> for DowncastResult<'a, F, T> {
    fn unwrap_or_throw<'b, C: Context<'b>>(self, cx: &mut C) -> JsResult<'a, T> {
        match self {
            Ok(v) => Ok(v),
            Err(e) => cx.throw_type_error(&e.description)
        }
    }
}

impl<'a, T: Value> Handle<'a, T> {

    /// Safely upcast a handle to a supertype.
    ///
    /// This method does not require an execution context because it only copies a handle.
    pub fn upcast<U: Value + SuperType<T>>(&self) -> Handle<'a, U> {
        Handle::new_internal(SuperType::upcast_internal(self.value))
    }

    /// Tests whether this value is an instance of the given type.
    ///
    /// # Example:
    ///
    /// ```no_run
    /// # use neon::prelude::*;
    /// # fn my_neon_function(mut cx: FunctionContext) -> JsResult<JsUndefined> {
    /// let v: Handle<JsValue> = cx.number(17).upcast();
    /// v.is_a::<JsString>(); // false
    /// v.is_a::<JsNumber>(); // true
    /// v.is_a::<JsValue>();  // true
    /// # Ok(cx.undefined())
    /// # }
    /// ```
    pub fn is_a<U: Value>(&self) -> bool {
        U::is_typeof(self.value)
    }

    /// Attempts to downcast a handle to another type, which may fail. A failure
    /// to downcast **does not** throw a JavaScript exception, so it's OK to
    /// continue interacting with the JS engine if this method produces an `Err`
    /// result.
    pub fn downcast<U: Value>(&self) -> DowncastResult<'a, T, U> {
        match U::downcast(self.value) {
            Some(v) => Ok(Handle::new_internal(v)),
            None => Err(DowncastError::new())
        }
    }

    /// Attempts to downcast a handle to another type, raising a JavaScript `TypeError`
    /// exception on failure. This method is a convenient shorthand, equivalent to
    /// `self.downcast::<U>().or_throw::<C>(cx)`.
    pub fn downcast_or_throw<'b, U: Value, C: Context<'b>>(&self, cx: &mut C) -> JsResult<'a, U> {
        self.downcast().unwrap_or_throw(cx)
    }

}

impl<'a, T: Managed> Deref for Handle<'a, T> {
    type Target = T;
    fn deref<'b>(&'b self) -> &'b T {
        &self.value
    }
}

impl<'a, T: Managed> DerefMut for Handle<'a, T> {
    fn deref_mut<'b>(&'b mut self) -> &'b mut T {
        &mut self.value
    }
}
<|endoftext|>


<commit_before>#![feature(macro_rules)]

extern crate "cargo-registry" as cargo_registry;
extern crate "conduit-middleware" as conduit_middleware;
extern crate "conduit-test" as conduit_test;
extern crate conduit;
extern crate curl;
extern crate git2;
extern crate serialize;
extern crate url;

use std::sync::{Once, ONCE_INIT, Arc};
use std::os;
use serialize::json;

use conduit::Request;
use conduit_test::MockRequest;
use cargo_registry::app::App;
use cargo_registry::db;
use cargo_registry::user::User;

macro_rules! t( ($e:expr) => (
    match $e {
        Ok(e) => e,
        Err(m) => fail!(concat!(stringify!($e), " failed with: {}"), m),
    }
) )

macro_rules! t_resp( ($e:expr) => ({
    t!($e.map_err(|e| (&*e).to_string()))
}) )

macro_rules! ok_resp( ($e:expr) => ({
    let resp = t_resp!($e);
    if !::ok_resp(&resp) { fail!("bad response: {}", resp.status); }
    resp
}) )

mod middleware;
mod package;
mod user;
mod record;
mod git;
mod version;

fn app() -> (record::Bomb, Arc<App>, conduit_middleware::MiddlewareBuilder) {
    static mut INIT: Once = ONCE_INIT;

    let (proxy, bomb) = record::proxy();
    let config = cargo_registry::Config {
        s3_bucket: os::getenv("S3_BUCKET").unwrap_or(String::new()),
        s3_access_key: os::getenv("S3_ACCESS_KEY").unwrap_or(String::new()),
        s3_secret_key: os::getenv("S3_SECRET_KEY").unwrap_or(String::new()),
        s3_proxy: Some(proxy),
        session_key: "test".to_string(),
        git_repo_checkout: git::checkout(),
        gh_client_id: "".to_string(),
        gh_client_secret: "".to_string(),
        db_url: env("TEST_DATABASE_URL"),
        env: cargo_registry::Test,
        max_upload_size: 100,
    };
    let app = App::new(&config);
    git::init();
    unsafe {
        INIT.doit(|| app.db_setup());
    }
    let app = Arc::new(app);
    return (bomb, app.clone(), cargo_registry::middleware(app));

    fn env(s: &str) -> String {
        match os::getenv(s) {
            Some(s) => s,
            None => fail!("must have `{}` defined", s),
        }
    }
}

fn req(app: Arc<App>, method: conduit::Method, path: &str) -> MockRequest {
    let mut req = MockRequest::new(method, path);
    req.mut_extensions().insert(db::Transaction::new(app));
    return req;
}

fn ok_resp(r: &conduit::Response) -> bool {
    r.status.val0() == 200
}

fn json<T>(r: &mut conduit::Response) -> T
           where T: serialize::Decodable<json::Decoder, json::DecoderError> {
    let data = r.body.read_to_end().unwrap();
    let s = std::str::from_utf8(data.as_slice()).unwrap();
    match json::decode(s) {
        Ok(t) => t,
        Err(e) => fail!("failed to decode: {}\n{}", e, s),
    }
}

fn user() -> User {
    User {
        id: 10000,
        email: "foo@example.com".to_string(),
        gh_access_token: User::new_api_token(), // just randomize it
        api_token: User::new_api_token(),
    }
}

fn package() -> cargo_registry::package::Package {
    cargo_registry::package::Package {
        id: 10000,
        name: "foo".to_string(),
        user_id: 100,
    }
}
<commit_msg>Move the git init sooner<commit_after>#![feature(macro_rules)]

extern crate "cargo-registry" as cargo_registry;
extern crate "conduit-middleware" as conduit_middleware;
extern crate "conduit-test" as conduit_test;
extern crate conduit;
extern crate curl;
extern crate git2;
extern crate serialize;
extern crate url;

use std::sync::{Once, ONCE_INIT, Arc};
use std::os;
use serialize::json;

use conduit::Request;
use conduit_test::MockRequest;
use cargo_registry::app::App;
use cargo_registry::db;
use cargo_registry::user::User;

macro_rules! t( ($e:expr) => (
    match $e {
        Ok(e) => e,
        Err(m) => fail!(concat!(stringify!($e), " failed with: {}"), m),
    }
) )

macro_rules! t_resp( ($e:expr) => ({
    t!($e.map_err(|e| (&*e).to_string()))
}) )

macro_rules! ok_resp( ($e:expr) => ({
    let resp = t_resp!($e);
    if !::ok_resp(&resp) { fail!("bad response: {}", resp.status); }
    resp
}) )

mod middleware;
mod package;
mod user;
mod record;
mod git;
mod version;

fn app() -> (record::Bomb, Arc<App>, conduit_middleware::MiddlewareBuilder) {
    static mut INIT: Once = ONCE_INIT;
    git::init();

    let (proxy, bomb) = record::proxy();
    let config = cargo_registry::Config {
        s3_bucket: os::getenv("S3_BUCKET").unwrap_or(String::new()),
        s3_access_key: os::getenv("S3_ACCESS_KEY").unwrap_or(String::new()),
        s3_secret_key: os::getenv("S3_SECRET_KEY").unwrap_or(String::new()),
        s3_proxy: Some(proxy),
        session_key: "test".to_string(),
        git_repo_checkout: git::checkout(),
        gh_client_id: "".to_string(),
        gh_client_secret: "".to_string(),
        db_url: env("TEST_DATABASE_URL"),
        env: cargo_registry::Test,
        max_upload_size: 100,
    };
    let app = App::new(&config);
    unsafe {
        INIT.doit(|| app.db_setup());
    }
    let app = Arc::new(app);
    return (bomb, app.clone(), cargo_registry::middleware(app));

    fn env(s: &str) -> String {
        match os::getenv(s) {
            Some(s) => s,
            None => fail!("must have `{}` defined", s),
        }
    }
}

fn req(app: Arc<App>, method: conduit::Method, path: &str) -> MockRequest {
    let mut req = MockRequest::new(method, path);
    req.mut_extensions().insert(db::Transaction::new(app));
    return req;
}

fn ok_resp(r: &conduit::Response) -> bool {
    r.status.val0() == 200
}

fn json<T>(r: &mut conduit::Response) -> T
           where T: serialize::Decodable<json::Decoder, json::DecoderError> {
    let data = r.body.read_to_end().unwrap();
    let s = std::str::from_utf8(data.as_slice()).unwrap();
    match json::decode(s) {
        Ok(t) => t,
        Err(e) => fail!("failed to decode: {}\n{}", e, s),
    }
}

fn user() -> User {
    User {
        id: 10000,
        email: "foo@example.com".to_string(),
        gh_access_token: User::new_api_token(), // just randomize it
        api_token: User::new_api_token(),
    }
}

fn package() -> cargo_registry::package::Package {
    cargo_registry::package::Package {
        id: 10000,
        name: "foo".to_string(),
        user_id: 100,
    }
}
<|endoftext|>


<commit_before>use std::path::Path;
use std::vec::Vec;
use error::IndentexError;

const LINESEP: &'static str = "\n";
const LATEX_TO_INDENTEX_FACTOR: f64 = 1.5;

// Indentation processing
#[inline]
fn count_left_indent<T: AsRef<str>>(line: T) -> Option<usize> {
    if line.as_ref().is_empty() {
        None
    } else {
        Some(line.as_ref().chars().count() - line.as_ref().trim_left().chars().count())
    }
}

fn scan_indents<T: AsRef<str>>(lines: &[T]) -> Vec<usize> {
    let raw_indents = lines.iter().map(count_left_indent).collect::<Vec<_>>();

    let mut adjusted_indents: Vec<usize> = Vec::with_capacity(raw_indents.len() + 1);
    let mut last_indent: usize = 0;

    for current_indent in raw_indents.iter().rev() {
        adjusted_indents.push(match *current_indent {
            None => last_indent,
            Some(ind) => {
                last_indent = ind;
                ind
            }
        });
    }

    adjusted_indents.reverse();
    adjusted_indents.push(0);

    adjusted_indents
}

// Transpilation
fn transpile<T: AsRef<str>>(lines: &[T]) -> String {
    use parsers::Environment;
    use parsers::Hashline::{PlainLine, OpenEnv};
    use parsers::process_line;

    // The number of environments is not known beforehand
    let mut env_stack: Vec<Environment> = Vec::new();

    // Input size is the sum of all line lengths plus the number of lines (for lineseps)
    let input_size = lines.iter().fold(0, |sum, l| sum + l.as_ref().len()) + lines.len();
    // We do not know how much larger the transpiled LaTeX file will be, but we can guess...
    let indentex_size = (LATEX_TO_INDENTEX_FACTOR * (input_size as f64)).round() as usize;
    let mut transpiled = String::with_capacity(indentex_size);

    let adjusted_indents = scan_indents(lines);

    for (line_num, line) in lines.iter().enumerate() {
        let list_like_active = match env_stack.last() {
            None => false, // No environment is active at all
            Some(d) => d.is_list_like(),
        };

        let tl = match process_line(line.as_ref(), list_like_active) {
            PlainLine(l) => l,
            OpenEnv(e) => {
                let tag_begin = e.latex_begin();
                env_stack.push(e);
                tag_begin
            }
        };
        transpiled.push_str(&tl);
        transpiled.push_str(LINESEP);

        // Check if we are in an environment and close as many as needed
        while match env_stack.last() {
            None => false,
            Some(d) => d.indent_depth() >= adjusted_indents[line_num + 1],
        } {
            // `unwrap()` is safe here since we have already checked if the stack is empty
            let tag_end = env_stack.pop().unwrap().latex_end();
            transpiled.push_str(&tag_end);
            transpiled.push_str(LINESEP);
        }
    }

    transpiled
}

pub fn transpile_file<T: AsRef<Path>>(path: T) -> Result<(), IndentexError> {
    use file_utils::{read_and_trim_lines, rename_indentex_file, write_to_file};

    let lines = read_and_trim_lines(path.as_ref())?;
    let transpiled_text = transpile(&lines);
    let path_out = rename_indentex_file(path)?;
    write_to_file(path_out, &transpiled_text)?;

    Ok(())
}

#[cfg(test)]
mod tests {
    #[test]
    fn count_left_indent() {
        use super::count_left_indent;

        assert_eq!(count_left_indent(""), None);
        assert_eq!(count_left_indent("foo"), Some(0));
        assert_eq!(count_left_indent("  bar"), Some(2));
    }
}
<commit_msg>Improved testing of indentation processing<commit_after>use std::path::Path;
use std::vec::Vec;
use error::IndentexError;

const LINESEP: &'static str = "\n";
const LATEX_TO_INDENTEX_FACTOR: f64 = 1.5;

// Indentation processing
#[inline]
fn count_left_indent<T: AsRef<str>>(line: T) -> Option<usize> {
    if line.as_ref().is_empty() {
        None
    } else {
        Some(line.as_ref().chars().count() - line.as_ref().trim_left().chars().count())
    }
}

fn scan_indents<T: AsRef<str>>(lines: &[T]) -> Vec<usize> {
    let raw_indents = lines.iter().map(count_left_indent).collect::<Vec<_>>();

    let mut adjusted_indents: Vec<usize> = Vec::with_capacity(raw_indents.len() + 1);
    let mut last_indent: usize = 0;

    for current_indent in raw_indents.iter().rev() {
        adjusted_indents.push(match *current_indent {
            None => last_indent,
            Some(ind) => {
                last_indent = ind;
                ind
            }
        });
    }

    adjusted_indents.reverse();
    adjusted_indents.push(0);

    adjusted_indents
}

// Transpilation
fn transpile<T: AsRef<str>>(lines: &[T]) -> String {
    use parsers::Environment;
    use parsers::Hashline::{PlainLine, OpenEnv};
    use parsers::process_line;

    // The number of environments is not known beforehand
    let mut env_stack: Vec<Environment> = Vec::new();

    // Input size is the sum of all line lengths plus the number of lines (for lineseps)
    let input_size = lines.iter().fold(0, |sum, l| sum + l.as_ref().len()) + lines.len();
    // We do not know how much larger the transpiled LaTeX file will be, but we can guess...
    let indentex_size = (LATEX_TO_INDENTEX_FACTOR * (input_size as f64)).round() as usize;
    let mut transpiled = String::with_capacity(indentex_size);

    let adjusted_indents = scan_indents(lines);

    for (line_num, line) in lines.iter().enumerate() {
        let list_like_active = match env_stack.last() {
            None => false, // No environment is active at all
            Some(d) => d.is_list_like(),
        };

        let tl = match process_line(line.as_ref(), list_like_active) {
            PlainLine(l) => l,
            OpenEnv(e) => {
                let tag_begin = e.latex_begin();
                env_stack.push(e);
                tag_begin
            }
        };
        transpiled.push_str(&tl);
        transpiled.push_str(LINESEP);

        // Check if we are in an environment and close as many as needed
        while match env_stack.last() {
            None => false,
            Some(d) => d.indent_depth() >= adjusted_indents[line_num + 1],
        } {
            // `unwrap()` is safe here since we have already checked if the stack is empty
            let tag_end = env_stack.pop().unwrap().latex_end();
            transpiled.push_str(&tag_end);
            transpiled.push_str(LINESEP);
        }
    }

    transpiled
}

pub fn transpile_file<T: AsRef<Path>>(path: T) -> Result<(), IndentexError> {
    use file_utils::{read_and_trim_lines, rename_indentex_file, write_to_file};

    let lines = read_and_trim_lines(path.as_ref())?;
    let transpiled_text = transpile(&lines);
    let path_out = rename_indentex_file(path)?;
    write_to_file(path_out, &transpiled_text)?;

    Ok(())
}

#[cfg(test)]
mod tests {
    #[test]
    fn count_left_indent() {
        use super::count_left_indent;

        assert_eq!(count_left_indent(""), None);
        assert_eq!(count_left_indent("foo"), Some(0));
        assert_eq!(count_left_indent("  bar"), Some(2));
        // We assume that the input has no trailing whitespaces
        // This is not a bug (but not a nice behaviour either)
        assert_eq!(count_left_indent("   "), Some(3));
    }

    #[test]
    fn scan_indents() {
        use super::scan_indents;

        // Always add a zero at the end
        let a = [" a"];
        assert_eq!(scan_indents(&a), [1, 0]);
        assert_eq!(scan_indents(&a).capacity(), 2);
        // Indents are propagated backwards
        let b = ["  b", "b", "", "  b"];
        assert_eq!(scan_indents(&b), [2, 0, 2, 2, 0]);
        assert_eq!(scan_indents(&b).capacity(), 5);
        // We assume that the input has no trailing whitespaces
        // This is not a bug (but not a nice behaviour either)
        let c = ["", "   "];
        assert_eq!(scan_indents(&c), [3, 3, 0]);
        assert_eq!(scan_indents(&c).capacity(), 3);

        let d = ["d", " d", "", " d", "", "   d", "  d", "     d"];
        assert_eq!(scan_indents(&d), [0, 1, 1, 1, 3, 3, 2, 5, 0]);
        assert_eq!(scan_indents(&d).capacity(), 9);
    }
}
<|endoftext|>


<commit_before>extern crate gcc;
extern crate walkdir;

use walkdir::{WalkDir};

use std::env;

fn main() {
  println!("cargo:rerun-if-changed=build.rs");
  for entry in WalkDir::new("kernels") {
    let entry = entry.unwrap();
    println!("cargo:rerun-if-changed={}", entry.path().display());
  }
  let out_dir = env::var("OUT_DIR").unwrap();
  let cc = env::var("CC").unwrap_or("gcc".to_owned());
  gcc::Config::new()
    .compiler(&cc)
    .opt_level(3)
    .pic(true)
    .flag("-std=gnu99")
    .flag("-march=native")
    .flag("-fno-strict-aliasing")
    .flag("-Ikernels")
    .file("kernels/vector.c")
    .compile("libdensearray_kernels.a");
  if cfg!(not(feature = "knl")) {
    gcc::Config::new()
      .compiler(&cc)
      .opt_level(3)
      .pic(true)
      .flag("-std=gnu99")
      .flag("-march=native")
      .flag("-fno-strict-aliasing")
      .flag("-fopenmp")
      .flag("-Ikernels")
      .flag("-DDENSEARRAY_OMP")
      .file("kernels/vector.c")
      .compile("libdensearray_omp_kernels.a");
  } else {
    gcc::Config::new()
      .compiler("icc")
      .opt_level(2)
      .pic(true)
      .flag("-std=c99")
      //.flag("-march=native")
      .flag("-fno-strict-aliasing")
      .flag("-openmp")
      .flag("-no-offload")
      .flag("-xMIC-AVX512")
      .flag("-Ikernels")
      .flag("-DDENSEARRAY_OMP")
      .file("kernels/vector.c")
      .compile("libdensearray_omp_kernels.a");
  }
  println!("cargo:rustc-link-search=native={}", out_dir);
}
<commit_msg>Rerun the build script by checking absolute paths.<commit_after>extern crate gcc;
extern crate walkdir;

use walkdir::{WalkDir};

use std::env;
use std::path::{PathBuf};

fn main() {
  println!("cargo:rerun-if-changed=build.rs");
  let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
  let out_dir = env::var("OUT_DIR").unwrap();
  let mut kernels_src_dir = PathBuf::from(manifest_dir);
  kernels_src_dir.push("kernels");
  for entry in WalkDir::new(&kernels_src_dir.to_str().unwrap()) {
    let entry = entry.unwrap();
    println!("cargo:rerun-if-changed={}", entry.path().display());
  }
  let cc = env::var("CC").unwrap_or("gcc".to_owned());
  gcc::Config::new()
    .compiler(&cc)
    .opt_level(3)
    .pic(true)
    .flag("-std=gnu99")
    .flag("-march=native")
    .flag("-fno-strict-aliasing")
    .flag("-Ikernels")
    .file("kernels/vector.c")
    .compile("libdensearray_kernels.a");
  if cfg!(not(feature = "knl")) {
    gcc::Config::new()
      .compiler(&cc)
      .opt_level(3)
      .pic(true)
      .flag("-std=gnu99")
      .flag("-march=native")
      .flag("-fno-strict-aliasing")
      .flag("-fopenmp")
      .flag("-Ikernels")
      .flag("-DDENSEARRAY_OMP")
      .file("kernels/vector.c")
      .compile("libdensearray_omp_kernels.a");
  } else {
    /*if cfg!(feature = "mkl_parallel") {
      // XXX: For debugging to check that both features are enabled.
      println!("cargo:rerun-if-changed=mkl_parallel_dummy");
    }*/
    gcc::Config::new()
      .compiler("icc")
      .opt_level(2)
      .pic(true)
      .flag("-std=c99")
      //.flag("-march=native")
      .flag("-fno-strict-aliasing")
      .flag("-openmp")
      .flag("-no-offload")
      .flag("-xMIC-AVX512")
      .flag("-Ikernels")
      .flag("-DDENSEARRAY_OMP")
      .file("kernels/vector.c")
      .compile("libdensearray_omp_kernels.a");
  }
  println!("cargo:rustc-link-search=native={}", out_dir);
}
<|endoftext|>


<commit_before>use std::env;
use std::fs;

fn main() {
    let home_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let out_dir: String;
    match env::var("CARGO_FEATURE_RELEASE") {
        Ok(_) => out_dir = format!("{}/target/release", home_dir),
        Err(_) => out_dir = format!("{}/target/debug", home_dir)
    };

    let in_icon_path = format!("{}/icon.png", home_dir);
    let out_icon_path = format!("{}/icon.png", out_dir);
    fs::copy(&in_icon_path, &out_icon_path).unwrap();
}<commit_msg>now includes rest of needed files, fixed target ignore<commit_after>use std::env;
use std::fs;

fn main() {
    let home_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let out_dir: String;
    let target = env::var("OUT_DIR").unwrap();
    match target {
        _ if target.contains("release") => out_dir = format!("{}/target/release", home_dir),
        _ if target.contains("debug") => out_dir = format!("{}/target/debug", home_dir),
        _ => panic!("Could not find target directory.")
    };

    let in_secret_path = format!("{}/secret", home_dir);
    let out_secret_path = format!("{}/secret", out_dir);

    let in_icon_path = format!("{}/icon.png", home_dir);
    let out_icon_path = format!("{}/icon.png", out_dir);

    fs::copy(&in_secret_path, &out_secret_path).unwrap();
    fs::copy(&in_icon_path, &out_icon_path).unwrap();
}<|endoftext|>


<commit_before>use std::fs;
use std::io::ErrorKind;
use std::path::PathBuf;
use std::process::Command;

fn main() {
    let out_dir = PathBuf::from(env!("OUT_DIR"));

    // Copy static assets to target directory.
    let copy_status = Command::new("cp")
        .arg("-r")
        .arg("static")
        .arg(out_dir.to_str().unwrap())
        .status()
        .unwrap();
    assert!(copy_status.success());

    // Compile SCSS
    let scss_out = &out_dir.join("styles.css");
    let sass_status = Command::new("sass")
        .arg("scss/main.scss")
        .arg(scss_out.to_str().unwrap())
        .status()
        .unwrap_or_else(|e| {
            match e.kind() {
                ErrorKind::NotFound => panic!("sass not installed"),
                _ => panic!(e),
            }
        });
    assert!(sass_status.success(), "there was a problem compiling scss");

    // Apply PostCSS
    let output = &out_dir.join("static/css/");
    fs::create_dir_all(&output).unwrap();
    let postcss_status = Command::new("postcss")
        .args(&["--use", "autoprefixer"])
        .arg(scss_out.to_str().unwrap())
        .args(&["-d", output.to_str().unwrap()])
        .status()
        .unwrap_or_else(|e| {
            match e.kind() {
                ErrorKind::NotFound => panic!("postcss not installed"),
                _ => panic!(e),
            }
        });
    assert!(postcss_status.success(),
            "there was a problem postprocessing css");
}
<commit_msg>fix(build): use OUT_DIR value at build time<commit_after>use std::env;
use std::fs;
use std::io::ErrorKind;
use std::path::PathBuf;
use std::process::Command;

fn main() {
    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());

    // Copy static assets to target directory.
    let copy_status = Command::new("cp")
        .arg("-r")
        .arg("static")
        .arg(out_dir.to_str().unwrap())
        .status()
        .unwrap();
    assert!(copy_status.success());

    // Compile SCSS
    let scss_out = &out_dir.join("styles.css");
    let sass_status = Command::new("sass")
        .arg("scss/main.scss")
        .arg(scss_out.to_str().unwrap())
        .status()
        .unwrap_or_else(|e| {
            match e.kind() {
                ErrorKind::NotFound => panic!("sass not installed"),
                _ => panic!(e),
            }
        });
    assert!(sass_status.success(), "there was a problem compiling scss");

    // Apply PostCSS
    let output = &out_dir.join("static/css/");
    fs::create_dir_all(&output).unwrap();
    let postcss_status = Command::new("postcss")
        .args(&["--use", "autoprefixer"])
        .arg(scss_out.to_str().unwrap())
        .args(&["-d", output.to_str().unwrap()])
        .status()
        .unwrap_or_else(|e| {
            match e.kind() {
                ErrorKind::NotFound => panic!("postcss not installed"),
                _ => panic!(e),
            }
        });
    assert!(postcss_status.success(),
            "there was a problem postprocessing css");
}
<|endoftext|>