<commit_before>use std::collections::LinkedList;
use std::borrow::Borrow;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
    fn is_es_identifier(self) -> bool;
    fn is_es_identifier_start(self) -> bool;
    fn is_es_identifier_continue(self) -> bool;
}

/*
// https://github.com/ariya/esprima/blob/master/tools/generate-identifier-regex.js
// https://gist.github.com/mathiasbynens/6334847
// http://unicode.org/reports/tr31/

pub trait UnicodeCharExt {
    fn is_L(self) -> bool;
    fn is_Lu(self) -> bool;
    fn is_Ll(self) -> bool;
    fn is_Lt(self) -> bool;
    fn is_Lm(self) -> bool;
    fn is_Lo(self) -> bool;
    fn is_Nl(self) -> bool;
    fn is_OtherID_Start(self) -> bool;
}

impl UnicodeCharExt for char {
    fn is_L(self) -> bool {
        self.is_Lu() ||
        self.is_Ll() ||
        self.is_Lt() ||
        self.is_Lm() ||
        self.is_Lo()
    }

    fn is_Lu(self) -> bool {
        false
    }

    fn is_Ll(self) -> bool {
        false
    }

    fn is_Lt(self) -> bool {
        false
    }

    fn is_Lm(self) -> bool {
        false
    }

    fn is_Lo(self) -> bool {
        false
    }

    fn is_Nl(self) -> bool {
        false
    }

    fn is_OtherID_Start(self) -> bool {
        false
    }
}
*/

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }

    fn is_es_identifier(self) -> bool {
        self.is_es_identifier_continue()
    }

    fn is_es_identifier_start(self) -> bool {
        self == '$' ||
        self == '_' ||
        self.is_alphabetic()
    }

    fn is_es_identifier_continue(self) -> bool {
        self.is_es_identifier_start() ||
        self.is_numeric()
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }
    pub fn next_char(&mut self) -> Option<char> { self.next_char }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

// test case: x=0;y=g=1;alert(eval("while(x)break\n/y/g.exec('y')"))
//       see: https://groups.google.com/d/msg/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E/Mxc7ZKc5r6sJ

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn is_empty(&mut self) -> bool {
        self.tokens.len() == 0
    }

    fn push_token(&mut self, token: Token) {
        assert!(self.tokens.len() == 0);
        self.tokens.push_back(token);
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }

    fn peek_token(&mut self) -> &Token {
        assert!(self.tokens.len() > 0);
        self.tokens.front().unwrap()
    }

    fn unread_token(&mut self, token: Token) {
        assert!(self.tokens.len() >= 0);
        assert!(self.tokens.len() < 3);
        self.tokens.push_front(token);
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn peek_token(&mut self) -> &Token {
        if self.lookahead.is_empty() {
            let token = self.read_next_token();
            self.lookahead.push_token(token);
        }
        self.lookahead.peek_token()
    }

    pub fn read_token(&mut self) -> Token {
        if self.lookahead.is_empty() {
            self.read_next_token()
        } else {
            self.lookahead.read_token()
        }
    }

    pub fn unread_token(&mut self, token: Token) {
        self.lookahead.unread_token(token);
    }

    // private methods

    fn skip_line_comment(&mut self) {
        self.bump();
        self.bump();
        while self.reader.curr_char().is_es_newline() {
            self.bump();
        }
    }

    fn skip_block_comment(&mut self) {
    }

    fn div_or_regexp(&mut self) -> Token {
        if self.cx.get().is_operator() {
            self.reader.bump();
            Token::Slash
        } else {
            unimplemented!()
        }
    }

    fn if_assign(&mut self, cons: Token, alt: Token) -> Token {
        self.reader.bump();
        if self.reader.curr_char() == Some('=') { self.reader.bump(); cons } else { alt }
    }

    fn if_equality(&mut self, zero: Token, one: Token, two: Token) -> Token {
        self.bump();
        if self.reader.curr_char() == Some('=') {
            self.bump();
            if self.reader.curr_char() == Some('=') {
                self.bump();
                two
            } else {
                one
            }
        } else {
            zero
        }
    }

    fn lt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('<') => {
                self.bump();
                if self.reader.curr_char() == Some('=') {
                    self.bump();
                    Token::LShiftAssign
                } else {
                    Token::LShift
                }
            },
            Some('=') => { self.bump(); Token::LEq }
            _ => Token::LAngle
        }
    }

    fn gt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('>') => {
                self.bump();
                match self.reader.curr_char() {
                    Some('>') => {
                        self.bump();
                        if self.reader.curr_char() == Some('=') {
                            self.bump();
                            Token::URShiftAssign
                        } else {
                            Token::URShift
                        }
                    },
                    Some('=') => { self.bump(); Token::RShiftAssign },
                    _ => Token::RShift
                }
            },
            Some('=') => { self.bump(); Token::GEq },
            _ => Token::RAngle
        }
    }

    fn plus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('+') => { self.bump(); Token::Inc },
            Some('=') => { self.bump(); Token::PlusAssign },
            _ => Token::Plus
        }
    }

    fn minus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('-') => { self.bump(); Token::Dec },
            Some('=') => { self.bump(); Token::MinusAssign },
            _ => Token::Minus
        }
    }

    fn number(&mut self) -> Token {
        if self.reader.curr_char() == Some('1') {
            self.bump();
            return Token::DecimalInt(String::from_str("1"));
        }
        unimplemented!()
    }

    fn string(&mut self) -> Token { unimplemented!() }
    fn word(&mut self) -> Token { unimplemented!() }

    fn read_next_token(&mut self) -> Token {
        self.skip_whitespace();
        println!("inspecting {:?}", self.reader.curr_char());
        loop {
            match self.reader.curr_char() {
                Some('/') => {
                    match self.reader.next_char() {
                        Some('/') => self.skip_line_comment(),
                        Some('*') => self.skip_block_comment(),
                        _ => return self.div_or_regexp()
                    }
                },
                Some('.') => {
                    match self.reader.next_char() {
                        Some(ch) if ch.is_digit(10) => { return self.number() },
                        _ => { self.reader.bump(); return Token::Dot }
                    }
                }
                Some('{') => { self.bump(); return Token::LBrace },
                Some('}') => { self.bump(); return Token::RBrace },
                Some('[') => { self.bump(); return Token::LBrack },
                Some(']') => { self.bump(); return Token::RBrack },
                Some('(') => { self.bump(); return Token::LParen },
                Some(')') => { self.bump(); return Token::RParen },
                Some(';') => { self.bump(); return Token::Semi },
                Some(':') => { self.bump(); return Token::Colon },
                Some(',') => { self.bump(); return Token::Comma },
                Some('<') => return self.lt(),
                Some('>') => return self.gt(),
                Some('=') => return self.if_equality(Token::Assign, Token::Eq, Token::StrictEq),
                Some('+') => return self.plus(),
                Some('-') => return self.minus(),
                Some('*') => return self.if_assign(Token::StarAssign, Token::Star),
                Some('%') => return self.if_assign(Token::ModAssign, Token::Mod),
                Some('^') => return self.if_assign(Token::BitXorAssign, Token::BitXor),
                Some('&') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('&') => { self.bump(); return Token::LogicalAnd },
                        _ => return Token::BitAnd
                    }
                },
                Some('|') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('|') => { self.bump(); return Token::LogicalOr },
                        _ => return Token::BitOr
                    }
                },
                Some('~') => { self.bump(); return Token::Tilde },
                Some('!') => return self.if_equality(Token::Bang, Token::NEq, Token::StrictNEq),
                Some('?') => { self.bump(); return Token::Question },
                Some('"') => return self.string(),
                Some('\'') => return self.string(),
                Some('\r') => {
                    self.bump();
                    if self.reader.curr_char() == Some('\n') {
                        self.bump();
                    }
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                },
                Some(ch) if ch.is_es_newline() => {
                    self.bump();
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                }
                Some(ch) if ch.is_digit(10) => return self.number(),
                Some(ch) if ch.is_es_identifier() => return self.word(),
                Some(ch) => return Token::Error(ch),
                None => return Token::EOF
            }
        }
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn bump(&mut self) {
        self.reader.bump();
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.read_token() {
            Token::EOF => None,
            t => Some(t)
        }
    }
}
<commit_msg>had to switch to `loop`, oops, or-pattern ambiguous with bitwise or src/lexer.rs:243:50: 243:54 error: mismatched types:  expected `bool`,     found `core::option::Option<_>` (expected bool,     found enum `core::option::Option`) [E0308] src/lexer.rs:243                 Some(ch) if ch.is_es_newline() | None => return,                                                                   ^~~~<commit_after>use std::collections::LinkedList;
use std::borrow::Borrow;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
    fn is_es_identifier(self) -> bool;
    fn is_es_identifier_start(self) -> bool;
    fn is_es_identifier_continue(self) -> bool;
}

/*
// https://github.com/ariya/esprima/blob/master/tools/generate-identifier-regex.js
// https://gist.github.com/mathiasbynens/6334847
// http://unicode.org/reports/tr31/

pub trait UnicodeCharExt {
    fn is_L(self) -> bool;
    fn is_Lu(self) -> bool;
    fn is_Ll(self) -> bool;
    fn is_Lt(self) -> bool;
    fn is_Lm(self) -> bool;
    fn is_Lo(self) -> bool;
    fn is_Nl(self) -> bool;
    fn is_OtherID_Start(self) -> bool;
}

impl UnicodeCharExt for char {
    fn is_L(self) -> bool {
        self.is_Lu() ||
        self.is_Ll() ||
        self.is_Lt() ||
        self.is_Lm() ||
        self.is_Lo()
    }

    fn is_Lu(self) -> bool {
        false
    }

    fn is_Ll(self) -> bool {
        false
    }

    fn is_Lt(self) -> bool {
        false
    }

    fn is_Lm(self) -> bool {
        false
    }

    fn is_Lo(self) -> bool {
        false
    }

    fn is_Nl(self) -> bool {
        false
    }

    fn is_OtherID_Start(self) -> bool {
        false
    }
}
*/

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }

    fn is_es_identifier(self) -> bool {
        self.is_es_identifier_continue()
    }

    fn is_es_identifier_start(self) -> bool {
        self == '$' ||
        self == '_' ||
        self.is_alphabetic()
    }

    fn is_es_identifier_continue(self) -> bool {
        self.is_es_identifier_start() ||
        self.is_numeric()
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }
    pub fn next_char(&mut self) -> Option<char> { self.next_char }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

// test case: x=0;y=g=1;alert(eval("while(x)break\n/y/g.exec('y')"))
//       see: https://groups.google.com/d/msg/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E/Mxc7ZKc5r6sJ

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn is_empty(&mut self) -> bool {
        self.tokens.len() == 0
    }

    fn push_token(&mut self, token: Token) {
        assert!(self.tokens.len() == 0);
        self.tokens.push_back(token);
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }

    fn peek_token(&mut self) -> &Token {
        assert!(self.tokens.len() > 0);
        self.tokens.front().unwrap()
    }

    fn unread_token(&mut self, token: Token) {
        assert!(self.tokens.len() >= 0);
        assert!(self.tokens.len() < 3);
        self.tokens.push_front(token);
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn peek_token(&mut self) -> &Token {
        if self.lookahead.is_empty() {
            let token = self.read_next_token();
            self.lookahead.push_token(token);
        }
        self.lookahead.peek_token()
    }

    pub fn read_token(&mut self) -> Token {
        if self.lookahead.is_empty() {
            self.read_next_token()
        } else {
            self.lookahead.read_token()
        }
    }

    pub fn unread_token(&mut self, token: Token) {
        self.lookahead.unread_token(token);
    }

    // private methods

    fn skip_line_comment(&mut self) {
        self.bump();
        self.bump();
        loop {
            match self.reader.curr_char() {
                Some(ch) if ch.is_es_newline() | None => return,
                _ => ()
            }
            self.bump();
        }
    }

    fn skip_block_comment(&mut self) {
    }

    fn div_or_regexp(&mut self) -> Token {
        if self.cx.get().is_operator() {
            self.reader.bump();
            Token::Slash
        } else {
            unimplemented!()
        }
    }

    fn if_assign(&mut self, cons: Token, alt: Token) -> Token {
        self.reader.bump();
        if self.reader.curr_char() == Some('=') { self.reader.bump(); cons } else { alt }
    }

    fn if_equality(&mut self, zero: Token, one: Token, two: Token) -> Token {
        self.bump();
        if self.reader.curr_char() == Some('=') {
            self.bump();
            if self.reader.curr_char() == Some('=') {
                self.bump();
                two
            } else {
                one
            }
        } else {
            zero
        }
    }

    fn lt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('<') => {
                self.bump();
                if self.reader.curr_char() == Some('=') {
                    self.bump();
                    Token::LShiftAssign
                } else {
                    Token::LShift
                }
            },
            Some('=') => { self.bump(); Token::LEq }
            _ => Token::LAngle
        }
    }

    fn gt(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('>') => {
                self.bump();
                match self.reader.curr_char() {
                    Some('>') => {
                        self.bump();
                        if self.reader.curr_char() == Some('=') {
                            self.bump();
                            Token::URShiftAssign
                        } else {
                            Token::URShift
                        }
                    },
                    Some('=') => { self.bump(); Token::RShiftAssign },
                    _ => Token::RShift
                }
            },
            Some('=') => { self.bump(); Token::GEq },
            _ => Token::RAngle
        }
    }

    fn plus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('+') => { self.bump(); Token::Inc },
            Some('=') => { self.bump(); Token::PlusAssign },
            _ => Token::Plus
        }
    }

    fn minus(&mut self) -> Token {
        self.bump();
        match self.reader.curr_char() {
            Some('-') => { self.bump(); Token::Dec },
            Some('=') => { self.bump(); Token::MinusAssign },
            _ => Token::Minus
        }
    }

    fn number(&mut self) -> Token {
        if self.reader.curr_char() == Some('1') {
            self.bump();
            return Token::DecimalInt(String::from_str("1"));
        }
        unimplemented!()
    }

    fn string(&mut self) -> Token { unimplemented!() }
    fn word(&mut self) -> Token { unimplemented!() }

    fn read_next_token(&mut self) -> Token {
        self.skip_whitespace();
        println!("inspecting {:?}", self.reader.curr_char());
        loop {
            match self.reader.curr_char() {
                Some('/') => {
                    match self.reader.next_char() {
                        Some('/') => self.skip_line_comment(),
                        Some('*') => self.skip_block_comment(),
                        _ => return self.div_or_regexp()
                    }
                },
                Some('.') => {
                    match self.reader.next_char() {
                        Some(ch) if ch.is_digit(10) => { return self.number() },
                        _ => { self.reader.bump(); return Token::Dot }
                    }
                }
                Some('{') => { self.bump(); return Token::LBrace },
                Some('}') => { self.bump(); return Token::RBrace },
                Some('[') => { self.bump(); return Token::LBrack },
                Some(']') => { self.bump(); return Token::RBrack },
                Some('(') => { self.bump(); return Token::LParen },
                Some(')') => { self.bump(); return Token::RParen },
                Some(';') => { self.bump(); return Token::Semi },
                Some(':') => { self.bump(); return Token::Colon },
                Some(',') => { self.bump(); return Token::Comma },
                Some('<') => return self.lt(),
                Some('>') => return self.gt(),
                Some('=') => return self.if_equality(Token::Assign, Token::Eq, Token::StrictEq),
                Some('+') => return self.plus(),
                Some('-') => return self.minus(),
                Some('*') => return self.if_assign(Token::StarAssign, Token::Star),
                Some('%') => return self.if_assign(Token::ModAssign, Token::Mod),
                Some('^') => return self.if_assign(Token::BitXorAssign, Token::BitXor),
                Some('&') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('&') => { self.bump(); return Token::LogicalAnd },
                        _ => return Token::BitAnd
                    }
                },
                Some('|') => {
                    self.bump();
                    match self.reader.curr_char() {
                        Some('|') => { self.bump(); return Token::LogicalOr },
                        _ => return Token::BitOr
                    }
                },
                Some('~') => { self.bump(); return Token::Tilde },
                Some('!') => return self.if_equality(Token::Bang, Token::NEq, Token::StrictNEq),
                Some('?') => { self.bump(); return Token::Question },
                Some('"') => return self.string(),
                Some('\'') => return self.string(),
                Some('\r') => {
                    self.bump();
                    if self.reader.curr_char() == Some('\n') {
                        self.bump();
                    }
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                },
                Some(ch) if ch.is_es_newline() => {
                    self.bump();
                    if self.cx.get().is_asi_possible() {
                        return Token::Newline;
                    }
                }
                Some(ch) if ch.is_digit(10) => return self.number(),
                Some(ch) if ch.is_es_identifier() => return self.word(),
                Some(ch) => return Token::Error(ch),
                None => return Token::EOF
            }
        }
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn bump(&mut self) {
        self.reader.bump();
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        match self.read_token() {
            Token::EOF => None,
            t => Some(t)
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::LinkedList;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn ensure_nonempty(&mut self, get: || -> Token) {
        if self.tokens.len() == 0 {
            self.tokens.push_back(get());
        }
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn read_token(&mut self) -> Token {
        self.lookahead.ensure_nonempty(|| {
            self.read_next_token()
        });
        unimplemented!()
    }

    // private methods

    fn read_next_token(&mut self) -> Token {
        unimplemented!()
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.reader.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.skip_whitespace();
        self.eat()
            .map(|ch| {
                match ch {
                    '1' => Token::DecimalInt(String::from_str("1")),
                    '+' => Token::Plus,
                    _ => Token::Error(ch)
                }
            })
    }
}
<commit_msg>guess: fn() -> Token? nope. can't understand the message src/lexer.rs:125:40: 127:10 error: mismatched types:  expected `fn() -> token::Token`,     found `[closure src/lexer.rs:125:40: 127:10]` (expected fn pointer,     found closure) [E0308] src/lexer.rs:125         self.lookahead.ensure_nonempty(|| { src/lexer.rs:126             self.read_next_token() src/lexer.rs:127         });<commit_after>use std::collections::LinkedList;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn ensure_nonempty(&mut self, get: fn() -> Token) {
        if self.tokens.len() == 0 {
            self.tokens.push_back(get());
        }
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn read_token(&mut self) -> Token {
        self.lookahead.ensure_nonempty(|| {
            self.read_next_token()
        });
        unimplemented!()
    }

    // private methods

    fn read_next_token(&mut self) -> Token {
        unimplemented!()
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.reader.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.skip_whitespace();
        self.eat()
            .map(|ch| {
                match ch {
                    '1' => Token::DecimalInt(String::from_str("1")),
                    '+' => Token::Plus,
                    _ => Token::Error(ch)
                }
            })
    }
}
<|endoftext|>


<commit_before>use std::collections::LinkedList;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn ensure_nonempty(&mut self, get: fn() -> Token) {
        if self.tokens.len() == 0 {
            self.tokens.push_back(get());
        }
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn read_token(&mut self) -> Token {
        self.lookahead.ensure_nonempty(|| {
            self.read_next_token()
        });
        unimplemented!()
    }

    // private methods

    fn read_next_token(&mut self) -> Token {
        unimplemented!()
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.reader.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.skip_whitespace();
        self.eat()
            .map(|ch| {
                match ch {
                    '1' => Token::DecimalInt(String::from_str("1")),
                    '+' => Token::Plus,
                    _ => Token::Error(ch)
                }
            })
    }
}
<commit_msg>guess: &fn() -> Token? nope. can't understand the message src/lexer.rs:125:40: 127:10 error: mismatched types:  expected `&fn() -> token::Token`,     found `[closure src/lexer.rs:125:40: 127:10]` (expected &-ptr,     found closure) [E0308] src/lexer.rs:125         self.lookahead.ensure_nonempty(|| { src/lexer.rs:126             self.read_next_token() src/lexer.rs:127         });<commit_after>use std::collections::LinkedList;

use token::Token;

use std::cell::Cell;
use std::rc::Rc;
use regex::Regex;
use context::Context;
use token::Posn;

pub trait ESCharExt {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharExt for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

struct LineOrientedReader<I> {
    chars: I,
    curr_char: Option<char>,
    next_char: Option<char>,
    curr_posn: Posn
}

impl<I> LineOrientedReader<I> where I: Iterator<Item=char> {
    pub fn new(mut chars: I) -> LineOrientedReader<I> {
        let curr_char = chars.next();
        let next_char = if curr_char.is_some() { chars.next() } else { None };
        LineOrientedReader {
            chars: chars,
            curr_char: curr_char,
            next_char: next_char,
            curr_posn: Posn {
                offset: 0,
                line: 0,
                column: 0
            }
        }
    }

    pub fn curr_char(&mut self) -> Option<char> { self.curr_char }
    pub fn curr_posn(&mut self) -> Posn { self.curr_posn }

    pub fn bump(&mut self) {
        let curr_char = self.next_char;
        let next_char = if curr_char.is_some() { self.chars.next() } else { None };

        self.curr_char = curr_char;
        self.next_char = next_char;

        if (curr_char == Some('\r') && next_char != Some('\n')) ||
           curr_char == Some('\n') ||
           curr_char == Some('\u{2028}') ||
           curr_char == Some('\u{2029}') {
            self.curr_posn.line += 1;
            self.curr_posn.column = 0;
        } else {
            self.curr_posn.column += 1;
        }

        self.curr_posn.offset += 1;
    }
}

struct TokenBuffer {
    tokens: LinkedList<Token>
}

impl TokenBuffer {
    fn new() -> TokenBuffer {
        TokenBuffer {
            tokens: LinkedList::new()
        }
    }

    fn ensure_nonempty(&mut self, get: &fn() -> Token) {
        if self.tokens.len() == 0 {
            self.tokens.push_back(get());
        }
    }

    fn read_token(&mut self) -> Token {
        assert!(self.tokens.len() > 0);
        self.tokens.pop_front().unwrap()
    }
}

pub struct Lexer<I> {
    reader: LineOrientedReader<I>,
    cx: Rc<Cell<Context>>,
    lookahead: TokenBuffer
}

impl<I> Lexer<I> where I: Iterator<Item=char> {
    // constructor

    pub fn new(chars: I, cx: Rc<Cell<Context>>) -> Lexer<I> {
        Lexer {
            reader: LineOrientedReader::new(chars),
            cx: cx,
            lookahead: TokenBuffer::new()
        }
    }

    // public methods

    pub fn read_token(&mut self) -> Token {
        self.lookahead.ensure_nonempty(|| {
            self.read_next_token()
        });
        unimplemented!()
    }

    // private methods

    fn read_next_token(&mut self) -> Token {
        unimplemented!()
    }

    fn is_whitespace(&mut self) -> bool {
        match self.reader.curr_char() {
            Some(ch) => ch.is_es_whitespace(),
            None => false
        }
    }

    fn eat(&mut self) -> Option<char> {
        let ch = self.reader.curr_char();
        self.reader.bump();
        ch
    }

    fn skip_whitespace(&mut self) {
        while self.is_whitespace() {
            self.reader.bump();
        }
    }
}

impl<I> Iterator for Lexer<I> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.skip_whitespace();
        self.eat()
            .map(|ch| {
                match ch {
                    '1' => Token::DecimalInt(String::from_str("1")),
                    '+' => Token::Plus,
                    _ => Token::Error(ch)
                }
            })
    }
}
<|endoftext|>


<commit_before>use std::io::{BufRead, BufReader};
use std::io;
use std::io::prelude::*;
use std::iter;
use std::str;

pub enum TokenKind {
    Identifier,
    IntNumber,
    FloatNumber,
    String,
    Char,
    Symbol,
    Newline,
    End,
}

pub struct Token {
    pub kind: TokenKind,
    pub val: String,
    pub line: i32,
}

impl Token {
    pub fn new(kind: TokenKind, val: &str, line: i32) -> Token {
        Token {
            kind: kind,
            val: val.to_string(),
            line: line,
        }
    }
}

pub struct Lexer<'a> {
    cur_line: i32,
    filename: String,
    peek: iter::Peekable<str::Chars<'a>>,
}

impl<'a> Lexer<'a> {
    pub fn new(filename: String, input: &'a str) -> Lexer<'a> {
        let l = Lexer {
            cur_line: 0,
            filename: filename.to_string(),
            peek: input.chars().peekable(),
        };
        l
    }
    pub fn get_filename(self) -> String {
        self.filename
    }

    pub fn read_identifier(&mut self) -> Token {
        let mut ident = String::new();
        loop {
            match self.peek.peek() {
                Some(&c) => {
                    match c {
                        'a'...'z' | 'A'...'Z' | '_' | '0'...'9' => ident.push(c),
                        _ => break,
                    }
                }
                _ => break,
            };
            self.peek.next();
        }
        Token::new(TokenKind::Identifier, ident.as_str(), self.cur_line)
    }
    pub fn read_newline(&mut self) -> Token {
        self.peek.next();
        Token::new(TokenKind::Newline, "", self.cur_line)
    }
    pub fn read_symbol(&mut self) -> Token {
        let c = self.peek.next();
        let mut sym = String::new();
        sym.push(c.unwrap());
        Token::new(TokenKind::Symbol, sym.as_str(), self.cur_line)
    }

    pub fn read_token(&mut self) -> Option<Token> {
        match self.peek.peek() {
            Some(&c) => {
                match c {
                    'a'...'z' | 'A'...'Z' | '_' => Some(self.read_identifier()),
                    ' ' | '\t' => {
                        self.peek.next();
                        self.read_token()
                    }
                    '\n' => Some(self.read_newline()),
                    _ => Some(self.read_symbol()),
                }
            }
            None => None as Option<Token>,
        };
    }
}
<commit_msg>refine code<commit_after>use std::io::{BufRead, BufReader};
use std::io;
use std::io::prelude::*;
use std::iter;
use std::str;

pub enum TokenKind {
    Identifier,
    IntNumber,
    FloatNumber,
    String,
    Char,
    Symbol,
    Newline,
    End,
}

pub struct Token {
    pub kind: TokenKind,
    pub val: String,
    pub line: i32,
}

impl Token {
    pub fn new(kind: TokenKind, val: &str, line: i32) -> Token {
        Token {
            kind: kind,
            val: val.to_string(),
            line: line,
        }
    }
}

pub struct Lexer<'a> {
    cur_line: i32,
    filename: String,
    peek: iter::Peekable<str::Chars<'a>>,
}

impl<'a> Lexer<'a> {
    pub fn new(filename: String, input: &'a str) -> Lexer<'a> {
        let l = Lexer {
            cur_line: 0,
            filename: filename.to_string(),
            peek: input.chars().peekable(),
        };
        l
    }
    pub fn get_filename(self) -> String {
        self.filename
    }

    pub fn read_identifier(&mut self) -> Token {
        let mut ident = String::new();
        loop {
            match self.peek.peek() {
                Some(&c) => {
                    match c {
                        'a'...'z' | 'A'...'Z' | '_' | '0'...'9' => ident.push(c),
                        _ => break,
                    }
                }
                _ => break,
            };
            self.peek.next();
        }
        Token::new(TokenKind::Identifier, ident.as_str(), self.cur_line)
    }
    pub fn read_newline(&mut self) -> Token {
        self.peek.next();
        Token::new(TokenKind::Newline, "", self.cur_line)
    }
    pub fn read_symbol(&mut self) -> Token {
        let c = self.peek.next();
        let mut sym = String::new();
        sym.push(c.unwrap());
        Token::new(TokenKind::Symbol, sym.as_str(), self.cur_line)
    }

    pub fn read_token(&mut self) -> Option<Token> {
        match self.peek.peek() {
            Some(&c) => {
                match c {
                    'a'...'z' | 'A'...'Z' | '_' => Some(self.read_identifier()),
                    ' ' | '\t' => {
                        self.peek.next();
                        self.read_token()
                    }
                    '\n' => Some(self.read_newline()),
                    _ => Some(self.read_symbol()),
                }
            }
            None => None as Option<Token>,
        }
    }
}
<|endoftext|>


<commit_before>#![allow(dead_code)]

use regex::Regex;

#[repr(u32)]
#[derive(Clone)]
pub enum SubToken {
    Plus     = 0,
    Minus    = 1,
    Multiply = 2,
    Divide   = 3,

    Number   = 4,

    LParen   = 5,
    RParen   = 6,

    End      = 7,
}

#[derive(Clone)]
pub struct Token {
    kind: SubToken,
    value: Option<f64>
}

impl Token {
    pub fn new(kind: SubToken, value: Option<f64>) -> Self {
        Token{kind: kind, value: value}
    }
    pub fn set_kind(&mut self, kind: SubToken) {
        self.kind = kind;
    }
    pub fn set_value(&mut self, value: Option<f64>) {
        self.value = value;
    }
    pub fn get_kind(&self) -> SubToken {
        self.kind.clone()
    }
    pub fn get_value(&self) -> Option<f64> {
        self.value.clone()
    }
}

pub struct Lexer {
    input: String,
    return_previous_token: bool,
    previous_token: Option<Token>
}

impl Lexer {
    pub fn new(input: String) -> Self {
        Lexer{input: input, return_previous_token: false, previous_token: None}
    }
    pub fn get_next_token(&mut self) -> Token {
        if self.return_previous_token {
            self.return_previous_token = false;
            return self.previous_token.clone().unwrap();
        }
        let strip_input = strip_white_space(self.input.clone());
        lazy_static! {
            static ref PLUS_RE: Regex = Regex::new(r"\A\+").unwrap();
            static ref MINUS_RE: Regex = Regex::new(r"\A-").unwrap();
            static ref MULT_RE: Regex = Regex::new(r"\*").unwrap();
            static ref DIV_RE: Regex = Regex::new(r"\A\/").unwrap();
            static ref NUM_RE: Regex = Regex::new(r"\A\d+(\.\d+)?").unwrap();
            static ref LPAREN_RE: Regex = Regex::new(r"\A\(").unwrap();
            static ref RPAREN_RE: Regex = Regex::new(r"\A\)").unwrap();
        }
        let token: Token;
        if PLUS_RE.is_match(&strip_input[..]) {
            PLUS_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Plus, None);
        } else if MINUS_RE.is_match(&strip_input[..]) {
            MINUS_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Minus, None);
        } else if MULT_RE.is_match(&strip_input[..]) {
            MULT_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Multiply, None);
        } else if DIV_RE.is_match(&strip_input[..]) {
            DIV_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Divide, None);
        } else if NUM_RE.is_match(&strip_input[..]) {
            use std::str::FromStr;
            let value: Option<f64> = f64::from_str(NUM_RE.captures(&strip_input[..]).unwrap().at(0).unwrap()).ok();
            NUM_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Number, value);
        } else if LPAREN_RE.is_match(&strip_input[..]) {
            LPAREN_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::LParen, None);
        } else if RPAREN_RE.is_match(&strip_input[..]) {
            RPAREN_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::RParen, None);
        } else if strip_input.is_empty() {
            token = Token::new(SubToken::End, None);
        } else {
            let mut error: String = String::from("Unknown token: ");
            error.push_str(&strip_input[..]);
            panic!(error);
        }
        self.previous_token = Some(token.clone());
        token
    }
    pub fn revert(&mut self) {
        self.return_previous_token = true;
    }
}

fn strip_white_space(input: String) -> String {
    input.split_whitespace().collect::<Vec<&str>>().join("")
}
<commit_msg>Fixed division regex<commit_after>#![allow(dead_code)]

use regex::Regex;

#[repr(u32)]
#[derive(Clone)]
pub enum SubToken {
    Plus     = 0,
    Minus    = 1,
    Multiply = 2,
    Divide   = 3,

    Number   = 4,

    LParen   = 5,
    RParen   = 6,

    End      = 7,
}

#[derive(Clone)]
pub struct Token {
    kind: SubToken,
    value: Option<f64>
}

impl Token {
    pub fn new(kind: SubToken, value: Option<f64>) -> Self {
        Token{kind: kind, value: value}
    }
    pub fn set_kind(&mut self, kind: SubToken) {
        self.kind = kind;
    }
    pub fn set_value(&mut self, value: Option<f64>) {
        self.value = value;
    }
    pub fn get_kind(&self) -> SubToken {
        self.kind.clone()
    }
    pub fn get_value(&self) -> Option<f64> {
        self.value.clone()
    }
}

pub struct Lexer {
    input: String,
    return_previous_token: bool,
    previous_token: Option<Token>
}

impl Lexer {
    pub fn new(input: String) -> Self {
        Lexer{input: input, return_previous_token: false, previous_token: None}
    }
    pub fn get_next_token(&mut self) -> Token {
        if self.return_previous_token {
            self.return_previous_token = false;
            return self.previous_token.clone().unwrap();
        }
        let strip_input = strip_white_space(self.input.clone());
        lazy_static! {
            static ref PLUS_RE: Regex = Regex::new(r"\A\+").unwrap();
            static ref MINUS_RE: Regex = Regex::new(r"\A-").unwrap();
            static ref MULT_RE: Regex = Regex::new(r"\*").unwrap();
            static ref DIV_RE: Regex = Regex::new(r"\A/").unwrap();
            static ref NUM_RE: Regex = Regex::new(r"\A\d+(\.\d+)?").unwrap();
            static ref LPAREN_RE: Regex = Regex::new(r"\A\(").unwrap();
            static ref RPAREN_RE: Regex = Regex::new(r"\A\)").unwrap();
        }
        let token: Token;
        if PLUS_RE.is_match(&strip_input[..]) {
            PLUS_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Plus, None);
        } else if MINUS_RE.is_match(&strip_input[..]) {
            MINUS_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Minus, None);
        } else if MULT_RE.is_match(&strip_input[..]) {
            MULT_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Multiply, None);
        } else if DIV_RE.is_match(&strip_input[..]) {
            DIV_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Divide, None);
        } else if NUM_RE.is_match(&strip_input[..]) {
            use std::str::FromStr;
            let value: Option<f64> = f64::from_str(NUM_RE.captures(&strip_input[..]).unwrap().at(0).unwrap()).ok();
            NUM_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::Number, value);
        } else if LPAREN_RE.is_match(&strip_input[..]) {
            LPAREN_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::LParen, None);
        } else if RPAREN_RE.is_match(&strip_input[..]) {
            RPAREN_RE.replace(&strip_input[..], "");
            token = Token::new(SubToken::RParen, None);
        } else if strip_input.is_empty() {
            token = Token::new(SubToken::End, None);
        } else {
            let mut error: String = String::from("Unknown token: ");
            error.push_str(&strip_input[..]);
            panic!(error);
        }
        self.previous_token = Some(token.clone());
        token
    }
    pub fn revert(&mut self) {
        self.return_previous_token = true;
    }
}

fn strip_white_space(input: String) -> String {
    input.split_whitespace().collect::<Vec<&str>>().join("")
}
<|endoftext|>


<commit_before>use std::collections::HashSet;

use super::{Graphics, Drawer};
use draw::resize;
use time::precise_time_s as time_s;

use glium::glutin::{Event, ElementState};

pub use glium::glutin::{VirtualKeyCode, MouseButton};

/// Manages events and frames
pub struct GameManager<'a>{
    graphics: Graphics<'a>,
    last: f64,
    down_keys: HashSet<VirtualKeyCode>,
    mousepos: (f32, f32),
}

impl<'a> GameManager<'a>{
    #[inline]
    /// Creates a new `Game` from a `Graphics` object
    pub fn new(graphics: Graphics<'a>) -> Self {
        GameManager{
            graphics: graphics,
            last: time_s(),
            mousepos: (0., 0.),
            down_keys: HashSet::new()
        }
    }

    /// Returns some tuple of a `FrameInfo` and a ´Drawer` or None if the window has been closed
    pub fn next_frame(&mut self) -> Option<(FrameInfo, Drawer)>{
        let mut keys = Vec::new();
        let mut mouses = Vec::new();

        let mut resized = None::<(u32, u32)>;

        for ev in self.graphics.poll_events() {
            match ev {
                Event::Closed => return None,
                Event::KeyboardInput(es, _, Some(vkc)) => match es{
                    ElementState::Pressed  => {
                        self.down_keys.insert( vkc);
                        keys.push((true , vkc));
                    },
                    ElementState::Released => {
                        self.down_keys.remove(&vkc);
                        keys.push((false, vkc));
                    }
                },
                Event::MouseMoved(pos) => {
                    let (w, h) = self.graphics.get_h_size();
                    let (x, y) = pos;
                    let (x, y) = (x as f32 - w, h - y as f32);

                    self.mousepos = (x, y);
                },
                // This is neccessary because `graphics` gets immutably borrowed for this for loop
                Event::Resized(w, h) => resized = Some((w, h)),
                Event::MouseInput(state, button) => mouses.push((state == ElementState::Pressed, button)),
                _ => ()
            }
        }

        if let Some((w, h)) = resized{
            resize(&mut self.graphics, w, h);
        }drop(resized);

        let now = time_s();
        let delta = now - self.last;
        self.last = now;

        let update = FrameInfo{
            delta    : delta,
            keyevents: keys,
            mouseevents: mouses,
            down_keys: &self.down_keys,
            mousepos : self.mousepos
        };

        Some((update, Drawer::new(&self.graphics)))
    }
}

/// Wraps together useful data about what has happened (e.g. events)
#[derive(Debug)]
pub struct FrameInfo<'a>{
    /// The time that has passed since last frame
    pub delta: f64,
    /// The last position of the mouse on the screen
    pub mousepos: (f32, f32),
    mouseevents: Vec<(bool, MouseButton)>,
    keyevents: Vec<(bool, VirtualKeyCode)>,

    // All keys that are pressed down
    down_keys: &'a HashSet<VirtualKeyCode>
}

impl<'a> FrameInfo<'a>{
    #[inline]
    /// Returns a slice of all key events that have happened
    pub fn get_key_events(&self) -> &[(bool, VirtualKeyCode)]{
        &self.keyevents
    }
    #[inline]
    /// Returns a slice of all key events that have happened
    pub fn get_mouse_events(&self) -> &[(bool, MouseButton)]{
        &self.mouseevents
    }
    #[inline]
    /// Checks whether a key is pressed down
    pub fn is_down(&self, key: &VirtualKeyCode) -> bool{
        self.down_keys.contains(key)
    }
}

/// Describes objects that can change because of events
pub trait Update{
    /// Changes the object depending on what has happened
    fn update(&mut self, &FrameInfo);
}

/// Macro for easily doing things if particular keys are down
/// # Example
///
/// ```rust
/// # macro_rules! is_down{($l_args:ident; $($($key:ident),+ => $b:block),+) => {}}
/// fn logic(player_y: &mut f32, info: korome::FrameInfo){
///     is_down!{info;
///         W, Up => {
///             player_y -= l_args.delta() as f32;
///         },
///         S, Down => {
///             player_y += l_args.delta() as f32;
///         }
///     };
/// }
/// ```
#[macro_export]
macro_rules! is_down{
    ( $info:ident; $( $( $key:ident ),+ => $b:block ),+ ) => {{
        $( if $( $info.is_down(&$crate::VirtualKeyCode::$key) )||+ $b )+
    }}
}
<commit_msg>Make the meaning of some comment more clear<commit_after>use std::collections::HashSet;

use super::{Graphics, Drawer};
use draw::resize;
use time::precise_time_s as time_s;

use glium::glutin::{Event, ElementState};

pub use glium::glutin::{VirtualKeyCode, MouseButton};

/// Manages events and frames
pub struct GameManager<'a>{
    graphics: Graphics<'a>,
    last: f64,
    down_keys: HashSet<VirtualKeyCode>,
    mousepos: (f32, f32),
}

impl<'a> GameManager<'a>{
    #[inline]
    /// Creates a new `Game` from a `Graphics` object
    pub fn new(graphics: Graphics<'a>) -> Self {
        GameManager{
            graphics: graphics,
            last: time_s(),
            mousepos: (0., 0.),
            down_keys: HashSet::new()
        }
    }

    /// Returns some tuple of a `FrameInfo` and a ´Drawer` or None if the window has been closed
    pub fn next_frame(&mut self) -> Option<(FrameInfo, Drawer)>{
        let mut keys = Vec::new();
        let mut mouses = Vec::new();

        let mut resized = None::<(u32, u32)>;

        for ev in self.graphics.poll_events() {
            match ev {
                Event::Closed => return None,
                Event::KeyboardInput(es, _, Some(vkc)) => match es{
                    ElementState::Pressed  => {
                        self.down_keys.insert( vkc);
                        keys.push((true , vkc));
                    },
                    ElementState::Released => {
                        self.down_keys.remove(&vkc);
                        keys.push((false, vkc));
                    }
                },
                Event::MouseMoved(pos) => {
                    let (w, h) = self.graphics.get_h_size();
                    let (x, y) = pos;
                    let (x, y) = (x as f32 - w, h - y as f32);

                    self.mousepos = (x, y);
                },
                // This is only neccessary because `graphics` gets immutably borrowed for this for-loop
                Event::Resized(w, h) => resized = Some((w, h)),
                Event::MouseInput(state, button) => mouses.push((state == ElementState::Pressed, button)),
                _ => ()
            }
        }

        if let Some((w, h)) = resized{
            resize(&mut self.graphics, w, h);
        }drop(resized);

        let now = time_s();
        let delta = now - self.last;
        self.last = now;

        let update = FrameInfo{
            delta    : delta,
            keyevents: keys,
            mouseevents: mouses,
            down_keys: &self.down_keys,
            mousepos : self.mousepos
        };

        Some((update, Drawer::new(&self.graphics)))
    }
}

/// Wraps together useful data about what has happened (e.g. events)
#[derive(Debug)]
pub struct FrameInfo<'a>{
    /// The time that has passed since last frame
    pub delta: f64,
    /// The last position of the mouse on the screen
    pub mousepos: (f32, f32),
    mouseevents: Vec<(bool, MouseButton)>,
    keyevents: Vec<(bool, VirtualKeyCode)>,

    // All keys that are pressed down
    down_keys: &'a HashSet<VirtualKeyCode>
}

impl<'a> FrameInfo<'a>{
    #[inline]
    /// Returns a slice of all key events that have happened
    pub fn get_key_events(&self) -> &[(bool, VirtualKeyCode)]{
        &self.keyevents
    }
    #[inline]
    /// Returns a slice of all key events that have happened
    pub fn get_mouse_events(&self) -> &[(bool, MouseButton)]{
        &self.mouseevents
    }
    #[inline]
    /// Checks whether a key is pressed down
    pub fn is_down(&self, key: &VirtualKeyCode) -> bool{
        self.down_keys.contains(key)
    }
}

/// Describes objects that can change because of events
pub trait Update{
    /// Changes the object depending on what has happened
    fn update(&mut self, &FrameInfo);
}

/// Macro for easily doing things if particular keys are down
/// # Example
///
/// ```rust
/// # macro_rules! is_down{($l_args:ident; $($($key:ident),+ => $b:block),+) => {}}
/// fn logic(player_y: &mut f32, info: korome::FrameInfo){
///     is_down!{info;
///         W, Up => {
///             player_y -= l_args.delta() as f32;
///         },
///         S, Down => {
///             player_y += l_args.delta() as f32;
///         }
///     };
/// }
/// ```
#[macro_export]
macro_rules! is_down{
    ( $info:ident; $( $( $key:ident ),+ => $b:block ),+ ) => {{
        $( if $( $info.is_down(&$crate::VirtualKeyCode::$key) )||+ $b )+
    }}
}
<|endoftext|>


<commit_before>
//use serde_json;

// macro_rules! query {
//     ($($key:expr => $val:tt),*) => {
//         $(
//             println!("e {:?} {:?}", $key, $val);
//         )*
//     };
//
// }

/*
{
  "selector": {
    "year": {
      "$gt": 2010
    }
  },
  "fields": ["_id", "_rev", "year", "title"],
  "sort": [{"year": "asc"}],
  "limit": 10,
  "skip": 0
}

*/

// macro_rules! query {
//      ({$($section:expr => {$($key:expr => {$comp:expr => $val:expr}),*}),*}) => {{
//         $(
//             $(
//                 println!("out ${:?} {:?} {:?} {:?}", $section, $key, $comp, $val);
//             )*
//         )*
//     }};
// }

macro_rules! query_type {
    ({
        $section:expr => {
            $(
                $key:expr => {
                    $comp:expr => $val:expr
                 }
             ),*
         }
    }) => {{
        let mut selector = Map::new();
        $(
            let mut kv = Map::new();
            kv.insert($comp.to_string(), serde_json::to_value($val));
            selector.insert($key.to_string(), kv);
            println!("selector {:?} {:?} {:?}", $key, $comp, $val);
        )*
        serde_json::to_value(selector)
    }};
    ({
        $field_section:expr => $fields:expr
    }) => {{
        let v = vec!["_boom"];
        let mut fields = Map::new();
        println!("field {:?}", $fields);
        fields.insert($field_section.to_string(), serde_json::to_value(v));
        serde_json::to_value(fields)
    }};
}

#[macro_export]
macro_rules! query {
    ( {$($section:expr => $content:tt),*} ) => {{
        use serde_json;
        use serde_json::{Map};
        let mut map = Map::new();
        $(
            println!("section {:?}", $section);
            map.insert($section.to_string(), query_type!({$section => $content}));
        )*
        map
    }};
}

// fn main() {
//     let b = query!({
//         "selector" => {
//             "_id" => {
//                 "$eq" => "boom"
//             },
//             "name" => {
//                 "$lte" => "garren"
//             }
//         }//,
//         //"fields" => ["_id", "_rev"] //tt
//     });
//
//     println!("B {:?}", b);
// }

/*
doc! { "title" => "Jaws",
    "array" => [ 1, 2, 3 ] };

#[macro_export]
macro_rules! doc {
    () => {{ $crate::Document::new() }};

    ( $($key:expr => $val:tt),* ) => {{
        let mut document = $crate::Document::new();

        $(
            document.insert_bson($key.to_owned(), bson!($val));
        )*

        document
    }};
}

*/

#[cfg(test)]
mod tests {
    use serde_json;
    use serde_json::{Map};

    #[test]
    fn create_selector_query () {
        let query = query!({
            "selector" => {
                "_id" => {
                    "$eq" => "boom"
                },
                "name" => {
                    "$lte" => "garren"
                }
            }
        });

        let map: Map<String, Map<String, String>> = serde_json::from_value(query.get("selector").unwrap().clone()).unwrap();
        let _id = map.get("_id").unwrap();
        let name = map.get("name").unwrap();

        assert_eq!(_id.get("$eq").unwrap().to_string(), "boom");
        assert_eq!(name.get("$lte").unwrap().to_string(), "garren");
    }

    #[test]
    fn set_keys_in_query () {
        let query = query!({
            "selector" => {
                "_id" => {
                    "$gt" => "1"
                }
            },
            "fields" => ["_id", "name"]
        });

        //assert_eq!(query.get("fields").unwrap()[0], "_id");
        //assert_eq!(query.get("fields").unwrap()[1], "name");
    }
}
<commit_msg>progress<commit_after>
//use serde_json;

// macro_rules! query {
//     ($($key:expr => $val:tt),*) => {
//         $(
//             println!("e {:?} {:?}", $key, $val);
//         )*
//     };
//
// }

/*
{
  "selector": {
    "year": {
      "$gt": 2010
    }
  },
  "fields": ["_id", "_rev", "year", "title"],
  "sort": [{"year": "asc"}],
  "limit": 10,
  "skip": 0
}

*/

// macro_rules! query {
//      ({$($section:expr => {$($key:expr => {$comp:expr => $val:expr}),*}),*}) => {{
//         $(
//             $(
//                 println!("out ${:?} {:?} {:?} {:?}", $section, $key, $comp, $val);
//             )*
//         )*
//     }};
// }

macro_rules! query_type {
    ({
        $section:expr => {
            $(
                $key:expr => {
                    $comp:expr => $val:expr
                 }
             ),*
         }
    }) => {{
        let mut selector = Map::new();
        $(
            let mut kv = Map::new();
            kv.insert($comp.to_string(), serde_json::to_value($val));
            selector.insert($key.to_string(), kv);
            println!("selector {:?} {:?} {:?}", $key, $comp, $val);
        )*
        serde_json::to_value(selector)
    }};
    ({
        $field_section:expr => $fields:expr
    }) => {{
        println!("field ${:?} {:?}", $field_section, $fields);
        let v: serde_json::Value = match $field_section {
            "fields" => serde_json::to_value($fields.to_vec()),
            _ => serde_json::to_value($fields)
        };

        let v = if $field_section == "fields" {
            serde_json::to_value($fields.to_vec())
        } else {
            serde_json::to_value($fields)
        };
        //let v = $fields.to_vec();
        let mut fields = Map::new();
        fields.insert($field_section.to_string(), v);
        serde_json::to_value(fields)
    }};
}

#[macro_export]
macro_rules! query {
    ( {$($section:expr => $content:tt),*} ) => {{
        use serde_json;
        use serde_json::{Map};
        let mut map = Map::new();
        $(
            println!("section {:?}", $section);
            map.insert($section.to_string(), query_type!({$section => $content}));
        )*
        map
    }};
}

/*
doc! { "title" => "Jaws",
    "array" => [ 1, 2, 3 ] };

#[macro_export]
macro_rules! doc {
    () => {{ $crate::Document::new() }};

    ( $($key:expr => $val:tt),* ) => {{
        let mut document = $crate::Document::new();

        $(
            document.insert_bson($key.to_owned(), bson!($val));
        )*

        document
    }};
}

*/

#[cfg(test)]
mod tests {
    use serde_json;
    use serde_json::{Map};

    #[test]
    fn create_selector_query () {
        let query = query!({
            "selector" => {
                "_id" => {
                    "$eq" => "boom"
                },
                "name" => {
                    "$lte" => "garren"
                }
            }
        });

        let map: Map<String, Map<String, String>> = serde_json::from_value(query.get("selector").unwrap().clone()).unwrap();
        let _id = map.get("_id").unwrap();
        let name = map.get("name").unwrap();

        assert_eq!(_id.get("$eq").unwrap().to_string(), "boom");
        assert_eq!(name.get("$lte").unwrap().to_string(), "garren");
    }

    #[test]
    fn set_fields_in_query () {
        let query = query!({
            "selector" => {
                "_id" => {
                    "$gt" => "1"
                }
            },
            "fields" => ["_id", "name"]
        });

        let fields: Map<String, Vec<String>> = serde_json::from_value(query.get("fields").unwrap().clone()).unwrap();
        assert_eq!(fields.get("fields").unwrap()[0], "_id");
        assert_eq!(fields.get("fields").unwrap()[1], "name");
    }

    #[test]
    fn set_limit_in_query () {
        let query = query!({
            "selector" => {
                "_id" => {
                    "$gt" => "1"
                }
            },
            "fields" => ["_id", "name"],
            "limit" => 10
        });

        let limit: i32 = serde_json::from_value(query.get("limit").unwrap().clone()).unwrap();
        assert_eq!(limit, 10);
    }
}
<|endoftext|>


<commit_before>use std::cell::UnsafeCell;
use std::ptr;
use std::sync::atomic::{AtomicUsize, fence, Ordering};

use super::MAX_JOBS;
use super::job::Job;

const MASK: usize = MAX_JOBS - 1;

/// A double-ended job queue.
/// This is currently locking, but will be made lock-free when performance demands it.
pub struct Queue {
    buf: UnsafeCell<[*mut Job; MAX_JOBS]>,
    top: AtomicUsize,
    bottom: AtomicUsize,
}

unsafe impl Send for Queue {}
unsafe impl Sync for Queue {}

macro_rules! memory_barrier {
    () => {
        fence(Ordering::SeqCst);
    }
}

#[cfg(nightly)]
macro_rules! compiler_barrier {
    () => {
        ::std::intrinsics::atomic_singlethreadfence()
    }
}

// do a full memory barrier on stable/beta.
#[cfg(not(nightly))]
macro_rules! compiler_barrier {
    () => {
        fence(Ordering::SeqCst);
    }
}

impl Queue {
    pub fn new() -> Self {
        Queue {
            buf: UnsafeCell::new([ptr::null_mut(); MAX_JOBS]),
            top: AtomicUsize::new(0),
            bottom: AtomicUsize::new(0),
        }
    }

    // push a job onto the private end of the queue.
    // this must be performed from the thread which logically
    // owns this queue.
    pub unsafe fn push(&self, job: *mut Job) {
        assert!(!job.is_null(), "Attempted to push null job onto queue");
        let b = self.bottom.load(Ordering::Acquire);
        (*self.buf.get())[b & MASK] = job;

        compiler_barrier!();
        self.bottom.store(b + 1, Ordering::Release);
    }

    // steal a job from the public end of the queue.
    // loops until it wins a race or the queue is empty.
    pub fn steal(&self) -> Option<*mut Job> {
        loop {
            unsafe {
                let t = self.top.load(Ordering::Acquire);

                compiler_barrier!();
                let b = self.bottom.load(Ordering::Acquire);
                if t < b {
                    // non-empty queue.
                    // may be racing against other steal/pop calls.
                    let job = (*self.buf.get())[t & MASK];
                    if self.top.compare_and_swap(t, t + 1, Ordering::AcqRel) != t {
                        // lost the race.
                        continue;
                    } else {
                        // won the race.
                        return Some(job);
                    }
                } else {
                    // empty queue.
                    return None;
                }
            }
        }
    }

    // pop a job from the private end of the queue.
    // this must be done from the thread which logically
    // owns this queue.
    pub unsafe fn pop(&self) -> Option<*mut Job> {
        let b = self.bottom.load(Ordering::Acquire).wrapping_sub(1);
        // check that bottom is not 0 before storing new b.
        if b == ::std::usize::MAX {
            return None;
        }
        self.bottom.store(b, Ordering::Release);
        memory_barrier!();

        let t = self.top.load(Ordering::Acquire);
        if t <= b {
            // non-empty queue.
            let mut job = Some((*self.buf.get())[b & MASK]);
            if t != b {
                job
            } else {
                if self.top.compare_and_swap(t, t + 1, Ordering::SeqCst) != t {
                    job = None;
                }

                self.bottom.store(t + 1, Ordering::Release);
                job
            }
        } else {
            // empty queue.
            self.bottom.store(t, Ordering::Release);
            None
        }
    }

    // get the length of this queue.
    pub fn len(&self) -> usize {
        self.bottom.load(Ordering::Acquire) - self.top.load(Ordering::Acquire)
    }

    // reset the counters, so they don't wrap around.
    pub fn reset_counters(&self) {
        debug_assert_eq!(self.len(), 0);

        self.bottom.store(0, Ordering::Release);
        self.top.store(0, Ordering::Release);
    }
}
<commit_msg>saturating sub for queue length<commit_after>use std::cell::UnsafeCell;
use std::ptr;
use std::sync::atomic::{AtomicUsize, fence, Ordering};

use super::MAX_JOBS;
use super::job::Job;

const MASK: usize = MAX_JOBS - 1;

/// A double-ended job queue.
/// This is currently locking, but will be made lock-free when performance demands it.
pub struct Queue {
    buf: UnsafeCell<[*mut Job; MAX_JOBS]>,
    top: AtomicUsize,
    bottom: AtomicUsize,
}

unsafe impl Send for Queue {}
unsafe impl Sync for Queue {}

macro_rules! memory_barrier {
    () => {
        fence(Ordering::SeqCst);
    }
}

#[cfg(nightly)]
macro_rules! compiler_barrier {
    () => {
        ::std::intrinsics::atomic_singlethreadfence()
    }
}

// do a full memory barrier on stable/beta.
#[cfg(not(nightly))]
macro_rules! compiler_barrier {
    () => {
        fence(Ordering::SeqCst);
    }
}

impl Queue {
    pub fn new() -> Self {
        Queue {
            buf: UnsafeCell::new([ptr::null_mut(); MAX_JOBS]),
            top: AtomicUsize::new(0),
            bottom: AtomicUsize::new(0),
        }
    }

    // push a job onto the private end of the queue.
    // this must be performed from the thread which logically
    // owns this queue.
    pub unsafe fn push(&self, job: *mut Job) {
        assert!(!job.is_null(), "Attempted to push null job onto queue");
        let b = self.bottom.load(Ordering::Acquire);
        (*self.buf.get())[b & MASK] = job;

        compiler_barrier!();
        self.bottom.store(b + 1, Ordering::Release);
    }

    // steal a job from the public end of the queue.
    // loops until it wins a race or the queue is empty.
    pub fn steal(&self) -> Option<*mut Job> {
        loop {
            unsafe {
                let t = self.top.load(Ordering::Acquire);

                compiler_barrier!();
                let b = self.bottom.load(Ordering::Acquire);
                if t < b {
                    // non-empty queue.
                    // may be racing against other steal/pop calls.
                    let job = (*self.buf.get())[t & MASK];
                    if self.top.compare_and_swap(t, t + 1, Ordering::AcqRel) != t {
                        // lost the race.
                        continue;
                    } else {
                        // won the race.
                        return Some(job);
                    }
                } else {
                    // empty queue.
                    return None;
                }
            }
        }
    }

    // pop a job from the private end of the queue.
    // this must be done from the thread which logically
    // owns this queue.
    pub unsafe fn pop(&self) -> Option<*mut Job> {
        let b = self.bottom.load(Ordering::Acquire).wrapping_sub(1);
        // check that bottom is not 0 before storing new b.
        if b == ::std::usize::MAX {
            return None;
        }
        self.bottom.store(b, Ordering::Release);
        memory_barrier!();

        let t = self.top.load(Ordering::Acquire);
        if t <= b {
            // non-empty queue.
            let mut job = Some((*self.buf.get())[b & MASK]);
            if t != b {
                job
            } else {
                if self.top.compare_and_swap(t, t + 1, Ordering::SeqCst) != t {
                    job = None;
                }

                self.bottom.store(t + 1, Ordering::Release);
                job
            }
        } else {
            // empty queue.
            self.bottom.store(t, Ordering::Release);
            None
        }
    }

    // get the length of this queue.
    pub fn len(&self) -> usize {
        self.bottom.load(Ordering::Acquire).saturating_sub(self.top.load(Ordering::Acquire))
    }

    // reset the counters, so they don't wrap around.
    pub fn reset_counters(&self) {
        debug_assert_eq!(self.len(), 0);

        self.bottom.store(0, Ordering::Release);
        self.top.store(0, Ordering::Release);
    }
}
<|endoftext|>


<commit_before>
use error::MpdResult;
use connection::{FromConn, MpdConnection, mpd_connection};
use songs::{MpdSong, MpdSongs, mpd_song};
use std::c_str::CString;
use libc::{c_uint, c_int, c_char};

extern {
    fn mpd_run_get_queue_song_pos(connection: *mut mpd_connection, pos: c_uint) -> *mut mpd_song;
    fn mpd_run_get_queue_song_id(connection: *mut mpd_connection, id: c_uint) -> *mut mpd_song;
    fn mpd_run_move_id(connection: *mut mpd_connection, from: c_uint, to: c_uint) -> bool;
    fn mpd_run_move(connection: *mut mpd_connection, from: c_uint, to: c_uint) -> bool;
    fn mpd_run_move_range(connection: *mut mpd_connection, start: c_uint, end: c_uint, to: c_uint) -> bool;
    fn mpd_run_swap_id(connection: *mut mpd_connection, id1: c_uint, id2: c_uint) -> bool;
    fn mpd_run_swap(connection: *mut mpd_connection, pos1: c_uint, pos2: c_uint) -> bool;
    fn mpd_run_add_id(connection: *mut mpd_connection, file: *const c_char) -> c_int;
    fn mpd_run_add_id_to(connection: *mut mpd_connection, uri: *const c_char, to: c_uint) -> c_int;
    fn mpd_send_list_queue_meta(connection: *mut mpd_connection) -> bool;
    fn mpd_send_list_queue_range_meta(connection: *mut mpd_connection, start: c_uint, end: c_uint) -> bool;
    fn mpd_run_delete(connection: *mut mpd_connection, pos: c_uint) -> bool;
    fn mpd_run_delete_range(connection: *mut mpd_connection, start: c_uint, end: c_uint) -> bool;
    fn mpd_run_delete_id(connection: *mut mpd_connection, id: c_uint) -> bool;
}

pub struct MpdQueue<'a> {
    pub conn: &'a MpdConnection
}

impl<'a> MpdQueue<'a> {
    pub fn from_conn(conn: &'a MpdConnection) -> MpdQueue<'a> {
        MpdQueue { conn: conn }
    }

    /// Get song at some position in queue
    pub fn nth(&self, index: uint) -> MpdResult<MpdSong> {
        let song = unsafe { mpd_run_get_queue_song_pos(self.conn.conn, index as c_uint) };
        if song.is_null() {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(MpdSong { song: song })
        }
    }

    /// Get song by queue id
    pub fn get(&self, id: uint) -> MpdResult<MpdSong> {
        let song = unsafe { mpd_run_get_queue_song_id(self.conn.conn, id as c_uint) };
        if song.is_null() {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(MpdSong { song: song })
        }
    }

    /// Insert new song into queue at given position
    pub fn insert<T: ToSongUri>(&mut self, pos: uint, song: T) -> MpdResult<uint> {
        let uid = unsafe { mpd_run_add_id_to(self.conn.conn, song.song_uri().as_ptr(), pos as c_uint) };
        if uid < 0 {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(uid as uint)
        }
    }

    /// Add song at the end of the queue
    pub fn push<T: ToSongUri>(&mut self, song: T) -> MpdResult<uint> {
        let uid = unsafe { mpd_run_add_id(self.conn.conn, song.song_uri().as_ptr()) };
        if uid < 0 {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(uid as uint)
        }
    }

    /// Move song to some position in queue
    pub fn move_pos(&mut self, to: uint, from: uint) -> MpdResult<()> {
        if unsafe { mpd_run_move(self.conn.conn, from as c_uint, to as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Move songs in given range
    pub fn move_range(&mut self, pos: uint, start: uint, end: uint) -> MpdResult<()> {
        if unsafe { mpd_run_move_range(self.conn.conn, start as c_uint, end as c_uint, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Move song to some position in queue by id
    pub fn move_to(&mut self, pos: uint, song: &MpdSong) -> MpdResult<()> {
        if unsafe { mpd_run_move_id(self.conn.conn, song.id() as c_uint, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Swap two songs in given positions
    pub fn swap_pos(&mut self, song1: uint, song2: uint) -> MpdResult<()> {
        if unsafe { mpd_run_swap(self.conn.conn, song1 as c_uint, song2 as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Swap two songs in given positions by id
    pub fn swap(&mut self, song1: &MpdSong, song2: &MpdSong) -> MpdResult<()> {
        if unsafe { mpd_run_swap_id(self.conn.conn, song1.id() as c_uint, song2.id() as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove a song
    pub fn remove(&mut self, song: &MpdSong) -> MpdResult<()> {
        if unsafe { mpd_run_delete_id(self.conn.conn, song.id() as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove songs at given position
    pub fn remove_pos(&mut self, pos: uint) -> MpdResult<()> {
        if unsafe { mpd_run_delete(self.conn.conn, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove songs in given range
    pub fn remove_range(&mut self, start: uint, end: uint) -> MpdResult<()> {
        if unsafe { mpd_run_delete_range(self.conn.conn, start as c_uint, end as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Iterate over songs in the queue
    pub fn iter(&self) -> MpdResult<MpdSongs> {
        if unsafe { mpd_send_list_queue_meta(self.conn.conn) } {
            Ok(MpdSongs { conn: self.conn })
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Iterate over songs in given range
    pub fn iter_range(&self, start: uint, end: uint) -> MpdResult<MpdSongs> {
        if unsafe { mpd_send_list_queue_range_meta(self.conn.conn, start as c_uint, end as c_uint) } {
            Ok(MpdSongs { conn: self.conn })
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Length of the queue
    pub fn len(&self) -> MpdResult<uint> {
        self.conn.status().map(|s| s.queue_len())
    }

    /// Returns true if queue is empty
    pub fn is_empty(&self) -> MpdResult<bool> {
        self.len().map(|v| v == 0)
    }
}

pub trait ToSongUri {
    fn song_uri(&self) -> CString;
}

impl ToSongUri for MpdSong {
    #[inline] fn song_uri(&self) -> CString {
        self.uri().to_c_str()
    }
}

impl<T: ToCStr> ToSongUri for T {
    #[inline] fn song_uri(&self) -> CString {
        self.to_c_str()
    }
}
<commit_msg>queue methods for ids<commit_after>
use error::MpdResult;
use connection::{FromConn, MpdConnection, mpd_connection};
use songs::{MpdSong, MpdSongs, mpd_song};
use std::c_str::CString;
use libc::{c_uint, c_int, c_char};

extern {
    fn mpd_run_get_queue_song_pos(connection: *mut mpd_connection, pos: c_uint) -> *mut mpd_song;
    fn mpd_run_get_queue_song_id(connection: *mut mpd_connection, id: c_uint) -> *mut mpd_song;
    fn mpd_run_move_id(connection: *mut mpd_connection, from: c_uint, to: c_uint) -> bool;
    fn mpd_run_move(connection: *mut mpd_connection, from: c_uint, to: c_uint) -> bool;
    fn mpd_run_move_range(connection: *mut mpd_connection, start: c_uint, end: c_uint, to: c_uint) -> bool;
    fn mpd_run_swap_id(connection: *mut mpd_connection, id1: c_uint, id2: c_uint) -> bool;
    fn mpd_run_swap(connection: *mut mpd_connection, pos1: c_uint, pos2: c_uint) -> bool;
    fn mpd_run_add_id(connection: *mut mpd_connection, file: *const c_char) -> c_int;
    fn mpd_run_add_id_to(connection: *mut mpd_connection, uri: *const c_char, to: c_uint) -> c_int;
    fn mpd_send_list_queue_meta(connection: *mut mpd_connection) -> bool;
    fn mpd_send_list_queue_range_meta(connection: *mut mpd_connection, start: c_uint, end: c_uint) -> bool;
    fn mpd_run_delete(connection: *mut mpd_connection, pos: c_uint) -> bool;
    fn mpd_run_delete_range(connection: *mut mpd_connection, start: c_uint, end: c_uint) -> bool;
    fn mpd_run_delete_id(connection: *mut mpd_connection, id: c_uint) -> bool;
}

pub struct MpdQueue<'a> {
    pub conn: &'a MpdConnection
}

impl<'a> MpdQueue<'a> {
    pub fn from_conn(conn: &'a MpdConnection) -> MpdQueue<'a> {
        MpdQueue { conn: conn }
    }

    /// Get song at some position in queue
    pub fn nth(&self, index: uint) -> MpdResult<MpdSong> {
        let song = unsafe { mpd_run_get_queue_song_pos(self.conn.conn, index as c_uint) };
        if song.is_null() {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(MpdSong { song: song })
        }
    }

    /// Get song by queue id
    pub fn get(&self, id: uint) -> MpdResult<MpdSong> {
        let song = unsafe { mpd_run_get_queue_song_id(self.conn.conn, id as c_uint) };
        if song.is_null() {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(MpdSong { song: song })
        }
    }

    /// Insert new song into queue at given position
    pub fn insert<T: ToSongUri>(&mut self, pos: uint, song: T) -> MpdResult<uint> {
        let uid = unsafe { mpd_run_add_id_to(self.conn.conn, song.song_uri().as_ptr(), pos as c_uint) };
        if uid < 0 {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(uid as uint)
        }
    }

    /// Add song at the end of the queue
    pub fn push<T: ToSongUri>(&mut self, song: T) -> MpdResult<uint> {
        let uid = unsafe { mpd_run_add_id(self.conn.conn, song.song_uri().as_ptr()) };
        if uid < 0 {
            Err(FromConn::from_conn(self.conn).unwrap())
        } else {
            Ok(uid as uint)
        }
    }

    /// Move song to some position in queue
    pub fn move_pos(&mut self, to: uint, from: uint) -> MpdResult<()> {
        if unsafe { mpd_run_move(self.conn.conn, from as c_uint, to as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Move songs in given range
    pub fn move_range(&mut self, pos: uint, start: uint, end: uint) -> MpdResult<()> {
        if unsafe { mpd_run_move_range(self.conn.conn, start as c_uint, end as c_uint, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Move song to some position in queue by id
    pub fn move_to(&mut self, pos: uint, song: &MpdSong) -> MpdResult<()> {
        self.move_id(pos, song.id())
    }

    pub fn move_id(&mut self, pos: uint, id: uint) -> MpdResult<()> {
        if unsafe { mpd_run_move_id(self.conn.conn, id as c_uint, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Swap two songs in given positions
    pub fn swap_pos(&mut self, song1: uint, song2: uint) -> MpdResult<()> {
        if unsafe { mpd_run_swap(self.conn.conn, song1 as c_uint, song2 as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Swap two songs in given positions by id
    pub fn swap(&mut self, song1: &MpdSong, song2: &MpdSong) -> MpdResult<()> {
        self.swap_id(song1.id(), song2.id())
    }

    pub fn swap_id(&mut self, song1: uint, song2: uint) -> MpdResult<()> {
        if unsafe { mpd_run_swap_id(self.conn.conn, song1 as c_uint, song2 as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove a song
    pub fn remove(&mut self, song: &MpdSong) -> MpdResult<()> {
        self.remove_id(song.id())
    }

    pub fn remove_id(&mut self, id: uint) -> MpdResult<()> {
        if unsafe { mpd_run_delete_id(self.conn.conn, id as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove songs at given position
    pub fn remove_pos(&mut self, pos: uint) -> MpdResult<()> {
        if unsafe { mpd_run_delete(self.conn.conn, pos as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Remove songs in given range
    pub fn remove_range(&mut self, start: uint, end: uint) -> MpdResult<()> {
        if unsafe { mpd_run_delete_range(self.conn.conn, start as c_uint, end as c_uint) } {
            Ok(())
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Iterate over songs in the queue
    pub fn iter(&self) -> MpdResult<MpdSongs> {
        if unsafe { mpd_send_list_queue_meta(self.conn.conn) } {
            Ok(MpdSongs { conn: self.conn })
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Iterate over songs in given range
    pub fn iter_range(&self, start: uint, end: uint) -> MpdResult<MpdSongs> {
        if unsafe { mpd_send_list_queue_range_meta(self.conn.conn, start as c_uint, end as c_uint) } {
            Ok(MpdSongs { conn: self.conn })
        } else {
            Err(FromConn::from_conn(self.conn).unwrap())
        }
    }

    /// Length of the queue
    pub fn len(&self) -> MpdResult<uint> {
        self.conn.status().map(|s| s.queue_len())
    }

    /// Returns true if queue is empty
    pub fn is_empty(&self) -> MpdResult<bool> {
        self.len().map(|v| v == 0)
    }
}

pub trait ToSongUri {
    fn song_uri(&self) -> CString;
}

impl ToSongUri for MpdSong {
    #[inline] fn song_uri(&self) -> CString {
        self.uri().to_c_str()
    }
}

impl<T: ToCStr> ToSongUri for T {
    #[inline] fn song_uri(&self) -> CString {
        self.to_c_str()
    }
}
<|endoftext|>