<commit_before>extern mod extra;
use extra::container::Deque;
use extra::ringbuf::RingBuf;
use std::iter::Peekable;
#[deriving(Clone, Eq, ToStr)]
pub enum TokenEnum {
	EOF,
	NAME,
	OPERATOR,
	NUMBER,
	FLOAT,
	LPARENS,
	RPARENS,
	LBRACKET,
	RBRACKET,
	LBRACE,
	RBRACE,
	INDENTSTART,
	INDENTLEVEL,
	COMMA,
	EQUALSSIGN,
	SEMICOLON,
	MODULE,
	CLASS,
	INSTANCE,
	WHERE,
	LET,
	IN,
	CASE,
	OF,
	ARROW,
	TYPEDECL,
	DATA
}

#[deriving(Clone, Eq, ToStr)]
pub struct Location {
    column : int,
    row : int,
    absolute : int
}

#[deriving(Clone)]
pub struct Token {
    token : TokenEnum,
    value : ~str,
    location : Location
}
impl Token {
    fn eof() -> Token {
        Token { token : EOF, value : ~"", location : Location { column : -1, row : -1, absolute : -1} }
    }

    fn new(token : TokenEnum, value : ~str, loc : Location) -> Token {
        Token { token : token, value : value, location : loc }
    }
    fn new_(token : TokenEnum, value : ~str) -> Token {
        Token::new(token, value, Location { column : -1, row : -1, absolute : -1 })
    }
}

impl Eq for Token {
    fn eq(&self, rhs : &Token) -> bool {
        self.token == rhs.token && self.value == rhs.value
    }
}

fn name_or_keyword(tok : &str) -> TokenEnum {
    match tok {
        "module" => MODULE,
        "class" => CLASS,
        "instance" => INSTANCE,
        "where" => WHERE,
        "let" => LET,
        "in" => IN,
        "case" => CASE,
        "of" => OF,
        "->" => ARROW,
        "data" => DATA,
        _ => NAME
    }
}

fn is_operator(first_char : char) -> bool {
    match first_char {
        '+' => true,
        '-' => true,
        '*' => true,
        '/' => true,
        '.' => true,
        '$' => true,
        ':' => true,
        '=' => true,
        '<' => true,
        '>' => true,
        '|' => true,
        '&' => true,
        _ => false
    }
}

pub struct Lexer<Stream> {
    priv input : Peekable<char, Stream>,
    priv location : Location,
    priv previousLocation : Location,
    priv unprocessedTokens : ~[Token],
    priv tokens : extra::ringbuf::RingBuf<Token>,
    priv indentLevels : ~[int],
    priv offset : uint
}

impl <Stream : Iterator<char>> Lexer<Stream> {

    pub fn new(input : Stream) -> Lexer<Stream> {
        let start = Location { column : 0, row : 0, absolute : 0};
        Lexer {
            input : input.peekable(),
            location : start,
            previousLocation : start,
            unprocessedTokens : ~[],
            tokens : extra::ringbuf::RingBuf::with_capacity(20),
            indentLevels : ~[],
            offset : 0}
    }
    pub fn module_next<'a>(&'a mut self) -> &'a Token {
        let mut newline = false;
        let n = self.next_indent_token(&mut newline);
        self.unprocessedTokens.push(n);
        let newTok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].token;
        let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;

        if newTok != LBRACE && newTok != MODULE {
            self.unprocessedTokens.push(Token::new(INDENTSTART, ~"{n}", loc));
        }
        if newline {
            self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", loc));
        }

        self.layout_independent_token(|_| false);
        self.current()
    }

    pub fn next_<'a>(&'a mut self) -> &'a Token {
        self.next(|_| false)
    }
    pub fn next<'a>(&'a mut self, parseError : |&Token| -> bool) -> &'a Token {
        if self.offset > 0 {
            self.offset -= 1;
            match self.tokens.iter().idx(self.tokens.len() - 1 - self.offset) {
                Some(token) => token,
                None => fail!("Impossible empty tokens stream")
            }
        }
        else if self.unprocessedTokens.len() > 0 {
            self.layout_independent_token(parseError);
            self.tokens.back().unwrap()
        }
        else {
            self.new_token(parseError)
        }
    }

    pub fn current<'a>(&'a self) -> &'a Token {
        match self.tokens.iter().idx(self.tokens.len() - 1 - self.offset) {
            Some(token) => token,
            None => fail!("Attempted to access Lexer::current() on when tokens is empty")
        }
    }

    pub fn backtrack(&mut self) {
        self.offset += 1;
    }

    pub fn valid(&self) -> bool {
        self.offset > 0 || match self.tokens.back() { None => true, Some(x) => x.token != EOF }
    }

    fn peek(&mut self) -> Option<char> {
        match self.input.peek() {
            Some(ch) => Some(*ch),
            None => None
        }
    }

    fn read_char(&mut self) -> Option<char> {
        match self.input.next() {
            Some(c) => {
                self.previousLocation = self.location;
                self.location.absolute += 1;
                self.location.column += 1;
                if (c == '\n' || c == '\r')
                {
                    self.location.column = 0;
                    self.location.row += 1;
                }
                Some(c)
            }
            None => None
        }
    }

    fn scan_digits(&mut self) -> ~str {
        let mut result = ~"";
        loop {
            match self.peek() {
                Some(x) => {
                    if !x.is_digit() {
                        return result;
                    }
                    self.read_char();
                    result.push_char(x)
                }
                None => break
            }
        }
        result
    }

    fn scan_number(&mut self, c : char, location : Location) -> Token {
        let mut number = c.to_str() + self.scan_digits();
        let mut token = NUMBER;
        match self.peek() {
            Some('.') => {
                token = FLOAT;
                number.push_char('.');
                number = number.append(self.scan_digits());
            }
            _ => ()
        }
        Token { token : token, value : number, location : location }
    }

    fn new_token<'a>(&'a mut self, parseError : |&Token| -> bool) -> &'a Token {
        let mut newline = false;
        let n = self.next_indent_token(&mut newline);
        self.unprocessedTokens.push(n);
        let newTok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].token;

        if newTok != LBRACE {
            match self.tokens.back() {
                Some(tok) => {
                    if tok.token == LET || tok.token == WHERE || tok.token == OF {
                        let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;
                        let indentstart = Token { token : INDENTSTART, value : ~"{n}", location : loc };
                        self.unprocessedTokens.push(indentstart);
                    }
                }
                None => ()
            }
        }
        if newline {
            let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;
            self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", loc));
        }
        self.layout_independent_token(parseError);
        self.tokens.back().unwrap()
    }

    fn layout_independent_token(&mut self, parseError : |&Token| -> bool) {
        if self.unprocessedTokens.len() > 0 {
            let tok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].clone();//TODO dont use clone
            match tok.token {
                INDENTLEVEL => {
                    if (self.indentLevels.len() > 0) {
                        //m:ms
                        let m = self.indentLevels[self.indentLevels.len() - 1];
                        //m == n
                        if (m == tok.location.column) {
                            debug!("Indents are same, inserted semicolon");
                            self.tokens.push_back(Token::new(SEMICOLON, ~";", tok.location));
                            self.unprocessedTokens.pop();
                            return;
                        }
                        else if (tok.location.column < m)// n < m
                        {
                            //TODO
                            debug!("n < m, insert \\}");
                            self.indentLevels.pop();
                            self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                            return;
                        }
                    }
                    self.unprocessedTokens.pop();
                    if (self.unprocessedTokens.len() == 0) {
                        self.new_token(parseError);
                        return;
                    }
                    else {
                        return self.layout_independent_token(parseError);
                    }
                }
                INDENTSTART => {
                    //{n} token
                    let n = tok.location.column;
                    if (self.indentLevels.len() != 0) {
                        //m:ms
                        let m = self.indentLevels[self.indentLevels.len() - 1];
                        if (n > m) {
                            debug!("n > m + INDENTSTART, insert \\{");
                            self.unprocessedTokens.pop();
                            self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                            self.indentLevels.push(n);
                            return;
                        }
                    }
                    if (n > 0)
                    {
                        self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                        self.unprocessedTokens.pop();
                        self.indentLevels.push(n);
                        return;
                    }
                    self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                    self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                    self.unprocessedTokens.pop();
                    self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", tok.location));
                    self.offset += 1;
                    return;
                }
                RBRACE => {
                    if (self.indentLevels.len() > 0 && self.indentLevels[self.indentLevels.len() - 1] == 0) {
                        self.tokens.push_back(self.unprocessedTokens.pop());
                        self.indentLevels.pop();
                        return;
                    }
                    else {
                        return;//parse-error
                    }
                }
                LBRACE => {
                    self.tokens.push_back(self.unprocessedTokens.pop());
                    self.indentLevels.push(0);
                    return;
                }

                _ => ()
            }
            if (self.indentLevels.len() != 0) {
                //L (t:ts) (m:ms) 	= 	} : (L (t:ts) ms) 	if m /= 0 and parse-error(t)
                let m = self.indentLevels[self.indentLevels.len() - 1];
                if (m != 0 && parseError(&tok))
                {
                    debug!("ParseError on token {:?}, inserting \\}", tok.token);
                    self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                    self.indentLevels.pop();
                    return;
                }
            }
            self.tokens.push_back(self.unprocessedTokens.pop());
            return;
        }
        else {
            if (self.indentLevels.len() == 0)//End of stream
            {
                return;
            }
            else if (self.indentLevels[self.indentLevels.len() - 1] != 0)//Keep pusing righ brackets
            {
                self.indentLevels.pop();
                self.tokens.push_back(Token::new_(RBRACE, ~"}"));
                return;
            }
        }
    }

    fn next_indent_token(&mut self, newline : &mut bool) -> Token {
        let mut c = ' ';
        //Skip all whitespace before the token
        while c.is_whitespace() {
            match self.read_char() {
                Some(x) => {
                    c = x;
                    if (self.location.column == 0)//newline detected
                    {
                        *newline = true;
                    }
                }
                None => { return Token::eof() }
            }
        }
        let startLocation = self.location;

        //Decide how to tokenize depending on what the first char is
        //ie if its an operator then more operators will follow
        if (is_operator(c))
        {
            let mut result = c.to_str();
            loop {
                match self.peek() {
                    Some(ch) => {
                        if !is_operator(ch) {
                            break;
                        }
                        self.read_char();
                        result.push_char(ch);
                    }
                    None => { break; }
                }
            }
            let tok = match result {
                ~"="  => EQUALSSIGN,
                ~"->" => ARROW,
                ~"::" => TYPEDECL,
                _    => OPERATOR
            };
            return Token { token : tok, value : result, location : startLocation };
        }
        else if (c.is_digit())
        {
            return self.scan_number(c, startLocation);
        }
        else if (c.is_alphabetic() || c == '_')
        {
            let mut result = c.to_str();
            loop {
                match self.peek() {
                    Some(ch) => {
                        if !ch.is_alphanumeric() && ch != '_' {
                            break;
                        }
                        self.read_char();
                        result.push_char(ch);
                    }
                    None => return Token::eof()
                }
            }
            return Token {
                token : name_or_keyword(result),
                location : startLocation,
                value : result};
        }
        let tok = match c {
            ';' => SEMICOLON,
            '(' => LPARENS,
            ')' => RPARENS,
            '[' => LBRACKET,
            ']' => RBRACKET,
            '{' => LBRACE,
            '}' => RBRACE,
            ',' => COMMA,
            _   => EOF
        };
        Token { token : tok, location : startLocation, value : c.to_str() }
    }
}

#[test]
fn simple() {
    let mut lexer = Lexer::new("test 2 + 3".chars());

    assert_eq!(*lexer.next_(), Token::new_(NAME, ~"test"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"2"));
    assert_eq!(*lexer.next_(), Token::new_(OPERATOR, ~"+"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"3"));
}
#[test]
fn let_bind() {
    let mut lexer = Lexer::new(
r"let
    test = 2 + 3
in test".chars());

    assert_eq!(*lexer.next_(), Token::new_(LET, ~"let"));
    assert_eq!(*lexer.next_(), Token::new_(LBRACE, ~"{"));
    assert_eq!(*lexer.next_(), Token::new_(NAME, ~"test"));
    assert_eq!(*lexer.next_(), Token::new_(EQUALSSIGN, ~"="));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"2"));
    assert_eq!(*lexer.next_(), Token::new_(OPERATOR, ~"+"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"3"));
}
<commit_msg>Fixed issue where the lexer returned EOF instead of the identifier<commit_after>extern mod extra;
use extra::container::Deque;
use extra::ringbuf::RingBuf;
use std::iter::Peekable;
#[deriving(Clone, Eq, ToStr)]
pub enum TokenEnum {
	EOF,
	NAME,
	OPERATOR,
	NUMBER,
	FLOAT,
	LPARENS,
	RPARENS,
	LBRACKET,
	RBRACKET,
	LBRACE,
	RBRACE,
	INDENTSTART,
	INDENTLEVEL,
	COMMA,
	EQUALSSIGN,
	SEMICOLON,
	MODULE,
	CLASS,
	INSTANCE,
	WHERE,
	LET,
	IN,
	CASE,
	OF,
	ARROW,
	TYPEDECL,
	DATA
}

#[deriving(Clone, Eq, ToStr)]
pub struct Location {
    column : int,
    row : int,
    absolute : int
}

#[deriving(Clone)]
pub struct Token {
    token : TokenEnum,
    value : ~str,
    location : Location
}
impl Token {
    fn eof() -> Token {
        Token { token : EOF, value : ~"", location : Location { column : -1, row : -1, absolute : -1} }
    }

    fn new(token : TokenEnum, value : ~str, loc : Location) -> Token {
        Token { token : token, value : value, location : loc }
    }
    fn new_(token : TokenEnum, value : ~str) -> Token {
        Token::new(token, value, Location { column : -1, row : -1, absolute : -1 })
    }
}

impl Eq for Token {
    fn eq(&self, rhs : &Token) -> bool {
        self.token == rhs.token && self.value == rhs.value
    }
}

fn name_or_keyword(tok : &str) -> TokenEnum {
    match tok {
        "module" => MODULE,
        "class" => CLASS,
        "instance" => INSTANCE,
        "where" => WHERE,
        "let" => LET,
        "in" => IN,
        "case" => CASE,
        "of" => OF,
        "->" => ARROW,
        "data" => DATA,
        _ => NAME
    }
}

fn is_operator(first_char : char) -> bool {
    match first_char {
        '+' => true,
        '-' => true,
        '*' => true,
        '/' => true,
        '.' => true,
        '$' => true,
        ':' => true,
        '=' => true,
        '<' => true,
        '>' => true,
        '|' => true,
        '&' => true,
        _ => false
    }
}

pub struct Lexer<Stream> {
    priv input : Peekable<char, Stream>,
    priv location : Location,
    priv previousLocation : Location,
    priv unprocessedTokens : ~[Token],
    priv tokens : extra::ringbuf::RingBuf<Token>,
    priv indentLevels : ~[int],
    priv offset : uint
}

impl <Stream : Iterator<char>> Lexer<Stream> {

    pub fn new(input : Stream) -> Lexer<Stream> {
        let start = Location { column : 0, row : 0, absolute : 0};
        Lexer {
            input : input.peekable(),
            location : start,
            previousLocation : start,
            unprocessedTokens : ~[],
            tokens : extra::ringbuf::RingBuf::with_capacity(20),
            indentLevels : ~[],
            offset : 0}
    }
    pub fn module_next<'a>(&'a mut self) -> &'a Token {
        let mut newline = false;
        let n = self.next_indent_token(&mut newline);
        self.unprocessedTokens.push(n);
        let newTok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].token;
        let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;

        if newTok != LBRACE && newTok != MODULE {
            self.unprocessedTokens.push(Token::new(INDENTSTART, ~"{n}", loc));
        }
        if newline {
            self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", loc));
        }

        self.layout_independent_token(|_| false);
        self.current()
    }

    pub fn next_<'a>(&'a mut self) -> &'a Token {
        self.next(|_| false)
    }
    pub fn next<'a>(&'a mut self, parseError : |&Token| -> bool) -> &'a Token {
        if self.offset > 0 {
            self.offset -= 1;
            match self.tokens.iter().idx(self.tokens.len() - 1 - self.offset) {
                Some(token) => token,
                None => fail!("Impossible empty tokens stream")
            }
        }
        else if self.unprocessedTokens.len() > 0 {
            self.layout_independent_token(parseError);
            self.tokens.back().unwrap()
        }
        else {
            self.new_token(parseError)
        }
    }

    pub fn current<'a>(&'a self) -> &'a Token {
        match self.tokens.iter().idx(self.tokens.len() - 1 - self.offset) {
            Some(token) => token,
            None => fail!("Attempted to access Lexer::current() on when tokens is empty")
        }
    }

    pub fn backtrack(&mut self) {
        self.offset += 1;
    }

    pub fn valid(&self) -> bool {
        self.offset > 0 || match self.tokens.back() { None => true, Some(x) => x.token != EOF }
    }

    fn peek(&mut self) -> Option<char> {
        match self.input.peek() {
            Some(ch) => Some(*ch),
            None => None
        }
    }

    fn read_char(&mut self) -> Option<char> {
        match self.input.next() {
            Some(c) => {
                self.previousLocation = self.location;
                self.location.absolute += 1;
                self.location.column += 1;
                if (c == '\n' || c == '\r')
                {
                    self.location.column = 0;
                    self.location.row += 1;
                }
                Some(c)
            }
            None => None
        }
    }

    fn scan_digits(&mut self) -> ~str {
        let mut result = ~"";
        loop {
            match self.peek() {
                Some(x) => {
                    if !x.is_digit() {
                        return result;
                    }
                    self.read_char();
                    result.push_char(x)
                }
                None => break
            }
        }
        result
    }

    fn scan_number(&mut self, c : char, location : Location) -> Token {
        let mut number = c.to_str() + self.scan_digits();
        let mut token = NUMBER;
        match self.peek() {
            Some('.') => {
                token = FLOAT;
                number.push_char('.');
                number = number.append(self.scan_digits());
            }
            _ => ()
        }
        Token { token : token, value : number, location : location }
    }

    fn new_token<'a>(&'a mut self, parseError : |&Token| -> bool) -> &'a Token {
        let mut newline = false;
        let n = self.next_indent_token(&mut newline);
        println!("{:?}", n);
        self.unprocessedTokens.push(n);
        let newTok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].token;

        if newTok != LBRACE {
            match self.tokens.back() {
                Some(tok) => {
                    if tok.token == LET || tok.token == WHERE || tok.token == OF {
                        let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;
                        let indentstart = Token { token : INDENTSTART, value : ~"{n}", location : loc };
                        self.unprocessedTokens.push(indentstart);
                    }
                }
                None => ()
            }
        }
        if newline {
            let loc = self.unprocessedTokens[self.unprocessedTokens.len() - 1].location;
            self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", loc));
        }
        self.layout_independent_token(parseError);
        self.tokens.back().unwrap()
    }

    fn layout_independent_token(&mut self, parseError : |&Token| -> bool) {
        if self.unprocessedTokens.len() > 0 {
            let tok = self.unprocessedTokens[self.unprocessedTokens.len() - 1].clone();//TODO dont use clone
            match tok.token {
                INDENTLEVEL => {
                    if (self.indentLevels.len() > 0) {
                        //m:ms
                        let m = self.indentLevels[self.indentLevels.len() - 1];
                        //m == n
                        if (m == tok.location.column) {
                            debug!("Indents are same, inserted semicolon");
                            self.tokens.push_back(Token::new(SEMICOLON, ~";", tok.location));
                            self.unprocessedTokens.pop();
                            return;
                        }
                        else if (tok.location.column < m)// n < m
                        {
                            //TODO
                            debug!("n < m, insert \\}");
                            self.indentLevels.pop();
                            self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                            return;
                        }
                    }
                    self.unprocessedTokens.pop();
                    if (self.unprocessedTokens.len() == 0) {
                        self.new_token(parseError);
                        return;
                    }
                    else {
                        return self.layout_independent_token(parseError);
                    }
                }
                INDENTSTART => {
                    //{n} token
                    let n = tok.location.column;
                    if (self.indentLevels.len() != 0) {
                        //m:ms
                        let m = self.indentLevels[self.indentLevels.len() - 1];
                        if (n > m) {
                            debug!("n > m + INDENTSTART, insert \\{");
                            self.unprocessedTokens.pop();
                            self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                            self.indentLevels.push(n);
                            return;
                        }
                    }
                    if (n > 0)
                    {
                        self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                        self.unprocessedTokens.pop();
                        self.indentLevels.push(n);
                        return;
                    }
                    self.tokens.push_back(Token::new(LBRACE, ~"{", tok.location));
                    self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                    self.unprocessedTokens.pop();
                    self.unprocessedTokens.push(Token::new(INDENTLEVEL, ~"<n>", tok.location));
                    self.offset += 1;
                    return;
                }
                RBRACE => {
                    if (self.indentLevels.len() > 0 && self.indentLevels[self.indentLevels.len() - 1] == 0) {
                        self.tokens.push_back(self.unprocessedTokens.pop());
                        self.indentLevels.pop();
                        return;
                    }
                    else {
                        return;//parse-error
                    }
                }
                LBRACE => {
                    self.tokens.push_back(self.unprocessedTokens.pop());
                    self.indentLevels.push(0);
                    return;
                }

                _ => ()
            }
            if (self.indentLevels.len() != 0) {
                //L (t:ts) (m:ms) 	= 	} : (L (t:ts) ms) 	if m /= 0 and parse-error(t)
                let m = self.indentLevels[self.indentLevels.len() - 1];
                if (m != 0 && parseError(&tok))
                {
                    debug!("ParseError on token {:?}, inserting \\}", tok.token);
                    self.tokens.push_back(Token::new(RBRACE, ~"}", tok.location));
                    self.indentLevels.pop();
                    return;
                }
            }
            self.tokens.push_back(self.unprocessedTokens.pop());
            return;
        }
        else {
            if (self.indentLevels.len() == 0)//End of stream
            {
                return;
            }
            else if (self.indentLevels[self.indentLevels.len() - 1] != 0)//Keep pusing righ brackets
            {
                self.indentLevels.pop();
                self.tokens.push_back(Token::new_(RBRACE, ~"}"));
                return;
            }
        }
    }

    fn next_indent_token(&mut self, newline : &mut bool) -> Token {
        let mut c = ' ';
        //Skip all whitespace before the token
        while c.is_whitespace() {
            match self.read_char() {
                Some(x) => {
                    c = x;
                    if (self.location.column == 0)//newline detected
                    {
                        *newline = true;
                    }
                }
                None => { return Token::eof() }
            }
        }
        let startLocation = self.location;

        //Decide how to tokenize depending on what the first char is
        //ie if its an operator then more operators will follow
        if (is_operator(c))
        {
            let mut result = c.to_str();
            loop {
                match self.peek() {
                    Some(ch) => {
                        if !is_operator(ch) {
                            break;
                        }
                        self.read_char();
                        result.push_char(ch);
                    }
                    None => { break; }
                }
            }
            let tok = match result {
                ~"="  => EQUALSSIGN,
                ~"->" => ARROW,
                ~"::" => TYPEDECL,
                _    => OPERATOR
            };
            return Token { token : tok, value : result, location : startLocation };
        }
        else if (c.is_digit())
        {
            return self.scan_number(c, startLocation);
        }
        else if (c.is_alphabetic() || c == '_')
        {
            let mut result = c.to_str();
            loop {
                match self.peek() {
                    Some(ch) => {
                        if !ch.is_alphanumeric() && ch != '_' {
                            break;
                        }
                        self.read_char();
                        result.push_char(ch);
                    }
                    None => break
                }
            }
            return Token {
                token : name_or_keyword(result),
                location : startLocation,
                value : result};
        }
        let tok = match c {
            ';' => SEMICOLON,
            '(' => LPARENS,
            ')' => RPARENS,
            '[' => LBRACKET,
            ']' => RBRACKET,
            '{' => LBRACE,
            '}' => RBRACE,
            ',' => COMMA,
            _   => EOF
        };
        Token { token : tok, location : startLocation, value : c.to_str() }
    }
}

#[test]
fn simple() {
    let mut lexer = Lexer::new("test 2 + 3".chars());

    assert_eq!(*lexer.next_(), Token::new_(NAME, ~"test"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"2"));
    assert_eq!(*lexer.next_(), Token::new_(OPERATOR, ~"+"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"3"));
}
#[test]
fn let_bind() {
    let mut lexer = Lexer::new(
r"let
    test = 2 + 3
in test".chars());

    assert_eq!(*lexer.next_(), Token::new_(LET, ~"let"));
    assert_eq!(*lexer.next_(), Token::new_(LBRACE, ~"{"));
    assert_eq!(*lexer.next_(), Token::new_(NAME, ~"test"));
    assert_eq!(*lexer.next_(), Token::new_(EQUALSSIGN, ~"="));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"2"));
    assert_eq!(*lexer.next_(), Token::new_(OPERATOR, ~"+"));
    assert_eq!(*lexer.next_(), Token::new_(NUMBER, ~"3"));
}
<|endoftext|>


<commit_before>#[warn(unused_must_use)];

use std::io::IoResult;
use std::io::net::ip::SocketAddr;
use std::io::net::tcp::{TcpListener,TcpStream};
use std::io::{Listener,Acceptor,Writer};
use std::io::BufferedStream;
use std::task;

fn read_ascii(io: &mut BufferedStream<TcpStream>) -> char {
    match io.read_byte() {
        Ok(byte) if byte < 127 => byte as char,
        _ => fail!("Invalid ASCII character")
    }
}

fn read_line(io: &mut BufferedStream<TcpStream>) -> ~str {
    match io.read_line() {
        Ok(mut line) => {
            assert!(line.pop_char() == '\n');
            assert!(line.pop_char() == '\r');
            line
        }
        _ => fail!("Invalid line")
    }
}

fn read_expect(io: &mut BufferedStream<TcpStream>, expect: &[u8]) -> bool {
    for &byte in expect.iter() {
        if read_ascii(io) != byte as char {
            return false
        }
    }
    return true;
}

enum Command {
    HELO(~str),
    EHLO(~str),
    MAIL_FROM(~str),
    RCPT_TO(~str),
    DATA,
    QUIT,
    Invalid
}

fn read_command(io: &mut BufferedStream<TcpStream>) -> Command {
    match read_ascii(io) {
        'H' => {
            if read_expect(io, bytes!("ELO ")) { HELO(read_line(io)) }
            else { Invalid }
        }
        'E' => {
            if read_expect(io, bytes!("HLO ")) { EHLO(read_line(io)) }
            else { Invalid }
        }
        'M' => {
            if read_expect(io, bytes!("AIL FROM:")) { MAIL_FROM(read_line(io)) }
            else { Invalid }
        }
        'R' => {
            if read_expect(io, bytes!("CPT TO:")) { RCPT_TO(read_line(io)) }
            else { Invalid }
        }
        'D' => {
            if read_expect(io, bytes!("ATA\r\n")) { DATA }
            else { Invalid }
        }
        'Q' => {
            if read_expect(io, bytes!("UIT\r\n")) { QUIT }
            else { Invalid }
        }
        _ => {
            Invalid
        }
    }
}

fn handle_connection(conn: TcpStream) {
    debug!("Got connection");

    let mut io = BufferedStream::new(conn);
    let server_hostname = "mail.ntecs.de";
    let server_agent = "rust-smtp";

    write!(&mut io, "220 {} ESMTP {}\r\n", server_hostname, server_agent);
    io.flush();

    let mut client_hostname = ~"";

    match read_command(&mut io) {
        EHLO(h) => client_hostname = h,
        HELO(h) => client_hostname = h,
        _ => fail!("Expected EHLO or HELO")
    }

    println!("Client hostname: {}", client_hostname);

    write!(&mut io, "250 Hello {}\r\n", client_hostname);
    io.flush();

    loop {
        let cmd = read_command(&mut io);
        match cmd {
            MAIL_FROM(mailfrom) => {
                println!("FROM: {}", mailfrom);
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            RCPT_TO(mailto) => {
                println!("TO: {}", mailto);
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            DATA => {
                println!("DATA");
                io.write("354 End data with <CR><LF>.<CR><LF>\r\n".as_bytes()); io.flush();
                loop {
                    let line = read_line(&mut io);
                    println!("Data|{}|", line);
                    if line.as_slice() == "." {
                        println!("Got end");
                        break;
                    }
                }
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            QUIT => {
                println!("QUIT");
                io.write("221 Bye\r\n".as_bytes()); io.flush();
                break;
            }
            _ => {
                fail!()
            }
        }
    }

    // XXX make sure to close the connection
    debug!("End handling connection");
}

fn main() {
    let addr: SocketAddr = from_str("127.0.0.1:2525").unwrap();

    match TcpListener::bind(addr) {
        Ok(listener) => {
            match listener.listen() {
                Ok(ref mut acceptor) => {
                    loop {
                        match acceptor.accept() {
                            Ok(conn) => {
                                task::spawn(proc() handle_connection(conn))
                            }
                            _ => { fail!() }
                        }
                    }
                }
                _ => { fail!() }
            }
        }
        _ => { fail!() }
    }

}
<commit_msg>Refactor<commit_after>#[warn(unused_must_use)];

use std::io::{IoResult,IoError,InvalidInput};
use std::io::net::ip::SocketAddr;
use std::io::net::tcp::{TcpListener,TcpStream};
use std::io::{Listener,Acceptor,Writer};
use std::io::BufferedStream;
use std::task;

fn read_ascii(io: &mut BufferedStream<TcpStream>) -> char {
    match io.read_byte() {
        Ok(byte) if byte < 127 => byte as char,
        _ => fail!("Invalid ASCII character")
    }
}

fn invalid_input(desc: &'static str) -> IoError {
    IoError {kind: InvalidInput, desc: desc, detail: None}
}

// RFC 5321 Section 2.3.8. Lines
static CR: u8 = 0x0D;
static LF: u8 = 0x0A;
fn read_line(io: &mut BufferedStream<TcpStream>) -> IoResult<~str> {
    let mut s = ~"";

    loop {
        match if_ok!(io.read_byte()) {
            CR   => { break }
            LF   => { return Err(invalid_input("CR expected before LF")) }
            byte => { s.push_char(byte as char); }
        }
    }

    if if_ok!(io.read_byte()) == LF {
        Ok(s)
    } else {
        Err(invalid_input("LF expected after CR"))
    }
}

fn read_expect(io: &mut BufferedStream<TcpStream>, expect: &[u8]) -> bool {
    for &byte in expect.iter() {
        if read_ascii(io) != byte as char {
            return false
        }
    }
    return true;
}

enum Command {
    HELO(~str),
    EHLO(~str),
    MAIL_FROM(~str),
    RCPT_TO(~str),
    DATA,
    QUIT,
    Invalid
}

fn read_command(io: &mut BufferedStream<TcpStream>) -> Command {
    match read_ascii(io) {
        'H' => {
            if read_expect(io, bytes!("ELO ")) { HELO(read_line(io).unwrap()) }
            else { Invalid }
        }
        'E' => {
            if read_expect(io, bytes!("HLO ")) { EHLO(read_line(io).unwrap()) }
            else { Invalid }
        }
        'M' => {
            if read_expect(io, bytes!("AIL FROM:")) { MAIL_FROM(read_line(io).unwrap()) }
            else { Invalid }
        }
        'R' => {
            if read_expect(io, bytes!("CPT TO:")) { RCPT_TO(read_line(io).unwrap()) }
            else { Invalid }
        }
        'D' => {
            if read_expect(io, bytes!("ATA\r\n")) { DATA }
            else { Invalid }
        }
        'Q' => {
            if read_expect(io, bytes!("UIT\r\n")) { QUIT }
            else { Invalid }
        }
        _ => {
            Invalid
        }
    }
}

fn handle_connection(conn: TcpStream) {
    debug!("Got connection");

    let mut io = BufferedStream::new(conn);
    let server_hostname = "mail.ntecs.de";
    let server_agent = "rust-smtp";

    write!(&mut io, "220 {} ESMTP {}\r\n", server_hostname, server_agent);
    io.flush();

    let mut client_hostname = ~"";

    match read_command(&mut io) {
        EHLO(h) => client_hostname = h,
        HELO(h) => client_hostname = h,
        _ => fail!("Expected EHLO or HELO")
    }

    println!("Client hostname: {}", client_hostname);

    write!(&mut io, "250 Hello {}\r\n", client_hostname);
    io.flush();

    loop {
        let cmd = read_command(&mut io);
        match cmd {
            MAIL_FROM(mailfrom) => {
                println!("FROM: {}", mailfrom);
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            RCPT_TO(mailto) => {
                println!("TO: {}", mailto);
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            DATA => {
                println!("DATA");
                io.write("354 End data with <CR><LF>.<CR><LF>\r\n".as_bytes()); io.flush();
                loop {
                    let line = read_line(&mut io).unwrap();
                    println!("Data|{}|", line);
                    if line.as_slice() == "." {
                        println!("Got end");
                        break;
                    }
                }
                io.write("250 Ok\r\n".as_bytes()); io.flush();
            }
            QUIT => {
                println!("QUIT");
                io.write("221 Bye\r\n".as_bytes()); io.flush();
                break;
            }
            _ => {
                fail!()
            }
        }
    }

    // XXX make sure to close the connection
    debug!("End handling connection");
}

fn main() {
    let addr: SocketAddr = from_str("127.0.0.1:2525").unwrap();

    match TcpListener::bind(addr) {
        Ok(listener) => {
            match listener.listen() {
                Ok(ref mut acceptor) => {
                    loop {
                        match acceptor.accept() {
                            Ok(conn) => {
                                task::spawn(proc() handle_connection(conn))
                            }
                            _ => { fail!() }
                        }
                    }
                }
                _ => { fail!() }
            }
        }
        _ => { fail!() }
    }

}
<|endoftext|>


<commit_before>// Based off of libcss's examples/example1.c
mod example1 {

    use CssResult;
    use wapcaplet::ll::lwc_string;
    use ll::properties::*;
    use types::*;
    use hint::*;
    use select::*;
    use values::*;
    use util::VoidPtrLike;
    use wapcaplet::LwcString;

    struct MyDomNode {
        name: @LwcString
    }

    impl MyDomNode: VoidPtrLike {
        static fn from_void_ptr(node: *libc::c_void) -> MyDomNode {
            assert node.is_not_null();
            MyDomNode {
                name: unsafe {
                    let box = cast::reinterpret_cast(&node);
                    cast::bump_box_refcount(box);
                    box
                }
            }
        }

        fn to_void_ptr(&self) -> *libc::c_void {
            unsafe { cast::reinterpret_cast(&self.name) }
        }
    }

    #[test]
    fn run() {
        let data = "h1 { color: red; }\
                    h4 { color: #321; }\
                    h4, h5 { color: #123456; }";

        let params: CssStylesheetParams = CssStylesheetParams {
            params_version: CssStylesheetParamsVersion1,
            level: CssLevel21,
            charset: ~"UTF-8",
            url: ~"foo",
            title: ~"foo",
            allow_quirks: false,
            inline_style: false,
            resolve: Some(resolve_url),
            import: None,
            color: None,
            font: None,
        };

        let mut sheet: CssStylesheet = css_stylesheet_create(&params);
        debug!("stylesheet: %?", sheet);
        debug!("stylesheet size: %?", sheet.size());

        sheet.append_data(str::to_bytes(data));
        sheet.data_done();
        debug!("stylesheet size: %?", sheet.size());

        let mut select_ctx: CssSelectCtx = css_select_ctx_create();
        assert select_ctx.count_sheets() == 0;
        select_ctx.append_sheet(move sheet, CSS_ORIGIN_AUTHOR, CSS_MEDIA_ALL);
        debug!("count sheets: %?", select_ctx.count_sheets());
        assert select_ctx.count_sheets() == 1;

        for uint::range(1, 7) |hh| {
            let element = fmt!("h%u", hh);
            let element_name: @LwcString = @from_rust_string(element);
            let node = MyDomNode { name: element_name };
            let select_handler = SelectHandler { bogus: () };
            let style: CssSelectResults = select_ctx.select_style(&node,
                                                                  CSS_MEDIA_SCREEN,
                                                                  None,
                                                                  &select_handler);

            let computed: CssComputedStyle = style.computed_style(CssPseudoElementNone);

            match computed.color() {
                CssColorInherit => {
                    debug!("color of h%u is 'inherit'", hh);
                },
                CssColorColor(color) => {
                    debug!("color of h%u is %x", hh, color.to_ll() as uint);
                }
            }
        }
    }

    fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
        fail ~"resolving url";
    }

    struct SelectHandler {
        bogus: ()
    }

    impl SelectHandler: CssSelectHandler<MyDomNode> {
        fn node_name(node: &MyDomNode) -> CssQName {
            debug!("HL node_name!");
            debug!("SS %?", (*node.name).to_str());

            CssQName {
                ns: None,
                name: node.name.clone()
            }
        }

        fn node_id(_node: &MyDomNode) -> Option<LwcString> { None }

        fn named_parent_node(_node: &MyDomNode, _qname: &CssQName) -> Option<MyDomNode> {
            None
        }

        fn parent_node(_node: &MyDomNode) -> Option<MyDomNode> {
            None
        }

        fn node_has_id(_node: &MyDomNode, _name: LwcString) -> bool { false }

        fn named_ancestor_node(_node: &MyDomNode, _qname: &CssQName) -> Option<MyDomNode> {
            None
        }

        fn node_is_root(_node: &MyDomNode) -> bool { false }

        fn node_is_link(_node: &MyDomNode) -> bool { false }

        fn ua_default_for_property(property: CssProperty) -> CssHint {
            match property {
                _ => CssHintDefault
            }
        }
    }
}

#[test]
fn test_arc() {
    use std::arc::ARC;

    let params: CssStylesheetParams = CssStylesheetParams {
        params_version: CssStylesheetParamsVersion1,
        level: CssLevel21,
        charset: ~"UTF-8",
        url: ~"foo",
        title: ~"foo",
        allow_quirks: false,
        inline_style: false,
        resolve: Some(resolve_url),
        import: None,
        color: None,
        font: None,
    };

    let sheet: CssStylesheet = css_stylesheet_create(&params);
    let _arc = ARC(move sheet);

    fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
        fail ~"resolving url";
    }
}
<commit_msg>Fix tests<commit_after>// Based off of libcss's examples/example1.c
mod example1 {

    use CssResult;
    use wapcaplet::ll::lwc_string;
    use ll::properties::*;
    use ll::types::*;
    use types::*;
    use hint::*;
    use select::*;
    use values::*;
    use stylesheet::*;
    use util::VoidPtrLike;
    use wapcaplet::{LwcString, from_rust_string};

    struct MyDomNode {
        name: @LwcString
    }

    impl MyDomNode: VoidPtrLike {
        static fn from_void_ptr(node: *libc::c_void) -> MyDomNode {
            assert node.is_not_null();
            MyDomNode {
                name: unsafe {
                    let box = cast::reinterpret_cast(&node);
                    cast::bump_box_refcount(box);
                    box
                }
            }
        }

        fn to_void_ptr(&self) -> *libc::c_void {
            unsafe { cast::reinterpret_cast(&self.name) }
        }
    }

    #[test]
    fn run() {
        let data = "h1 { color: red; }\
                    h4 { color: #321; }\
                    h4, h5 { color: #123456; }";

        let resolve: CssUrlResolutionFn = |a,b| resolve_url(a, b);

        let params: CssStylesheetParams = CssStylesheetParams {
            params_version: CssStylesheetParamsVersion1,
            level: CssLevel21,
            charset: ~"UTF-8",
            url: ~"foo",
            title: ~"foo",
            allow_quirks: false,
            inline_style: false,
            resolve: Some(resolve),
            import: None,
            color: None,
            font: None,
        };

        let mut sheet: CssStylesheet = css_stylesheet_create(&params);
        debug!("stylesheet: %?", sheet);
        debug!("stylesheet size: %?", sheet.size());

        sheet.append_data(str::to_bytes(data));
        sheet.data_done();
        debug!("stylesheet size: %?", sheet.size());

        let mut select_ctx: CssSelectCtx = css_select_ctx_create();
        assert select_ctx.count_sheets() == 0;
        select_ctx.append_sheet(move sheet, CSS_ORIGIN_AUTHOR, CSS_MEDIA_ALL);
        debug!("count sheets: %?", select_ctx.count_sheets());
        assert select_ctx.count_sheets() == 1;

        for uint::range(1, 7) |hh| {
            let element = fmt!("h%u", hh);
            let element_name: @LwcString = @from_rust_string(element);
            let node = MyDomNode { name: element_name };
            let select_handler = SelectHandler { bogus: () };
            let style: CssSelectResults = select_ctx.select_style(&node,
                                                                  CSS_MEDIA_SCREEN,
                                                                  None,
                                                                  &select_handler);

            let computed: CssComputedStyle = style.computed_style(CssPseudoElementNone);

            match computed.color() {
                CssColorInherit => {
                    debug!("color of h%u is 'inherit'", hh);
                },
                CssColorColor(color) => {
                    debug!("color of h%u is %x", hh, color.to_ll() as uint);
                }
            }
        }
    }

    fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
        fail ~"resolving url";
    }

    struct SelectHandler {
        bogus: ()
    }

    impl SelectHandler: CssSelectHandler<MyDomNode> {
        fn node_name(node: &MyDomNode) -> CssQName {
            debug!("HL node_name!");
            debug!("SS %?", (*node.name).to_str());

            CssQName {
                ns: None,
                name: node.name.clone()
            }
        }

        fn node_id(_node: &MyDomNode) -> Option<LwcString> { None }

        fn named_parent_node(_node: &MyDomNode, _qname: &CssQName) -> Option<MyDomNode> {
            None
        }

        fn parent_node(_node: &MyDomNode) -> Option<MyDomNode> {
            None
        }

        fn node_has_id(_node: &MyDomNode, _name: LwcString) -> bool { false }

        fn named_ancestor_node(_node: &MyDomNode, _qname: &CssQName) -> Option<MyDomNode> {
            None
        }

        fn node_is_root(_node: &MyDomNode) -> bool { false }

        fn node_is_link(_node: &MyDomNode) -> bool { false }

        fn ua_default_for_property(property: CssProperty) -> CssHint {
            match property {
                _ => CssHintDefault
            }
        }
    }
}

#[test]
fn test_arc() {
    use std::arc::ARC;
    use stylesheet::*;
    use types::CssLevel21;

    let resolve: CssUrlResolutionFn = |a,b| resolve_url(a, b);
    let params: CssStylesheetParams = CssStylesheetParams {
        params_version: CssStylesheetParamsVersion1,
        level: CssLevel21,
        charset: ~"UTF-8",
        url: ~"foo",
        title: ~"foo",
        allow_quirks: false,
        inline_style: false,
        resolve: Some(resolve),
        import: None,
        color: None,
        font: None,
    };

    let sheet: CssStylesheet = css_stylesheet_create(&params);
    let _arc = ARC(move sheet);

    fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
        fail ~"resolving url";
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(plugin_registrar, rustc_private)]

extern crate syntax;
extern crate rustc_plugin;

use syntax::ast::{MetaItem, Expr};
use syntax::ast;
use syntax::codemap::Span;
use syntax::ext::base::{ExtCtxt, Annotatable};
use syntax::ext::build::AstBuilder;
use syntax::ext::deriving::generic::*;
use syntax::ext::deriving::generic::ty::*;
use syntax::parse::token::InternedString;
use syntax::ptr::P;
use syntax::ext::base::MultiDecorator;
use syntax::parse::token;

use rustc_plugin::Registry;

macro_rules! pathvec {
    ($($x:ident)::+) => (
        vec![ $( stringify!($x) ),+ ]
    )
}

macro_rules! path {
    ($($x:tt)*) => (
        ::syntax::ext::deriving::generic::ty::Path::new( pathvec!( $($x)* ) )
    )
}

macro_rules! path_local {
    ($x:ident) => (
        ::syntax::ext::deriving::generic::ty::Path::new_local(stringify!($x))
    )
}

macro_rules! pathvec_std {
    ($cx:expr, $first:ident :: $($rest:ident)::+) => ({
        let mut v = pathvec!($($rest)::+);
        if let Some(s) = $cx.crate_root {
            v.insert(0, s);
        }
        v
    })
}

pub fn expand_deriving_from_primitive(cx: &mut ExtCtxt,
                                      span: Span,
                                      mitem: &MetaItem,
                                      item: &Annotatable,
                                      push: &mut FnMut(Annotatable))
{
    let inline = cx.meta_word(span, InternedString::new("inline"));
    let attrs = vec!(cx.attribute(span, inline));
    let trait_def = TraitDef {
        is_unsafe: false,
        span: span,
        attributes: Vec::new(),
        path: path!(num::FromPrimitive),
        additional_bounds: Vec::new(),
        generics: LifetimeBounds::empty(),
        methods: vec!(
            MethodDef {
                name: "from_i64",
                is_unsafe: false,
                generics: LifetimeBounds::empty(),
                explicit_self: None,
                args: vec!(Literal(path_local!(i64))),
                ret_ty: Literal(Path::new_(pathvec_std!(cx, core::option::Option),
                                           None,
                                           vec!(Box::new(Self_)),
                                           true)),
                // #[inline] liable to cause code-bloat
                attributes: attrs.clone(),
                combine_substructure: combine_substructure(Box::new(|c, s, sub| {
                    cs_from("i64", c, s, sub)
                })),
            },
            MethodDef {
                name: "from_u64",
                is_unsafe: false,
                generics: LifetimeBounds::empty(),
                explicit_self: None,
                args: vec!(Literal(path_local!(u64))),
                ret_ty: Literal(Path::new_(pathvec_std!(cx, core::option::Option),
                                           None,
                                           vec!(Box::new(Self_)),
                                           true)),
                // #[inline] liable to cause code-bloat
                attributes: attrs,
                combine_substructure: combine_substructure(Box::new(|c, s, sub| {
                    cs_from("u64", c, s, sub)
                })),
            }
        ),
        associated_types: Vec::new(),
    };

    trait_def.expand(cx, mitem, &item, push)
}

fn cs_from(name: &str, cx: &mut ExtCtxt, trait_span: Span, substr: &Substructure) -> P<Expr> {
    if substr.nonself_args.len() != 1 {
        cx.span_bug(trait_span, "incorrect number of arguments in `derive(FromPrimitive)`")
    }

    let n = &substr.nonself_args[0];

    match *substr.fields {
        StaticStruct(..) => {
            cx.span_err(trait_span, "`FromPrimitive` cannot be derived for structs");
            return cx.expr_fail(trait_span, InternedString::new(""));
        }
        StaticEnum(enum_def, _) => {
            if enum_def.variants.is_empty() {
                cx.span_err(trait_span,
                            "`FromPrimitive` cannot be derived for enums with no variants");
                return cx.expr_fail(trait_span, InternedString::new(""));
            }

            let mut arms = Vec::new();

            for variant in &enum_def.variants {
                match variant.node.data {
                    ast::VariantData::Unit(..) => {
                        let span = variant.span;

                        // expr for `$n == $variant as $name`
                        let path = cx.path(span, vec![substr.type_ident, variant.node.name]);
                        let variant = cx.expr_path(path);
                        let ty = cx.ty_ident(span, cx.ident_of(name));
                        let cast = cx.expr_cast(span, variant.clone(), ty);
                        let guard = cx.expr_binary(span, ast::BiEq, n.clone(), cast);

                        // expr for `Some($variant)`
                        let body = cx.expr_some(span, variant);

                        // arm for `_ if $guard => $body`
                        let arm = ast::Arm {
                            attrs: vec!(),
                            pats: vec!(cx.pat_wild(span)),
                            guard: Some(guard),
                            body: body,
                        };

                        arms.push(arm);
                    }
                    ast::VariantData::Tuple(..) => {
                        cx.span_err(trait_span,
                                    "`FromPrimitive` cannot be derived for \
                                    enum variants with arguments");
                        return cx.expr_fail(trait_span,
                                            InternedString::new(""));
                    }
                    ast::VariantData::Struct(..) => {
                        cx.span_err(trait_span,
                                    "`FromPrimitive` cannot be derived for enums \
                                    with struct variants");
                        return cx.expr_fail(trait_span,
                                            InternedString::new(""));
                    }
                }
            }

            // arm for `_ => None`
            let arm = ast::Arm {
                attrs: vec!(),
                pats: vec!(cx.pat_wild(trait_span)),
                guard: None,
                body: cx.expr_none(trait_span),
            };
            arms.push(arm);

            cx.expr_match(trait_span, n.clone(), arms)
        }
        _ => cx.span_bug(trait_span, "expected StaticEnum in derive(FromPrimitive)")
    }
}

#[plugin_registrar]
#[doc(hidden)]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_syntax_extension(
        token::intern("derive_NumFromPrimitive"),
        MultiDecorator(Box::new(expand_deriving_from_primitive)));
}
<commit_msg>Auto merge of #144 - bluss:syntax-ext, r=cuviper<commit_after>// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(plugin_registrar, rustc_private)]

extern crate syntax;
extern crate syntax_ext;
extern crate rustc_plugin;

use syntax::ast::{MetaItem, Expr};
use syntax::ast;
use syntax::codemap::Span;
use syntax::ext::base::{ExtCtxt, Annotatable};
use syntax::ext::build::AstBuilder;
use syntax_ext::deriving::generic::*;
use syntax_ext::deriving::generic::ty::*;
use syntax::parse::token::InternedString;
use syntax::ptr::P;
use syntax::ext::base::MultiDecorator;
use syntax::parse::token;

use rustc_plugin::Registry;

macro_rules! pathvec {
    ($($x:ident)::+) => (
        vec![ $( stringify!($x) ),+ ]
    )
}

macro_rules! path {
    ($($x:tt)*) => (
        ::syntax_ext::deriving::generic::ty::Path::new( pathvec!( $($x)* ) )
    )
}

macro_rules! path_local {
    ($x:ident) => (
        ::syntax_ext::deriving::generic::ty::Path::new_local(stringify!($x))
    )
}

macro_rules! pathvec_std {
    ($cx:expr, $first:ident :: $($rest:ident)::+) => ({
        let mut v = pathvec!($($rest)::+);
        if let Some(s) = $cx.crate_root {
            v.insert(0, s);
        }
        v
    })
}

pub fn expand_deriving_from_primitive(cx: &mut ExtCtxt,
                                      span: Span,
                                      mitem: &MetaItem,
                                      item: &Annotatable,
                                      push: &mut FnMut(Annotatable))
{
    let inline = cx.meta_word(span, InternedString::new("inline"));
    let attrs = vec!(cx.attribute(span, inline));
    let trait_def = TraitDef {
        is_unsafe: false,
        span: span,
        attributes: Vec::new(),
        path: path!(num::FromPrimitive),
        additional_bounds: Vec::new(),
        generics: LifetimeBounds::empty(),
        methods: vec!(
            MethodDef {
                name: "from_i64",
                is_unsafe: false,
                generics: LifetimeBounds::empty(),
                explicit_self: None,
                args: vec!(Literal(path_local!(i64))),
                ret_ty: Literal(Path::new_(pathvec_std!(cx, core::option::Option),
                                           None,
                                           vec!(Box::new(Self_)),
                                           true)),
                // #[inline] liable to cause code-bloat
                attributes: attrs.clone(),
                combine_substructure: combine_substructure(Box::new(|c, s, sub| {
                    cs_from("i64", c, s, sub)
                })),
            },
            MethodDef {
                name: "from_u64",
                is_unsafe: false,
                generics: LifetimeBounds::empty(),
                explicit_self: None,
                args: vec!(Literal(path_local!(u64))),
                ret_ty: Literal(Path::new_(pathvec_std!(cx, core::option::Option),
                                           None,
                                           vec!(Box::new(Self_)),
                                           true)),
                // #[inline] liable to cause code-bloat
                attributes: attrs,
                combine_substructure: combine_substructure(Box::new(|c, s, sub| {
                    cs_from("u64", c, s, sub)
                })),
            }
        ),
        associated_types: Vec::new(),
    };

    trait_def.expand(cx, mitem, &item, push)
}

fn cs_from(name: &str, cx: &mut ExtCtxt, trait_span: Span, substr: &Substructure) -> P<Expr> {
    if substr.nonself_args.len() != 1 {
        cx.span_bug(trait_span, "incorrect number of arguments in `derive(FromPrimitive)`")
    }

    let n = &substr.nonself_args[0];

    match *substr.fields {
        StaticStruct(..) => {
            cx.span_err(trait_span, "`FromPrimitive` cannot be derived for structs");
            return cx.expr_fail(trait_span, InternedString::new(""));
        }
        StaticEnum(enum_def, _) => {
            if enum_def.variants.is_empty() {
                cx.span_err(trait_span,
                            "`FromPrimitive` cannot be derived for enums with no variants");
                return cx.expr_fail(trait_span, InternedString::new(""));
            }

            let mut arms = Vec::new();

            for variant in &enum_def.variants {
                match variant.node.data {
                    ast::VariantData::Unit(..) => {
                        let span = variant.span;

                        // expr for `$n == $variant as $name`
                        let path = cx.path(span, vec![substr.type_ident, variant.node.name]);
                        let variant = cx.expr_path(path);
                        let ty = cx.ty_ident(span, cx.ident_of(name));
                        let cast = cx.expr_cast(span, variant.clone(), ty);
                        let guard = cx.expr_binary(span, ast::BiEq, n.clone(), cast);

                        // expr for `Some($variant)`
                        let body = cx.expr_some(span, variant);

                        // arm for `_ if $guard => $body`
                        let arm = ast::Arm {
                            attrs: vec!(),
                            pats: vec!(cx.pat_wild(span)),
                            guard: Some(guard),
                            body: body,
                        };

                        arms.push(arm);
                    }
                    ast::VariantData::Tuple(..) => {
                        cx.span_err(trait_span,
                                    "`FromPrimitive` cannot be derived for \
                                    enum variants with arguments");
                        return cx.expr_fail(trait_span,
                                            InternedString::new(""));
                    }
                    ast::VariantData::Struct(..) => {
                        cx.span_err(trait_span,
                                    "`FromPrimitive` cannot be derived for enums \
                                    with struct variants");
                        return cx.expr_fail(trait_span,
                                            InternedString::new(""));
                    }
                }
            }

            // arm for `_ => None`
            let arm = ast::Arm {
                attrs: vec!(),
                pats: vec!(cx.pat_wild(trait_span)),
                guard: None,
                body: cx.expr_none(trait_span),
            };
            arms.push(arm);

            cx.expr_match(trait_span, n.clone(), arms)
        }
        _ => cx.span_bug(trait_span, "expected StaticEnum in derive(FromPrimitive)")
    }
}

#[plugin_registrar]
#[doc(hidden)]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_syntax_extension(
        token::intern("derive_NumFromPrimitive"),
        MultiDecorator(Box::new(expand_deriving_from_primitive)));
}
<|endoftext|>


<commit_before>use parser::Span;
use parser::Span::Text;

mod br;
mod code;
mod emphasis;
mod strong;
mod link;
mod image;
use self::br::parse_break;
use self::code::parse_code;
use self::emphasis::parse_emphasis;
use self::strong::parse_strong;
use self::link::parse_link;
use self::image::parse_image;

pub fn parse_spans(text: &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut t = String::new();
    let mut i = 0;
    while i < text.len() {
        match parse_span(&text[i .. text.len()]){
            Some((span, consumed_chars)) => {
                if !t.is_empty(){
                    // if this text is on the very left
                    // trim the left whitespace
                    if tokens.is_empty() {
                        t = t.trim_left().to_owned()
                    }
                    tokens.push(Text(t));
                }
                tokens.push(span);
                t = String::new();
                i += consumed_chars;
            }
            None => {
                t.push_str(&text[i..i + 1]);
                i += 1;
            }
        }
    }
    if !t.is_empty(){
        // if this text is on the very left
        // trim the left whitespace
        if tokens.is_empty() {
            t = t.trim_left().to_owned();
        }
        // we're at the very end of this line,
        // trim trailing whitespace
        t = t.trim_right().to_owned();
        tokens.push(Text(t));
    }
    tokens
}

fn parse_span(text: &str) -> Option<(Span, usize)>{
    pipe_opt!(
        text
        => parse_code
        => parse_strong
        => parse_emphasis
        => parse_break
        => parse_image
        => parse_link
        )
}

#[cfg(test)]
mod test {
    use parser::Span::{Text, Break, Code, Emphasis, Strong, Link, Image};
    use parser::span::parse_spans;

    #[test]
    fn converts_into_text() {
        assert_eq!(parse_spans("this is a test"), vec![Text("this is a test".to_owned())]);
    }

    #[test]
    fn finds_breaks() {
        assert_eq!(parse_spans("this is a test  "), vec![Text("this is a test".to_owned()), Break]);
    }

    #[test]
    fn finds_code() {
        assert_eq!(parse_spans("this `is a` test"), vec![Text("this ".to_owned()), Code("is a".to_owned()), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this ``is a`` test"), vec![Text("this ".to_owned()), Code("is a".to_owned()), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_emphasis() {
        assert_eq!(parse_spans("this _is a_ test"), vec![Text("this ".to_owned()), Emphasis(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this *is a* test"), vec![Text("this ".to_owned()), Emphasis(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_strong() {
        assert_eq!(parse_spans("this __is a__ test"), vec![Text("this ".to_owned()), Strong(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this **is a** test"), vec![Text("this ".to_owned()), Strong(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_link() {
        assert_eq!(
            parse_spans("this is [an example](example.com) test"),
            vec![Text("this is ".to_owned()), Link("an example".to_owned(), "example.com".to_owned(), None), Text(" test".to_owned())]
            );
    }

    #[test]
    fn finds_image() {
        assert_eq!(
            parse_spans("this is ![an example](example.com) test"),
            vec![Text("this is ".to_owned()), Image("an example".to_owned(), "example.com".to_owned(), None), Text(" test".to_owned())]
            );
    }

    #[test]
    fn finds_everything() {
        assert_eq!(
            parse_spans("some text ![an image](image.com) _emphasis_ __strong__ `teh codez` [a link](example.com)  "),
            vec![
            Text("some text ".to_owned()),
            Image("an image".to_owned(), "image.com".to_owned(), None),
            Text(" ".to_owned()),
            Emphasis(vec![Text("emphasis".to_owned())]),
            Text(" ".to_owned()),
            Strong(vec![Text("strong".to_owned())]),
            Text(" ".to_owned()),
            Code("teh codez".to_owned()),
            Text(" ".to_owned()),
            Link("a link".to_owned(), "example.com".to_owned(), None),
            Break
            ]
            );
    }
}

<commit_msg>Fix panic when parsing multi-byte UTF-8 (#20)<commit_after>use parser::Span;
use parser::Span::Text;

mod br;
mod code;
mod emphasis;
mod strong;
mod link;
mod image;
use self::br::parse_break;
use self::code::parse_code;
use self::emphasis::parse_emphasis;
use self::strong::parse_strong;
use self::link::parse_link;
use self::image::parse_image;

pub fn parse_spans(text: &str) -> Vec<Span>{
    let mut tokens = vec![];
    let mut t = String::new();
    let mut i = 0;
    while i < text.len() {
        match parse_span(&text[i .. text.len()]){
            Some((span, consumed_chars)) => {
                if !t.is_empty(){
                    // if this text is on the very left
                    // trim the left whitespace
                    if tokens.is_empty() {
                        t = t.trim_left().to_owned()
                    }
                    tokens.push(Text(t));
                }
                tokens.push(span);
                t = String::new();
                i += consumed_chars;
            }
            None => {
                let mut e = i + 1;
                while !text.is_char_boundary(e) {
                    e += 1;
                }

                t.push_str(&text[i..e]);
                i += e - i;
            }
        }
    }
    if !t.is_empty(){
        // if this text is on the very left
        // trim the left whitespace
        if tokens.is_empty() {
            t = t.trim_left().to_owned();
        }
        // we're at the very end of this line,
        // trim trailing whitespace
        t = t.trim_right().to_owned();
        tokens.push(Text(t));
    }
    tokens
}

fn parse_span(text: &str) -> Option<(Span, usize)>{
    pipe_opt!(
        text
        => parse_code
        => parse_strong
        => parse_emphasis
        => parse_break
        => parse_image
        => parse_link
        )
}

#[cfg(test)]
mod test {
    use parser::Span::{Text, Break, Code, Emphasis, Strong, Link, Image};
    use parser::span::parse_spans;
    use std::str;

    #[test]
    fn converts_into_text() {
        assert_eq!(parse_spans("this is a test"), vec![Text("this is a test".to_owned())]);
    }

    #[test]
    fn finds_breaks() {
        assert_eq!(parse_spans("this is a test  "), vec![Text("this is a test".to_owned()), Break]);
    }

    #[test]
    fn finds_code() {
        assert_eq!(parse_spans("this `is a` test"), vec![Text("this ".to_owned()), Code("is a".to_owned()), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this ``is a`` test"), vec![Text("this ".to_owned()), Code("is a".to_owned()), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_emphasis() {
        assert_eq!(parse_spans("this _is a_ test"), vec![Text("this ".to_owned()), Emphasis(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this *is a* test"), vec![Text("this ".to_owned()), Emphasis(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_strong() {
        assert_eq!(parse_spans("this __is a__ test"), vec![Text("this ".to_owned()), Strong(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
        assert_eq!(parse_spans("this **is a** test"), vec![Text("this ".to_owned()), Strong(vec![Text("is a".to_owned())]), Text(" test".to_owned())]);
    }

    #[test]
    fn finds_link() {
        assert_eq!(
            parse_spans("this is [an example](example.com) test"),
            vec![Text("this is ".to_owned()), Link("an example".to_owned(), "example.com".to_owned(), None), Text(" test".to_owned())]
            );
    }

    #[test]
    fn finds_image() {
        assert_eq!(
            parse_spans("this is ![an example](example.com) test"),
            vec![Text("this is ".to_owned()), Image("an example".to_owned(), "example.com".to_owned(), None), Text(" test".to_owned())]
            );
    }

    #[test]
    fn finds_everything() {
        assert_eq!(
            parse_spans("some text ![an image](image.com) _emphasis_ __strong__ `teh codez` [a link](example.com)  "),
            vec![
            Text("some text ".to_owned()),
            Image("an image".to_owned(), "image.com".to_owned(), None),
            Text(" ".to_owned()),
            Emphasis(vec![Text("emphasis".to_owned())]),
            Text(" ".to_owned()),
            Strong(vec![Text("strong".to_owned())]),
            Text(" ".to_owned()),
            Code("teh codez".to_owned()),
            Text(" ".to_owned()),
            Link("a link".to_owned(), "example.com".to_owned(), None),
            Break
            ]
            );
    }

    #[test]
    fn properly_consumes_multibyte_utf8() {
        let test_phrase = str::from_utf8(b"This shouldn\xE2\x80\x99t panic").unwrap();
        let _ = parse_spans(&test_phrase);
    }
}

<|endoftext|>


<commit_before><commit_msg>Benchmark CStrBuf::from_c_str{,_unchecked}<commit_after><|endoftext|>


<commit_before><commit_msg>Implement Error trait<commit_after><|endoftext|>


<commit_before>//! Various helpers for code-generation.

use std::io;
use std::cmp;
use std::char;
use std::iter;
use std::usize;
use std::ops::{Add, AddAssign, Range};
#[cfg(test)]
use std::iter::FromIterator;

use super::{Atom, Class, ClassMember};

// ================================================================
/// Trait for atom types that can be converted to byte sequences.
pub trait Bytes<'a> {
    /// Iterator type returned by `bytes`.
    type Iter: 'a + Iterator<Item=u8>;

    /// Fetch an iterator over "self" as a series of bytes.
    fn bytes(&self) -> Self::Iter;
}

/// Interface for types which can bound the number of bytes required to match
/// a particular instance of themselves.
pub trait SizedRead {
    /// Get the size of this object when read directly from something
    /// implementing `std::io::Read`.
    fn read_size(&self) -> ReadSize;
}

/// Trait for use with fixed-size types
pub trait ReadToBuffer {
    /// Read at most `max_count` instances of this atom type from the given
    /// stream, placing them into the supplied buffer.
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read;
}

/// Possible results from `SizedRead::read_size`
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum ReadSize {
    /// Exact size is known in advance
    Exact(usize),

    /// Size can be within the given range of values.
    Range(usize, usize)
}

impl ReadSize {
    /// Get the minimum number of bytes specified by this ReadSize.
    #[inline]
    fn min(&self) -> usize {
        match self {
            &ReadSize::Exact(s) => s,
            &ReadSize::Range(a, _) => a,
        }
    }
    /// Get the maximum number of bytes specified by this ReadSize.
    #[inline]
    fn max(&self) -> usize {
        match self {
            &ReadSize::Exact(s) => s,
            &ReadSize::Range(_, b) => b,
        }
    }

}

impl iter::Sum<ReadSize> for ReadSize {
    #[inline]
    fn sum<I>(iter: I) -> Self where I: Iterator<Item=ReadSize> {
        let mut sum = ReadSize::Exact(0);
        for rs in iter {
            sum += rs;
        }
        sum
    }
}

impl Add<ReadSize> for ReadSize {
    type Output = ReadSize;
    #[inline]
    fn add(self, rhs: ReadSize) -> Self::Output {
        let min = self.min() + rhs.min();
        let max = self.max() + rhs.max();
        if min == max { ReadSize::Exact(min) }
        else { ReadSize::Range(min, max) }
    }
}

impl AddAssign<ReadSize> for ReadSize {
    #[inline]
    fn add_assign(&mut self, rhs: ReadSize) {
        *self = *self + rhs;
    }
}

impl From<usize> for ReadSize {
    #[inline]
    fn from(s: usize) -> Self {
        ReadSize::Exact(s)
    }
}

impl From<Range<usize>> for ReadSize {
    #[inline]
    fn from(r: Range<usize>) -> Self {
        if r.len() > 1 { ReadSize::Range(r.start, r.end - 1) }
        else { ReadSize::Exact(r.start) }
    }
}

// ----------------------------------------------------------------
// Implementations

impl<'a> Bytes<'a> for u8 {
    type Iter = iter::Once<u8>;
    fn bytes(&self) -> Self::Iter {
        iter::once(*self)
    }
}

impl SizedRead for u8 {
    #[inline(always)]
    fn read_size(&self) -> ReadSize {
        ReadSize::Exact(1)
    }
}

impl ReadToBuffer for u8 {
    #[inline(always)]
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read {
        r.read(&mut dest[0..max_count])
    }
}

// ----------------------------------------------------------------

// This table was copied from the Rust source tree,
// file "src/libcore/str/mod.rs".
// https://tools.ietf.org/html/rfc3629
static UTF8_CHAR_WIDTH: [u8; 256] = [
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
];

impl<'a> Bytes<'a> for char {
    type Iter = char::EncodeUtf8;
    fn bytes(&self) -> Self::Iter {
        self.encode_utf8()
    }
}

impl SizedRead for char {
    #[inline]
    fn read_size(&self) -> ReadSize {
        self.len_utf8().into()
    }
}

impl ReadToBuffer for char {
    #[inline]
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read {
        let mut i = 0;
        let mut n = 0;
        while i < dest.len() && n < max_count {
            try!(r.read_exact(&mut dest[i..(i+1)]));
            let w = UTF8_CHAR_WIDTH[dest[i] as usize];
            try!(r.read_exact(&mut dest[(i+1)..(i + w as usize)]));
            i += w as usize;
            n += 1;
        }
        Ok(i)
    }

}

#[cfg(test)]
#[test]
fn test_sizedread_char() {
    assert_eq!(ReadSize::Exact(1), 'a'.read_size());
    assert_eq!(ReadSize::Exact(3), '✓'.read_size());
}

#[cfg(test)]
#[test]
fn test_readable_char() {
    let mut buf = ['\0' as u8; 32];
    {
        let mut r = io::repeat('a' as u8);
        assert_eq!(3, char::read_to_buffer(&mut r, &mut buf, 3).unwrap());
        assert_eq!(&['a' as u8 as u8, 'a' as u8, 'a' as u8, '\0' as u8], &buf[0..4]);
    }

    {
        let s = "✓☠⚔";
        let mut r = io::Cursor::new(s.as_bytes());
        assert_eq!(9, char::read_to_buffer(&mut r, &mut buf, 3).unwrap());
        assert_eq!(s.as_bytes(), &buf[0..9]);
    }
}

// ----------------------------------------------------------------

/*impl<T> SizedRead for Transition<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        match self {
            &Transition::Atom(a) => a.read_size(),
            &Transition::Literal(ref atoms) => atoms.iter().map(|a| a.read_size()).sum(),
            &Transition::Wildcard => ReadSize::Range(1, 4),
            &Transition::Anchor(..) => ReadSize::Exact(0),
            &Transition::Class(ref c) => c.read_size(),
        }
    }
}*/

#[cfg(test)]
#[test]
fn test_sizedread_transition() {
    assert_eq!(ReadSize::Exact(1), 'a'.read_size());
    assert_eq!(ReadSize::Exact(3), '✓'.read_size());
    assert_eq!(ReadSize::Range(1, 3), Class::from_iter(['x', 'y', '⚔'].into_iter().cloned()).read_size());
}

// ----------------------------------------------------------------

impl<T> SizedRead for ClassMember<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        match self {
            &ClassMember::Atom(a) => a.read_size(),
            &ClassMember::Range(first, last) => {
                // We'll assume that any differences in read size between
                // `first` and `last` follow a linear function between the two;
                // this _assumption_ allows us to avoid evaluating `read_size`
                // for every single member of the range.
                let frs = first.read_size();
                let lrs = last.read_size();
                (cmp::min(frs.min(), lrs.min())..(cmp::max(frs.max(), lrs.max()) + 1)).into()
            }
        }
    }
}

impl<T> SizedRead for Class<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        let mut min = usize::MAX;
        let mut max = usize::MIN;

        for m in self.iter_members() {
            let rs = m.read_size();
            min = cmp::min(min, rs.min());
            max = cmp::max(max, rs.max());
        }

        (min..(max + 1)).into()
    }
}
<commit_msg>pattern/codegen: rename a test fn<commit_after>//! Various helpers for code-generation.

use std::io;
use std::cmp;
use std::char;
use std::iter;
use std::usize;
use std::ops::{Add, AddAssign, Range};
#[cfg(test)]
use std::iter::FromIterator;

use super::{Atom, Class, ClassMember};

// ================================================================
/// Trait for atom types that can be converted to byte sequences.
pub trait Bytes<'a> {
    /// Iterator type returned by `bytes`.
    type Iter: 'a + Iterator<Item=u8>;

    /// Fetch an iterator over "self" as a series of bytes.
    fn bytes(&self) -> Self::Iter;
}

/// Interface for types which can bound the number of bytes required to match
/// a particular instance of themselves.
pub trait SizedRead {
    /// Get the size of this object when read directly from something
    /// implementing `std::io::Read`.
    fn read_size(&self) -> ReadSize;
}

/// Trait for use with fixed-size types
pub trait ReadToBuffer {
    /// Read at most `max_count` instances of this atom type from the given
    /// stream, placing them into the supplied buffer.
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read;
}

/// Possible results from `SizedRead::read_size`
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum ReadSize {
    /// Exact size is known in advance
    Exact(usize),

    /// Size can be within the given range of values.
    Range(usize, usize)
}

impl ReadSize {
    /// Get the minimum number of bytes specified by this ReadSize.
    #[inline]
    fn min(&self) -> usize {
        match self {
            &ReadSize::Exact(s) => s,
            &ReadSize::Range(a, _) => a,
        }
    }
    /// Get the maximum number of bytes specified by this ReadSize.
    #[inline]
    fn max(&self) -> usize {
        match self {
            &ReadSize::Exact(s) => s,
            &ReadSize::Range(_, b) => b,
        }
    }

}

impl iter::Sum<ReadSize> for ReadSize {
    #[inline]
    fn sum<I>(iter: I) -> Self where I: Iterator<Item=ReadSize> {
        let mut sum = ReadSize::Exact(0);
        for rs in iter {
            sum += rs;
        }
        sum
    }
}

impl Add<ReadSize> for ReadSize {
    type Output = ReadSize;
    #[inline]
    fn add(self, rhs: ReadSize) -> Self::Output {
        let min = self.min() + rhs.min();
        let max = self.max() + rhs.max();
        if min == max { ReadSize::Exact(min) }
        else { ReadSize::Range(min, max) }
    }
}

impl AddAssign<ReadSize> for ReadSize {
    #[inline]
    fn add_assign(&mut self, rhs: ReadSize) {
        *self = *self + rhs;
    }
}

impl From<usize> for ReadSize {
    #[inline]
    fn from(s: usize) -> Self {
        ReadSize::Exact(s)
    }
}

impl From<Range<usize>> for ReadSize {
    #[inline]
    fn from(r: Range<usize>) -> Self {
        if r.len() > 1 { ReadSize::Range(r.start, r.end - 1) }
        else { ReadSize::Exact(r.start) }
    }
}

// ----------------------------------------------------------------
// Implementations

impl<'a> Bytes<'a> for u8 {
    type Iter = iter::Once<u8>;
    fn bytes(&self) -> Self::Iter {
        iter::once(*self)
    }
}

impl SizedRead for u8 {
    #[inline(always)]
    fn read_size(&self) -> ReadSize {
        ReadSize::Exact(1)
    }
}

impl ReadToBuffer for u8 {
    #[inline(always)]
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read {
        r.read(&mut dest[0..max_count])
    }
}

// ----------------------------------------------------------------

// This table was copied from the Rust source tree,
// file "src/libcore/str/mod.rs".
// https://tools.ietf.org/html/rfc3629
static UTF8_CHAR_WIDTH: [u8; 256] = [
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
];

impl<'a> Bytes<'a> for char {
    type Iter = char::EncodeUtf8;
    fn bytes(&self) -> Self::Iter {
        self.encode_utf8()
    }
}

impl SizedRead for char {
    #[inline]
    fn read_size(&self) -> ReadSize {
        self.len_utf8().into()
    }
}

impl ReadToBuffer for char {
    #[inline]
    fn read_to_buffer<R>(r: &mut R, dest: &mut [u8], max_count: usize) -> io::Result<usize>
        where R: io::Read {
        let mut i = 0;
        let mut n = 0;
        while i < dest.len() && n < max_count {
            try!(r.read_exact(&mut dest[i..(i+1)]));
            let w = UTF8_CHAR_WIDTH[dest[i] as usize];
            try!(r.read_exact(&mut dest[(i+1)..(i + w as usize)]));
            i += w as usize;
            n += 1;
        }
        Ok(i)
    }

}

#[cfg(test)]
#[test]
fn test_sizedread_char() {
    assert_eq!(ReadSize::Exact(1), 'a'.read_size());
    assert_eq!(ReadSize::Exact(3), '✓'.read_size());
}

#[cfg(test)]
#[test]
fn test_readable_char() {
    let mut buf = ['\0' as u8; 32];
    {
        let mut r = io::repeat('a' as u8);
        assert_eq!(3, char::read_to_buffer(&mut r, &mut buf, 3).unwrap());
        assert_eq!(&['a' as u8 as u8, 'a' as u8, 'a' as u8, '\0' as u8], &buf[0..4]);
    }

    {
        let s = "✓☠⚔";
        let mut r = io::Cursor::new(s.as_bytes());
        assert_eq!(9, char::read_to_buffer(&mut r, &mut buf, 3).unwrap());
        assert_eq!(s.as_bytes(), &buf[0..9]);
    }
}

// ----------------------------------------------------------------

/*impl<T> SizedRead for Transition<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        match self {
            &Transition::Atom(a) => a.read_size(),
            &Transition::Literal(ref atoms) => atoms.iter().map(|a| a.read_size()).sum(),
            &Transition::Wildcard => ReadSize::Range(1, 4),
            &Transition::Anchor(..) => ReadSize::Exact(0),
            &Transition::Class(ref c) => c.read_size(),
        }
    }
}*/

#[cfg(test)]
#[test]
fn test_sizedread() {
    assert_eq!(ReadSize::Exact(1), 'a'.read_size());
    assert_eq!(ReadSize::Exact(3), '✓'.read_size());
    assert_eq!(ReadSize::Range(1, 3), Class::from_iter(['x', 'y', '⚔'].into_iter().cloned()).read_size());
}

// ----------------------------------------------------------------

impl<T> SizedRead for ClassMember<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        match self {
            &ClassMember::Atom(a) => a.read_size(),
            &ClassMember::Range(first, last) => {
                // We'll assume that any differences in read size between
                // `first` and `last` follow a linear function between the two;
                // this _assumption_ allows us to avoid evaluating `read_size`
                // for every single member of the range.
                let frs = first.read_size();
                let lrs = last.read_size();
                (cmp::min(frs.min(), lrs.min())..(cmp::max(frs.max(), lrs.max()) + 1)).into()
            }
        }
    }
}

impl<T> SizedRead for Class<T>
    where T: Atom + SizedRead
{
    fn read_size(&self) -> ReadSize {
        let mut min = usize::MAX;
        let mut max = usize::MIN;

        for m in self.iter_members() {
            let rs = m.read_size();
            min = cmp::min(min, rs.min());
            max = cmp::max(max, rs.max());
        }

        (min..(max + 1)).into()
    }
}
<|endoftext|>


<commit_before>use utils;
use errors::*;
use temp;
use dist;
use component::{Components, TarGzPackage, Transaction, Package};

use std::path::{Path, PathBuf};
use std::env;

const REL_MANIFEST_DIR: &'static str = "lib/rustlib";

#[derive(Clone, Debug)]
pub struct InstallPrefix {
    path: PathBuf,
}
#[derive(Debug)]
pub enum InstallMethod<'a> {
    Copy(&'a Path),
    Link(&'a Path),
    Installer(&'a Path, &'a temp::Cfg),
    Dist(&'a str, Option<&'a Path>, dist::DownloadCfg<'a>),
}

impl<'a> InstallMethod<'a> {
    pub fn run(self, prefix: &InstallPrefix, notify_handler: NotifyHandler) -> Result<()> {
        if prefix.is_installed_here() {
            // Don't uninstall first for Dist method
            match self {
                InstallMethod::Dist(_, _, _) |
                InstallMethod::Installer(_, _) => {}
                _ => {
                    try!(prefix.uninstall(notify_handler));
                }
            }
        }

        match self {
            InstallMethod::Copy(src) => {
                try!(utils::copy_dir(src, &prefix.path, ntfy!(&notify_handler)));
                Ok(())
            }
            InstallMethod::Link(src) => {
                try!(utils::symlink_dir(src, &prefix.path, ntfy!(&notify_handler)));
                Ok(())
            }
            InstallMethod::Installer(src, temp_cfg) => {
                InstallMethod::tar_gz(src, prefix, &temp_cfg, notify_handler)
            }
            InstallMethod::Dist(toolchain, update_hash, dl_cfg) => {
                let maybe_new_hash =
                    try!(dist::update_from_dist(
                        dl_cfg,
                        update_hash,
                        toolchain,
                        prefix,
                        &[], &[]));

                if let Some(hash) = maybe_new_hash {
                    if let Some(hash_file) = update_hash {
                        try!(utils::write_file("update hash", hash_file, &hash));
                    }
                }

                Ok(())
            }
        }
    }

    fn tar_gz(src: &Path, prefix: &InstallPrefix, temp_cfg: &temp::Cfg,
              notify_handler: NotifyHandler) -> Result<()> {
        notify_handler.call(Notification::Extracting(src, prefix.path()));

        let installation = try!(Components::open(prefix.clone()));
        let package = try!(TarGzPackage::new_file(src, temp_cfg));

        let mut tx = Transaction::new(prefix.clone(), temp_cfg, notify_handler);

        for component in package.components() {
            tx = try!(package.install(&installation, &component, None, tx));
        }

        tx.commit();

        Ok(())
    }
}

pub fn bin_path(name: &str) -> PathBuf {
    let mut path = PathBuf::from("bin");
    path.push(name.to_owned() + env::consts::EXE_SUFFIX);
    path
}

impl InstallPrefix {
    pub fn from(path: PathBuf) -> Self {
        InstallPrefix {
            path: path,
        }
    }
    pub fn path(&self) -> &Path {
        &self.path
    }
    pub fn abs_path<P: AsRef<Path>>(&self, path: P) -> PathBuf {
        self.path.join(path)
    }
    pub fn manifest_dir(&self) -> PathBuf {
        let mut path = self.path.clone();
        path.push(REL_MANIFEST_DIR);
        path
    }
    pub fn manifest_file(&self, name: &str) -> PathBuf {
        let mut path = self.manifest_dir();
        path.push(name);
        path
    }
    pub fn rel_manifest_file(&self, name: &str) -> PathBuf {
        let mut path = PathBuf::from(REL_MANIFEST_DIR);
        path.push(name);
        path
    }
    pub fn binary_file(&self, name: &str) -> PathBuf {
        let mut path = self.path.clone();
        path.push(bin_path(name));
        path
    }
    pub fn is_installed_here(&self) -> bool {
        utils::is_directory(&self.path)
    }
    pub fn uninstall(&self, notify_handler: NotifyHandler) -> Result<()> {
        if self.is_installed_here() {
            Ok(try!(utils::remove_dir("install", &self.path, ntfy!(&notify_handler))))
        } else {
            Err(Error::NotInstalledHere)
        }
    }
    pub fn install(&self, method: InstallMethod, notify_handler: NotifyHandler) -> Result<()> {
        method.run(self, notify_handler)
    }
}
<commit_msg>Make InstallPrefix::is_installed_here private<commit_after>use utils;
use errors::*;
use temp;
use dist;
use component::{Components, TarGzPackage, Transaction, Package};

use std::path::{Path, PathBuf};
use std::env;

const REL_MANIFEST_DIR: &'static str = "lib/rustlib";

#[derive(Clone, Debug)]
pub struct InstallPrefix {
    path: PathBuf,
}
#[derive(Debug)]
pub enum InstallMethod<'a> {
    Copy(&'a Path),
    Link(&'a Path),
    Installer(&'a Path, &'a temp::Cfg),
    Dist(&'a str, Option<&'a Path>, dist::DownloadCfg<'a>),
}

impl<'a> InstallMethod<'a> {
    pub fn run(self, prefix: &InstallPrefix, notify_handler: NotifyHandler) -> Result<()> {
        if prefix.is_installed_here() {
            // Don't uninstall first for Dist method
            match self {
                InstallMethod::Dist(_, _, _) |
                InstallMethod::Installer(_, _) => {}
                _ => {
                    try!(prefix.uninstall(notify_handler));
                }
            }
        }

        match self {
            InstallMethod::Copy(src) => {
                try!(utils::copy_dir(src, &prefix.path, ntfy!(&notify_handler)));
                Ok(())
            }
            InstallMethod::Link(src) => {
                try!(utils::symlink_dir(src, &prefix.path, ntfy!(&notify_handler)));
                Ok(())
            }
            InstallMethod::Installer(src, temp_cfg) => {
                InstallMethod::tar_gz(src, prefix, &temp_cfg, notify_handler)
            }
            InstallMethod::Dist(toolchain, update_hash, dl_cfg) => {
                let maybe_new_hash =
                    try!(dist::update_from_dist(
                        dl_cfg,
                        update_hash,
                        toolchain,
                        prefix,
                        &[], &[]));

                if let Some(hash) = maybe_new_hash {
                    if let Some(hash_file) = update_hash {
                        try!(utils::write_file("update hash", hash_file, &hash));
                    }
                }

                Ok(())
            }
        }
    }

    fn tar_gz(src: &Path, prefix: &InstallPrefix, temp_cfg: &temp::Cfg,
              notify_handler: NotifyHandler) -> Result<()> {
        notify_handler.call(Notification::Extracting(src, prefix.path()));

        let installation = try!(Components::open(prefix.clone()));
        let package = try!(TarGzPackage::new_file(src, temp_cfg));

        let mut tx = Transaction::new(prefix.clone(), temp_cfg, notify_handler);

        for component in package.components() {
            tx = try!(package.install(&installation, &component, None, tx));
        }

        tx.commit();

        Ok(())
    }
}

pub fn bin_path(name: &str) -> PathBuf {
    let mut path = PathBuf::from("bin");
    path.push(name.to_owned() + env::consts::EXE_SUFFIX);
    path
}

impl InstallPrefix {
    pub fn from(path: PathBuf) -> Self {
        InstallPrefix {
            path: path,
        }
    }
    pub fn path(&self) -> &Path {
        &self.path
    }
    pub fn abs_path<P: AsRef<Path>>(&self, path: P) -> PathBuf {
        self.path.join(path)
    }
    pub fn manifest_dir(&self) -> PathBuf {
        let mut path = self.path.clone();
        path.push(REL_MANIFEST_DIR);
        path
    }
    pub fn manifest_file(&self, name: &str) -> PathBuf {
        let mut path = self.manifest_dir();
        path.push(name);
        path
    }
    pub fn rel_manifest_file(&self, name: &str) -> PathBuf {
        let mut path = PathBuf::from(REL_MANIFEST_DIR);
        path.push(name);
        path
    }
    pub fn binary_file(&self, name: &str) -> PathBuf {
        let mut path = self.path.clone();
        path.push(bin_path(name));
        path
    }
    fn is_installed_here(&self) -> bool {
        utils::is_directory(&self.path)
    }
    pub fn uninstall(&self, notify_handler: NotifyHandler) -> Result<()> {
        if self.is_installed_here() {
            Ok(try!(utils::remove_dir("install", &self.path, ntfy!(&notify_handler))))
        } else {
            Err(Error::NotInstalledHere)
        }
    }
    pub fn install(&self, method: InstallMethod, notify_handler: NotifyHandler) -> Result<()> {
        method.run(self, notify_handler)
    }
}
<|endoftext|>