<commit_before>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Eof,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    pub fn next(&mut self) -> Token {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return Token::new(self.position, self.position, TokenKind::Eof),
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Token::new(self.token_start, self.position, kind)
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => continue };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),
                                            ErrorLevel::Error,
                                            format!("invalid character escape: {}", c));
                            c
                        }
                    });
                }

                '"' => break,
                c => string.push(c),
            }
        }

        string
    }

    // fn report_error<S: Into<String>>(&self, message: S) {
    //     errors().report(Span::new(self.token_start, self.position), message.into());
    // }

    // fn peek(&self) -> Option<char> {
    //     self.current.and_then(|c| self.char_at(self.position + c.len_utf8()))
    // }

    fn char_at(&self, pos: usize) -> Option<char> {
        self.source[pos..].chars().next()
    }

    /// Move to the next char in the source code.
    fn advance(&mut self) {
        if let Some(c) = self.current {
            self.position += c.len_utf8();
            self.current = self.char_at(self.position);
        } else {
            panic!("lexer attempted to advance past the end of the source code");
        }
    }
}

// pub fn tokenize(source: &str) -> Vec<Token> {
//     let mut lexer = Lexer::new(source);
//     vec![]
// }

////////////////////////////////////////////////////////////////////////////////
// Parser
////////////////////////////////////////////////////////////////////////////////

// #[derive(Clone, Debug)]
// struct Parser<'src> {
//     lexer: Lexer<'src>,
//     errors: Vec<Error>,
// }

    // fn parse_module(&mut self) -> ParseResult<ast::Module> {
    //     let mut fns = Vec::new();
    //     while self.token != Token::Eof {
    //         fns.push(try!(self.parse_fn_def()));
    //     }
    //     Ok(ast::Module { fns: fns })
    // }

    // fn parse_fn_def(&mut self) -> ParseResult<ast::FnDef> {
    //     try!(self.expect(&Token::KeywordFn));

    //     let name = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     // TODO(tsion): Parse arguments.

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::BraceL));

    //     let mut body = Vec::new();

    //     while self.token != Token::BraceR {
    //         body.push(try!(self.parse_expr()));
    //     }

    //     // Skip the closing brace.
    //     self.advance();

    //     Ok(ast::FnDef {
    //         name: name,
    //         return_ty: ast::Type::Unit,
    //         args: Vec::new(),
    //         body: body,
    //     })
    // }

    // fn parse_expr(&mut self) -> ParseResult<ast::Expr> {
    //     let func = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     let str = match self.advance_and_get() {
    //         Token::StrLit(str) => str,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::Semicolon));

    //     Ok(ast::Expr::FnCall {
    //         func: func,
    //         args: vec![ast::Expr::StrLit(str)],
    //     })
    // }

////////////////////////////////////////////////////////////////////////////////
// Character classes
////////////////////////////////////////////////////////////////////////////////

/// Returns `true` if the given character is whitespace.
fn is_whitespace(c: char) -> bool {
    match c {
        ' ' | '\t' | '\r' | '\n' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is a digit.
fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid at the start of an identifier.
fn is_ident_start(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '_' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid after the start of an identifier.
fn is_ident_continue(c: char) -> bool {
    is_ident_start(c) || is_digit(c)
}

////////////////////////////////////////////////////////////////////////////////
// Errors
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error {
    pub message: String,
    pub level: ErrorLevel,
    pub span: Span,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ErrorLevel {
    Error,
    Warning,
    Note,
}

pub struct ErrorReporter {
    pub errors: RefCell<Vec<Error>>,
}

impl ErrorReporter {
    fn new() -> Self {
        ErrorReporter { errors: RefCell::new(Vec::new()) }
    }

    fn report<S: Into<String>>(&self, span: Span, level: ErrorLevel, message: S) {
        self.errors.borrow_mut().push(Error {
            message: message.into(),
            level: level,
            span: span,
        });
    }

    fn is_empty(&self) -> bool {
        self.errors.borrow().is_empty()
    }

    fn reset(&self) {
        self.errors.borrow_mut().clear();
    }
}

pub fn errors() -> Rc<ErrorReporter> {
    thread_local!(static KEY: Rc<ErrorReporter> = Rc::new(ErrorReporter::new()));
    KEY.with(|k| k.clone())
}

#[cfg(test)]
mod test {
    use super::{errors, ErrorLevel, Lexer, Span, TokenKind};
    use super::ErrorLevel::*;
    use super::TokenKind::*;

    #[test]
    fn lex_empty_string() {
        lexer_test(r#""""#, StrLit(String::from("")), &[]);
    }

    #[test]
    fn lex_unterminated_empty_string() {
        lexer_test(r#"""#, StrLit(String::from("")), &[
            (1, 1, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_empty_string() {
        lexer_test(r#""\"#, StrLit(String::from("")), &[
            (2, 2, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_string() {
        lexer_test(r#""foo\"#, StrLit(String::from("foo")), &[
            (5, 5, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string() {
        lexer_test(r#""foobar""#, StrLit(String::from("foobar")), &[]);
    }

    #[test]
    fn lex_unterminated_string() {
        lexer_test(r#""foobar"#, StrLit(String::from("foobar")), &[
            (7, 7, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_just_newline() {
        lexer_test(r#""\n""#, StrLit(String::from("\n")), &[]);
    }

    #[test]
    fn lex_unterminated_string_with_just_newline() {
        lexer_test(r#""\n"#, StrLit(String::from("\n")), &[
            (3, 3, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_newline() {
        lexer_test(r#""foo\nbar""#, StrLit(String::from("foo\nbar")), &[]);
    }

    #[test]
    fn lex_unterminated_string_with_newline() {
        lexer_test(r#""foo\nbar"#, StrLit(String::from("foo\nbar")), &[
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_invalid_escape_string() {
        lexer_test(r#""foo\cbar""#, StrLit(String::from("foocbar")), &[
            (4, 6, Error, "invalid character escape: c"),
        ]);
    }

    #[test]
    fn lex_unterminated_invalid_escape_string() {
        lexer_test(r#""foo\cbar"#, StrLit(String::from("foocbar")), &[
            (4, 6, Error, "invalid character escape: c"),
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    fn lexer_test(source: &str,
                  expected: TokenKind,
                  expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let mut lexer = Lexer::new(source);
        assert_eq!(lexer.next().kind, expected);
        assert_errors(expected_errors);
    }

    fn assert_errors(expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let reporter = errors();

        {
            let errors = reporter.errors.borrow();

            for (i, &(start, end, level, message)) in expected_errors.iter().enumerate() {
                assert_eq!(errors[i], super::Error {
                    message: String::from(message),
                    level: level,
                    span: Span::new(start, end),
                });
            }

            // Checking that there are no unexpected errors this way will pretty-print the
            // unexpected errors if they exist.
            assert_eq!(&errors[expected_errors.len()..], &[]);
        }

        reporter.reset();
    }
}
<commit_msg>Add heading for tests.<commit_after>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Eof,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    pub fn next(&mut self) -> Token {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return Token::new(self.position, self.position, TokenKind::Eof),
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Token::new(self.token_start, self.position, kind)
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => continue };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),
                                            ErrorLevel::Error,
                                            format!("invalid character escape: {}", c));
                            c
                        }
                    });
                }

                '"' => break,
                c => string.push(c),
            }
        }

        string
    }

    // fn report_error<S: Into<String>>(&self, message: S) {
    //     errors().report(Span::new(self.token_start, self.position), message.into());
    // }

    // fn peek(&self) -> Option<char> {
    //     self.current.and_then(|c| self.char_at(self.position + c.len_utf8()))
    // }

    fn char_at(&self, pos: usize) -> Option<char> {
        self.source[pos..].chars().next()
    }

    /// Move to the next char in the source code.
    fn advance(&mut self) {
        if let Some(c) = self.current {
            self.position += c.len_utf8();
            self.current = self.char_at(self.position);
        } else {
            panic!("lexer attempted to advance past the end of the source code");
        }
    }
}

// pub fn tokenize(source: &str) -> Vec<Token> {
//     let mut lexer = Lexer::new(source);
//     vec![]
// }

////////////////////////////////////////////////////////////////////////////////
// Parser
////////////////////////////////////////////////////////////////////////////////

// #[derive(Clone, Debug)]
// struct Parser<'src> {
//     lexer: Lexer<'src>,
//     errors: Vec<Error>,
// }

    // fn parse_module(&mut self) -> ParseResult<ast::Module> {
    //     let mut fns = Vec::new();
    //     while self.token != Token::Eof {
    //         fns.push(try!(self.parse_fn_def()));
    //     }
    //     Ok(ast::Module { fns: fns })
    // }

    // fn parse_fn_def(&mut self) -> ParseResult<ast::FnDef> {
    //     try!(self.expect(&Token::KeywordFn));

    //     let name = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     // TODO(tsion): Parse arguments.

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::BraceL));

    //     let mut body = Vec::new();

    //     while self.token != Token::BraceR {
    //         body.push(try!(self.parse_expr()));
    //     }

    //     // Skip the closing brace.
    //     self.advance();

    //     Ok(ast::FnDef {
    //         name: name,
    //         return_ty: ast::Type::Unit,
    //         args: Vec::new(),
    //         body: body,
    //     })
    // }

    // fn parse_expr(&mut self) -> ParseResult<ast::Expr> {
    //     let func = match self.advance_and_get() {
    //         Token::Ident(name) => name,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenL));

    //     let str = match self.advance_and_get() {
    //         Token::StrLit(str) => str,

    //         // TODO(tsion): Add an error to self.errors.
    //         _ => return Err(()),
    //     };

    //     try!(self.expect(&Token::ParenR));
    //     try!(self.expect(&Token::Semicolon));

    //     Ok(ast::Expr::FnCall {
    //         func: func,
    //         args: vec![ast::Expr::StrLit(str)],
    //     })
    // }

////////////////////////////////////////////////////////////////////////////////
// Character classes
////////////////////////////////////////////////////////////////////////////////

/// Returns `true` if the given character is whitespace.
fn is_whitespace(c: char) -> bool {
    match c {
        ' ' | '\t' | '\r' | '\n' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is a digit.
fn is_digit(c: char) -> bool {
    match c {
        '0'...'9' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid at the start of an identifier.
fn is_ident_start(c: char) -> bool {
    match c {
        'a'...'z' | 'A'...'Z' | '_' => true,
        _ => false,
    }
}

/// Returns `true` if the given character is valid after the start of an identifier.
fn is_ident_continue(c: char) -> bool {
    is_ident_start(c) || is_digit(c)
}

////////////////////////////////////////////////////////////////////////////////
// Errors
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error {
    pub message: String,
    pub level: ErrorLevel,
    pub span: Span,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ErrorLevel {
    Error,
    Warning,
    Note,
}

pub struct ErrorReporter {
    pub errors: RefCell<Vec<Error>>,
}

impl ErrorReporter {
    fn new() -> Self {
        ErrorReporter { errors: RefCell::new(Vec::new()) }
    }

    fn report<S: Into<String>>(&self, span: Span, level: ErrorLevel, message: S) {
        self.errors.borrow_mut().push(Error {
            message: message.into(),
            level: level,
            span: span,
        });
    }

    fn is_empty(&self) -> bool {
        self.errors.borrow().is_empty()
    }

    fn reset(&self) {
        self.errors.borrow_mut().clear();
    }
}

pub fn errors() -> Rc<ErrorReporter> {
    thread_local!(static KEY: Rc<ErrorReporter> = Rc::new(ErrorReporter::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////////////

#[cfg(test)]
mod test {
    use super::{errors, ErrorLevel, Lexer, Span, TokenKind};
    use super::ErrorLevel::*;
    use super::TokenKind::*;

    #[test]
    fn lex_empty_string() {
        lexer_test(r#""""#, StrLit(String::from("")), &[]);
    }

    #[test]
    fn lex_unterminated_empty_string() {
        lexer_test(r#"""#, StrLit(String::from("")), &[
            (1, 1, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_empty_string() {
        lexer_test(r#""\"#, StrLit(String::from("")), &[
            (2, 2, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_unterminated_escape_string() {
        lexer_test(r#""foo\"#, StrLit(String::from("foo")), &[
            (5, 5, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string() {
        lexer_test(r#""foobar""#, StrLit(String::from("foobar")), &[]);
    }

    #[test]
    fn lex_unterminated_string() {
        lexer_test(r#""foobar"#, StrLit(String::from("foobar")), &[
            (7, 7, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_just_newline() {
        lexer_test(r#""\n""#, StrLit(String::from("\n")), &[]);
    }

    #[test]
    fn lex_unterminated_string_with_just_newline() {
        lexer_test(r#""\n"#, StrLit(String::from("\n")), &[
            (3, 3, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_string_with_newline() {
        lexer_test(r#""foo\nbar""#, StrLit(String::from("foo\nbar")), &[]);
    }

    #[test]
    fn lex_unterminated_string_with_newline() {
        lexer_test(r#""foo\nbar"#, StrLit(String::from("foo\nbar")), &[
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    #[test]
    fn lex_invalid_escape_string() {
        lexer_test(r#""foo\cbar""#, StrLit(String::from("foocbar")), &[
            (4, 6, Error, "invalid character escape: c"),
        ]);
    }

    #[test]
    fn lex_unterminated_invalid_escape_string() {
        lexer_test(r#""foo\cbar"#, StrLit(String::from("foocbar")), &[
            (4, 6, Error, "invalid character escape: c"),
            (9, 9, Error, "unterminated string literal"),
            (0, 0, Note, "string literal began here"),
        ]);
    }

    fn lexer_test(source: &str,
                  expected: TokenKind,
                  expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let mut lexer = Lexer::new(source);
        assert_eq!(lexer.next().kind, expected);
        assert_errors(expected_errors);
    }

    fn assert_errors(expected_errors: &[(usize, usize, ErrorLevel, &str)]) {
        let reporter = errors();

        {
            let errors = reporter.errors.borrow();

            for (i, &(start, end, level, message)) in expected_errors.iter().enumerate() {
                assert_eq!(errors[i], super::Error {
                    message: String::from(message),
                    level: level,
                    span: Span::new(start, end),
                });
            }

            // Checking that there are no unexpected errors this way will pretty-print the
            // unexpected errors if they exist.
            assert_eq!(&errors[expected_errors.len()..], &[]);
        }

        reporter.reset();
    }
}
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        if function_parts.is_none() {
            Ok(Statement::Expression(try!(self.expr(&statement_tokens[..]))))
        }
        else {
            let (lhs, rhs) = function_parts.unwrap();

            if lhs[0] == Token::Backslash {
                return self.anonymous_function(lhs, rhs);
            }

            match lhs.len() {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[0..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[0..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        Ok(Expr::new())
        //if tokens[0] == Token::Backslash {
        //}
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> Option<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                break;
            }
        }

        // The 1 or many assumption being assumed in this
        // is only okay because of the i > 0 above
        debug_assert!(!equals.is_none() && equals.unwrap() > 0);

        equals.map(|index| (&tokens[..index], &tokens[(index+1)..]))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Partially reimplementing partition_function with some more granular errors<commit_after>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    UnexpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");

        let function_parts = self.partition_function(&statement_tokens);

        match function_parts {
            // no problem if we just expected equals, it is probably
            // an expr
            Err(ParseError::ExpectedToken(Token::Equals)) => {
                Ok(Statement::Expression(
                    try!(self.expr(&statement_tokens[..]))
                ))
            },
            Err(e) => return Err(e),
            Ok(parts) => {
                let (lhs, rhs) = parts;

                // partition_function is guarenteed to return non-empty slices
                debug_assert!(!lhs.is_empty() && !rhs.is_empty());

                if lhs[0] == Token::Backslash {
                    return self.anonymous_function(lhs, rhs);
                }

                match lhs.len() {
                    // only one argument: name = ...
                    1 => self.assignment(lhs, rhs),
                    // more than one argument: name arg1 ... = ...
                    _ => self.named_function(lhs, rhs),
                }
            },
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[0..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = lhs[0].clone().unwrap_symbol();

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[0..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> ParseResult<Function> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(|ref sym| (*sym).clone().unwrap_symbol()).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs.len() == 1);

        Ok(Statement::Assignment {
            name: lhs[0].clone().unwrap_symbol(),
            value: try!(self.expr(rhs)),
        })
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        Ok(Expr::new())
        //if tokens[0] == Token::Backslash {
        //    let (lhs, rhs) = try!(self.partition_function(tokens));
        //}
    }

    // partitions a function by its Equals token, returning None if no
    // valid function is found
    // Returns (lhs, rhs)
    fn partition_function<'a>(&self, tokens: &'a [Token]) -> ParseResult<(&'a [Token], &'a [Token])> {
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                return Err(ParseError::UnexpectedToken(token.clone()));
            }
        }

        if equals.is_none() {
            Err(ParseError::ExpectedToken(Token::Equals))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            Ok((&tokens[..equals], &tokens[(equals+1)..]))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>


<commit_before>use context::Context;
#[cfg(not(feature = "no_logging"))]
use error::RenderError;
use helpers::{HelperDef, HelperResult};
use output::Output;
use registry::Registry;
use render::{Helper, RenderContext};
#[cfg(not(feature = "no_logging"))]
use value::JsonRender;

#[derive(Clone, Copy)]
pub struct LogHelper;

#[cfg(not(feature = "no_logging"))]
impl HelperDef for LogHelper {
    fn call<'reg: 'rc, 'rc>(
        &self,
        h: &Helper,
        _: &Registry,
        _: &Context,
        _: &mut RenderContext,
        _: &mut Output,
    ) -> HelperResult {
        let param = h
            .param(0)
            .ok_or_else(|| RenderError::new("Param not found for helper \"log\""))?;
        let level = h.hash_get("level")
            .and_then(|v| v.value().as_str())
            .unwrap_or("info");

        match level {
            "trace" => trace!("{}: {}",
                              param.path().unwrap_or(&"".to_owned()),
                              param.value().render()),
            "debug" => debug!("{}: {}",
                            param.path().unwrap_or(&"".to_owned()),
                            param.value().render()),
            "info" => info!("{}: {}",
                            param.path().unwrap_or(&"".to_owned()),
                            param.value().render()),
            "warn" => warn!("{}: {}",
                            param.path().unwrap_or(&"".to_owned()),
                            param.value().render()),
            "error" => error!("{}: {}",
                              param.path().unwrap_or(&"".to_owned()),
                              param.value().render()),
            _ => {}
        };
        Ok(())
    }
}

#[cfg(feature = "no_logging")]
impl HelperDef for LogHelper {
    fn call<'reg: 'rc, 'rc>(
        &self,
        _: &Helper,
        _: &Registry,
        _: &Context,
        _: &mut RenderContext,
        _: &mut Output,
    ) -> HelperResult {
        Ok(())
    }
}

pub static LOG_HELPER: LogHelper = LogHelper;
<commit_msg>(refactor) use level string<commit_after>use context::Context;
#[cfg(not(feature = "no_logging"))]
use error::RenderError;
use helpers::{HelperDef, HelperResult};
use output::Output;
use registry::Registry;
use render::{Helper, RenderContext};
#[cfg(not(feature = "no_logging"))]
use value::JsonRender;
#[cfg(not(feature = "no_logging"))]
use log::Level;
#[cfg(not(feature = "no_logging"))]
use std::str::FromStr;

#[derive(Clone, Copy)]
pub struct LogHelper;

#[cfg(not(feature = "no_logging"))]
impl HelperDef for LogHelper {
    fn call<'reg: 'rc, 'rc>(
        &self,
        h: &Helper,
        _: &Registry,
        _: &Context,
        _: &mut RenderContext,
        _: &mut Output,
    ) -> HelperResult {
        let param = h
            .param(0)
            .ok_or_else(|| RenderError::new("Param not found for helper \"log\""))?;
        let level = h.hash_get("level")
            .and_then(|v| v.value().as_str())
            .unwrap_or("info");

        if let Ok(log_level) = Level::from_str(level) {
            log!(log_level, "{}: {}",
                 param.path().unwrap_or(&"".to_owned()),
                 param.value().render())
        } else {
            return Err(RenderError::new(&format!("Unsupported logging level {}", level)));
        }
        Ok(())
    }
}

#[cfg(feature = "no_logging")]
impl HelperDef for LogHelper {
    fn call<'reg: 'rc, 'rc>(
        &self,
        _: &Helper,
        _: &Registry,
        _: &Context,
        _: &mut RenderContext,
        _: &mut Output,
    ) -> HelperResult {
        Ok(())
    }
}

pub static LOG_HELPER: LogHelper = LogHelper;
<|endoftext|>


<commit_before>//! # Router module.
//!
//! The Router module forward sources to sinks.
use std::thread;
use std::time::Duration;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use time;
use std::cmp;
use std::collections::HashMap;
use std::io::prelude::*;
use std::fs;
use std::fs::File;
use std::error::Error;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};

use config;

/// Thread sleeping time.
const REST_TIME: u64 = 10;

/// Router loop.
pub fn router(sinks: &Vec<config::Sink>,
              labels: &HashMap<String, String>,
              parameters: &config::Parameters,
              sigint: Arc<AtomicBool>) {

    let labels: String = labels.iter()
        .fold(String::new(), |acc, (k, v)| {
            let sep = if acc.is_empty() { "" } else { "," };
            acc + sep + k + "=" + v
        });

    loop {
        let start = time::now_utc();

        match route(sinks, parameters, &labels) {
            Err(err) => error!("route fail: {}", err),
            Ok(_) => info!("route success"),
        }

        let elapsed = (time::now_utc() - start).num_milliseconds() as u64;
        let sleep_time = if elapsed > parameters.scan_period {
            REST_TIME
        } else {
            cmp::max(parameters.scan_period - elapsed, REST_TIME)
        };
        for _ in 0..sleep_time / REST_TIME {
            thread::sleep(Duration::from_millis(REST_TIME));
            if sigint.load(Ordering::Relaxed) {
                return;
            }
        }
    }
}

/// Route handle sources forwarding.
fn route(sinks: &Vec<config::Sink>,
         parameters: &config::Parameters,
         labels: &String)
         -> Result<(), Box<Error>> {
    debug!("route");
    loop {
        let entries = try!(fs::read_dir(&parameters.source_dir));
        let mut files = Vec::with_capacity(parameters.batch_count as usize);
        let mut metrics: Vec<String> = Vec::new();

        // Load metrics
        let mut batch_size = 0;
        for (i, entry) in entries.enumerate() {
            let entry = try!(entry);
            // Look only for metrics files
            if entry.path().extension() != Some(OsStr::new("metrics")) {
                continue;
            }

            // Split metrics in capped batch
            if i > parameters.batch_count as usize || batch_size > parameters.batch_size as usize {
                break;
            }

            debug!("open source file {}", format!("{:?}", entry.path()));
            let file = match read(entry.path()) {
                Err(err) => {
                    warn!(err);
                    continue;
                }
                Ok(v) => v,
            };

            for line in file.lines() {
                if labels.is_empty() {
                    metrics.push(String::from(line));
                    continue;
                }
                let mut parts = line.splitn(2, "{");

                let class = match parts.next() {
                    None => {
                        warn!("no_class");
                        continue;
                    }
                    Some(v) => v,
                };
                let class = String::from(class);
                let plabels = match parts.next() {
                    None => {
                        warn!("no_labels");
                        continue;
                    }
                    Some(v) => v,
                };
                let plabels = String::from(plabels);

                let slabels = labels.clone() +
                              if plabels.trim().starts_with("}") {
                    ""
                } else {
                    ","
                } + &plabels;

                metrics.push(format!("{}{{{}", class, slabels))
            }

            files.push(entry.path());
            batch_size += file.len();
        }

        // Nothing to do
        if files.len() == 0 {
            break;
        }

        // Setup sinks files
        let dir = Path::new(&parameters.sink_dir);
        {
            let mut sink_files = Vec::with_capacity(sinks.len() as usize);
            // Open tmp files
            for sink in sinks {
                let sink_file = dir.join(format!("{}.tmp", sink.name));
                debug!("open tmp sink file {}", format!("{:?}", sink_file));
                sink_files.push(try!(File::create(sink_file)));
            }

            // Write metrics
            debug!("write sink files");
            for line in metrics {
                if line.is_empty() {
                    continue;
                }

                for (i, sink) in sinks.iter().enumerate() {
                    if sink.selector.is_some() {
                        let selector = sink.selector.as_ref().unwrap();
                        if line.split_whitespace()
                            .nth(1)
                            .map_or(false, |class| selector.is_match(class)) {
                            continue;
                        }
                    }
                    try!(sink_files[i].write(line.as_bytes()));
                    try!(sink_files[i].write(b"\n"));
                }
            }

            // Flush
            for i in 0..sinks.len() {
                try!(sink_files[i].flush());
            }
        }

        // Rotate
        let now = time::now_utc().to_timespec().sec;
        for sink in sinks {
            let dest_file = dir.join(format!("{}-{}.metrics", sink.name, now));
            debug!("rotate tmp sink file to {}", format!("{:?}", dest_file));
            try!(fs::rename(dir.join(format!("{}.tmp", sink.name)), dest_file));
        }

        // Delete forwarded data
        for f in files {
            debug!("delete source file {}", format!("{:?}", f));
            try!(fs::remove_file(f));
        }
    }

    Ok(())
}

/// Read a file as String
fn read(path: PathBuf) -> Result<String, Box<Error>> {
    let mut file = try!(File::open(path));

    let mut content = String::new();
    try!(file.read_to_string(&mut content));

    Ok(content)
}
<commit_msg>fix: selector is reverse<commit_after>//! # Router module.
//!
//! The Router module forward sources to sinks.
use std::thread;
use std::time::Duration;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use time;
use std::cmp;
use std::collections::HashMap;
use std::io::prelude::*;
use std::fs;
use std::fs::File;
use std::error::Error;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};

use config;

/// Thread sleeping time.
const REST_TIME: u64 = 10;

/// Router loop.
pub fn router(sinks: &Vec<config::Sink>,
              labels: &HashMap<String, String>,
              parameters: &config::Parameters,
              sigint: Arc<AtomicBool>) {

    let labels: String = labels.iter()
        .fold(String::new(), |acc, (k, v)| {
            let sep = if acc.is_empty() { "" } else { "," };
            acc + sep + k + "=" + v
        });

    loop {
        let start = time::now_utc();

        match route(sinks, parameters, &labels) {
            Err(err) => error!("route fail: {}", err),
            Ok(_) => info!("route success"),
        }

        let elapsed = (time::now_utc() - start).num_milliseconds() as u64;
        let sleep_time = if elapsed > parameters.scan_period {
            REST_TIME
        } else {
            cmp::max(parameters.scan_period - elapsed, REST_TIME)
        };
        for _ in 0..sleep_time / REST_TIME {
            thread::sleep(Duration::from_millis(REST_TIME));
            if sigint.load(Ordering::Relaxed) {
                return;
            }
        }
    }
}

/// Route handle sources forwarding.
fn route(sinks: &Vec<config::Sink>,
         parameters: &config::Parameters,
         labels: &String)
         -> Result<(), Box<Error>> {
    debug!("route");
    loop {
        let entries = try!(fs::read_dir(&parameters.source_dir));
        let mut files = Vec::with_capacity(parameters.batch_count as usize);
        let mut metrics: Vec<String> = Vec::new();

        // Load metrics
        let mut batch_size = 0;
        for (i, entry) in entries.enumerate() {
            let entry = try!(entry);
            // Look only for metrics files
            if entry.path().extension() != Some(OsStr::new("metrics")) {
                continue;
            }

            // Split metrics in capped batch
            if i > parameters.batch_count as usize || batch_size > parameters.batch_size as usize {
                break;
            }

            debug!("open source file {}", format!("{:?}", entry.path()));
            let file = match read(entry.path()) {
                Err(err) => {
                    warn!(err);
                    continue;
                }
                Ok(v) => v,
            };

            for line in file.lines() {
                if labels.is_empty() {
                    metrics.push(String::from(line));
                    continue;
                }
                let mut parts = line.splitn(2, "{");

                let class = match parts.next() {
                    None => {
                        warn!("no_class");
                        continue;
                    }
                    Some(v) => v,
                };
                let class = String::from(class);
                let plabels = match parts.next() {
                    None => {
                        warn!("no_labels");
                        continue;
                    }
                    Some(v) => v,
                };
                let plabels = String::from(plabels);

                let slabels = labels.clone() +
                              if plabels.trim().starts_with("}") {
                    ""
                } else {
                    ","
                } + &plabels;

                metrics.push(format!("{}{{{}", class, slabels))
            }

            files.push(entry.path());
            batch_size += file.len();
        }

        // Nothing to do
        if files.len() == 0 {
            break;
        }

        // Setup sinks files
        let dir = Path::new(&parameters.sink_dir);
        {
            let mut sink_files = Vec::with_capacity(sinks.len() as usize);
            // Open tmp files
            for sink in sinks {
                let sink_file = dir.join(format!("{}.tmp", sink.name));
                debug!("open tmp sink file {}", format!("{:?}", sink_file));
                sink_files.push(try!(File::create(sink_file)));
            }

            // Write metrics
            debug!("write sink files");
            for line in metrics {
                if line.is_empty() {
                    continue;
                }

                for (i, sink) in sinks.iter().enumerate() {
                    if sink.selector.is_some() {
                        let selector = sink.selector.as_ref().unwrap();
                        if !line.split_whitespace()
                            .nth(1)
                            .map_or(false, |class| selector.is_match(class)) {
                            continue;
                        }
                    }
                    try!(sink_files[i].write(line.as_bytes()));
                    try!(sink_files[i].write(b"\n"));
                }
            }

            // Flush
            for i in 0..sinks.len() {
                try!(sink_files[i].flush());
            }
        }

        // Rotate
        let now = time::now_utc().to_timespec().sec;
        for sink in sinks {
            let dest_file = dir.join(format!("{}-{}.metrics", sink.name, now));
            debug!("rotate tmp sink file to {}", format!("{:?}", dest_file));
            try!(fs::rename(dir.join(format!("{}.tmp", sink.name)), dest_file));
        }

        // Delete forwarded data
        for f in files {
            debug!("delete source file {}", format!("{:?}", f));
            try!(fs::remove_file(f));
        }
    }

    Ok(())
}

/// Read a file as String
fn read(path: PathBuf) -> Result<String, Box<Error>> {
    let mut file = try!(File::open(path));

    let mut content = String::new();
    try!(file.read_to_string(&mut content));

    Ok(content)
}
<|endoftext|>


<commit_before>use futures_core::{ready, Stream};
use std::fmt;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore, TryAcquireError};

use super::ReusableBoxFuture;

/// A wrapper around [`Semaphore`] that provides a `poll_acquire` method.
///
/// [`Semaphore`]: tokio::sync::Semaphore
pub struct PollSemaphore {
    semaphore: Arc<Semaphore>,
    permit_fut: Option<ReusableBoxFuture<Result<OwnedSemaphorePermit, AcquireError>>>,
}

impl PollSemaphore {
    /// Create a new `PollSemaphore`.
    pub fn new(semaphore: Arc<Semaphore>) -> Self {
        Self {
            semaphore,
            permit_fut: None,
        }
    }

    /// Closes the semaphore.
    pub fn close(&self) {
        self.semaphore.close()
    }

    /// Obtain a clone of the inner semaphore.
    pub fn clone_inner(&self) -> Arc<Semaphore> {
        self.semaphore.clone()
    }

    /// Get back the inner semaphore.
    pub fn into_inner(self) -> Arc<Semaphore> {
        self.semaphore
    }

    /// Poll to acquire a permit from the semaphore.
    ///
    /// This can return the following values:
    ///
    ///  - `Poll::Pending` if a permit is not currently available.
    ///  - `Poll::Ready(Some(permit))` if a permit was acquired.
    ///  - `Poll::Ready(None)` if the semaphore has been closed.
    ///
    /// When this method returns `Poll::Pending`, the current task is scheduled
    /// to receive a wakeup when a permit becomes available, or when the
    /// semaphore is closed. Note that on multiple calls to `poll_acquire`, only
    /// the `Waker` from the `Context` passed to the most recent call is
    /// scheduled to receive a wakeup.
    pub fn poll_acquire(&mut self, cx: &mut Context<'_>) -> Poll<Option<OwnedSemaphorePermit>> {
        let permit_future = match self.permit_fut.as_mut() {
            Some(fut) => fut,
            None => {
                // avoid allocations completely if we can grab a permit immediately
                match Arc::clone(&self.semaphore).try_acquire_owned() {
                    Ok(permit) => return Poll::Ready(Some(permit)),
                    Err(TryAcquireError::Closed) => return Poll::Ready(None),
                    Err(TryAcquireError::NoPermits) => {}
                }

                let next_fut = Arc::clone(&self.semaphore).acquire_owned();
                self.permit_fut
                    .get_or_insert(ReusableBoxFuture::new(next_fut))
            }
        };

        let result = ready!(permit_future.poll(cx));

        let next_fut = Arc::clone(&self.semaphore).acquire_owned();
        permit_future.set(next_fut);

        match result {
            Ok(permit) => Poll::Ready(Some(permit)),
            Err(_closed) => {
                self.permit_fut = None;
                Poll::Ready(None)
            }
        }
    }
}

impl Stream for PollSemaphore {
    type Item = OwnedSemaphorePermit;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<OwnedSemaphorePermit>> {
        Pin::into_inner(self).poll_acquire(cx)
    }
}

impl Clone for PollSemaphore {
    fn clone(&self) -> PollSemaphore {
        PollSemaphore::new(self.clone_inner())
    }
}

impl fmt::Debug for PollSemaphore {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("PollSemaphore")
            .field("semaphore", &self.semaphore)
            .finish()
    }
}
<commit_msg>util: add `PollSemaphore::{add_permits, available_permits}` (#3683)<commit_after>use futures_core::{ready, Stream};
use std::fmt;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore, TryAcquireError};

use super::ReusableBoxFuture;

/// A wrapper around [`Semaphore`] that provides a `poll_acquire` method.
///
/// [`Semaphore`]: tokio::sync::Semaphore
pub struct PollSemaphore {
    semaphore: Arc<Semaphore>,
    permit_fut: Option<ReusableBoxFuture<Result<OwnedSemaphorePermit, AcquireError>>>,
}

impl PollSemaphore {
    /// Create a new `PollSemaphore`.
    pub fn new(semaphore: Arc<Semaphore>) -> Self {
        Self {
            semaphore,
            permit_fut: None,
        }
    }

    /// Closes the semaphore.
    pub fn close(&self) {
        self.semaphore.close()
    }

    /// Obtain a clone of the inner semaphore.
    pub fn clone_inner(&self) -> Arc<Semaphore> {
        self.semaphore.clone()
    }

    /// Get back the inner semaphore.
    pub fn into_inner(self) -> Arc<Semaphore> {
        self.semaphore
    }

    /// Poll to acquire a permit from the semaphore.
    ///
    /// This can return the following values:
    ///
    ///  - `Poll::Pending` if a permit is not currently available.
    ///  - `Poll::Ready(Some(permit))` if a permit was acquired.
    ///  - `Poll::Ready(None)` if the semaphore has been closed.
    ///
    /// When this method returns `Poll::Pending`, the current task is scheduled
    /// to receive a wakeup when a permit becomes available, or when the
    /// semaphore is closed. Note that on multiple calls to `poll_acquire`, only
    /// the `Waker` from the `Context` passed to the most recent call is
    /// scheduled to receive a wakeup.
    pub fn poll_acquire(&mut self, cx: &mut Context<'_>) -> Poll<Option<OwnedSemaphorePermit>> {
        let permit_future = match self.permit_fut.as_mut() {
            Some(fut) => fut,
            None => {
                // avoid allocations completely if we can grab a permit immediately
                match Arc::clone(&self.semaphore).try_acquire_owned() {
                    Ok(permit) => return Poll::Ready(Some(permit)),
                    Err(TryAcquireError::Closed) => return Poll::Ready(None),
                    Err(TryAcquireError::NoPermits) => {}
                }

                let next_fut = Arc::clone(&self.semaphore).acquire_owned();
                self.permit_fut
                    .get_or_insert(ReusableBoxFuture::new(next_fut))
            }
        };

        let result = ready!(permit_future.poll(cx));

        let next_fut = Arc::clone(&self.semaphore).acquire_owned();
        permit_future.set(next_fut);

        match result {
            Ok(permit) => Poll::Ready(Some(permit)),
            Err(_closed) => {
                self.permit_fut = None;
                Poll::Ready(None)
            }
        }
    }

    /// Returns the current number of available permits.
    ///
    /// This is equivalent to the [`Semaphore::available_permits`] method on the
    /// `tokio::sync::Semaphore` type.
    ///
    /// [`Semaphore::available_permits`]: tokio::sync::Semaphore::available_permits
    pub fn available_permits(&self) -> usize {
        self.semaphore.available_permits()
    }

    /// Adds `n` new permits to the semaphore.
    ///
    /// The maximum number of permits is `usize::MAX >> 3`, and this function
    /// will panic if the limit is exceeded.
    ///
    /// This is equivalent to the [`Semaphore::add_permits`] method on the
    /// `tokio::sync::Semaphore` type.
    ///
    /// [`Semaphore::add_permits`]: tokio::sync::Semaphore::add_permits
    pub fn add_permits(&self, n: usize) {
        self.semaphore.add_permits(n);
    }
}

impl Stream for PollSemaphore {
    type Item = OwnedSemaphorePermit;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<OwnedSemaphorePermit>> {
        Pin::into_inner(self).poll_acquire(cx)
    }
}

impl Clone for PollSemaphore {
    fn clone(&self) -> PollSemaphore {
        PollSemaphore::new(self.clone_inner())
    }
}

impl fmt::Debug for PollSemaphore {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("PollSemaphore")
            .field("semaphore", &self.semaphore)
            .finish()
    }
}

impl AsRef<Semaphore> for PollSemaphore {
    fn as_ref(&self) -> &Semaphore {
        &*self.semaphore
    }
}
<|endoftext|>


<commit_before>//! Provides the server and handles the incomming requests
//! All ports are handled by the same function

// System modules:
use std::net::{TcpListener, TcpStream, SocketAddr};
use std::thread::spawn;
use std::io::prelude::*;
use std::sync::{Arc, Mutex};
use std::fs::OpenOptions;
use std::io::BufWriter;
use std::io::Result;
use time::{now, Tm};

// Internal modules:
use configuration::{Configuration, HEADER_LENGTH};

fn write_data(tm: Tm, local_port: u16, buffer: &[u8], file_name: &str) -> Result<()> {
    let mut file_handle = BufWriter::new(try!(OpenOptions::new()
        .write(true).create(true).append(true).open(file_name)));

    let current_date_time = tm.strftime("%Y.%m.%d - %H:%M:%S").unwrap();

    try!(write!(file_handle, "<measure>\n"));
    try!(write!(file_handle, "<port>{}</port>\n", local_port));
    try!(write!(file_handle, "<date_time>{}</date_time>\n", &current_date_time));
    try!(write!(file_handle, "<data>\n"));
    try!(write!(file_handle, "{:?}\n", buffer));
    try!(write!(file_handle, "</data>\n"));
    try!(write!(file_handle, "</measure>\n\n"));

    Ok(())
}

fn handle_client(stream: &mut TcpStream, remote_addr: &SocketAddr,
    all_data_file: &Arc<Mutex<String>>, monthly_data_folder: &Arc<Mutex<String>>) -> Result<()> {
    info!("Client socket address: {}", remote_addr);

    let local_addr = try!(stream.local_addr());

    let local_port = match local_addr {
        SocketAddr::V4(local_addr) => local_addr.port(),
        SocketAddr::V6(local_addr) => local_addr.port()
    };

    info!("Port: {}", local_port);

    let mut buffer = Vec::new();

    let len = try!(stream.read_to_end(&mut buffer));
    info!("Number of bytes received: {}", len);

    // TODO: use str::from_utf8(buf) if data is sent in clear text
    // Otherwise parse binary data to floats

    // base16 2 byte floats:
    // https://en.wikipedia.org/wiki/Half-precision_floating-point_format
    // https://github.com/sgothel/jogl/blob/master/src/jogl/classes/com/jogamp/opengl/math/Binary16.java
    // https://books.google.de/books?id=FPlICAAAQBAJ&pg=PA84&lpg=PA84&dq=binary16&source=bl&ots=0FAzD4XOqn&sig=98h_pzPlLzUXjB4uY1T8MRIZOnA&hl=de&sa=X&ved=0ahUKEwjkpvXU5ZzLAhVD9HIKHQOfAxYQ6AEITzAH#v=onepage&q=binary16&f=false
    // http://www.gamedev.net/topic/557338-ieee-754-2008-binary-16-inaccuracy-in-wikipedia/

    if buffer.len() > HEADER_LENGTH {
        let (buffer_left, buffer_right) = buffer.split_at(HEADER_LENGTH);

        let str_header = String::from_utf8_lossy(buffer_left);
        let str_data = String::from_utf8_lossy(buffer_right);

        info!("Header: {:?}", buffer_left);
        info!("Data: {:?}", buffer_right);

        info!("Header (ASCII): '{}'", str_header);
        info!("Data (ASCII): '{}'", str_data);

        match all_data_file.lock() {
            Ok(all_data_file) => {
                let tm = now();
                let file_name = format!("{}/{}.txt", local_port, *all_data_file);
                try!(write_data(tm, local_port, buffer_right, &file_name));
            },
            Err(e) => info!("Mutex (poison) error (all_data_file): {}", e)
        }

        match monthly_data_folder.lock() {
            Ok(monthly_data_folder) => {
                let tm = now();
                let current_year = tm.strftime("%Y").unwrap();
                let current_month = tm.strftime("%m").unwrap();
                // TODO: create separate folder for year and month in Rust
                let file_name = format!("{}/{}/{}_{}.txt", *monthly_data_folder, local_port, current_year, current_month);
                try!(write_data(tm, local_port, buffer_right, &file_name));
            },
            Err(e) => info!("Mutex (poison) error (monthly_data_folder): {}", e)
        }
    } else if buffer.len() < HEADER_LENGTH {
        info!("Invalid header (less than {} bytes received)!", HEADER_LENGTH);
        info!("Bytes: {:?}", buffer);
        let str_buffer = String::from_utf8_lossy(&buffer);
        info!("Bytes (ASCII): '{}'", str_buffer);
    } else { // buffer.len() == HEADER_LENGTH -> no data, only header
        info!("No data received, just header.");
        info!("Bytes: {:?}", buffer);
        let str_buffer = String::from_utf8_lossy(&buffer);
        info!("Bytes (ASCII): '{}'", str_buffer);
    }

    Ok(())
}

pub fn start_service(config: Configuration) {
    let mut listeners = Vec::new();

    for port in config.ports {
        match TcpListener::bind(("0.0.0.0", port)) {
            Ok(listener) => {
                info!("Create listener for port {}", port);
                listeners.push(listener);
            },
            Err(e) => info!("Network error: {}", e)
        }
    }

    let all_data_file = Arc::new(Mutex::new(config.all_data_file.clone()));
    let monthly_data_folder = Arc::new(Mutex::new(config.monthly_data_folder.clone()));

    for listener in listeners {
        let all_data_file = all_data_file.clone();
        let monthly_data_folder = monthly_data_folder.clone();
        spawn(move|| {
            loop {
                let result = listener.accept();
                if let Ok(result) = result {
                    let (mut stream, addr) = result;
                    if let Err(io_error) = handle_client(&mut stream, &addr,
                            &all_data_file, &monthly_data_folder) {
                        info!("IOError: {}", io_error);
                    }
                }
            }
        });
    }
}
<commit_msg>refactoring ASCII output for buffer<commit_after>//! Provides the server and handles the incomming requests
//! All ports are handled by the same function

// System modules:
use std::net::{TcpListener, TcpStream, SocketAddr};
use std::thread::spawn;
use std::io::prelude::*;
use std::sync::{Arc, Mutex};
use std::fs::OpenOptions;
use std::io::BufWriter;
use std::io::Result;
use time::{now, Tm};

// Internal modules:
use configuration::{Configuration, HEADER_LENGTH};

fn write_data(tm: Tm, local_port: u16, buffer: &[u8], file_name: &str) -> Result<()> {
    let mut file_handle = BufWriter::new(try!(OpenOptions::new()
        .write(true).create(true).append(true).open(file_name)));

    let current_date_time = tm.strftime("%Y.%m.%d - %H:%M:%S").unwrap();

    try!(write!(file_handle, "<measure>\n"));
    try!(write!(file_handle, "<port>{}</port>\n", local_port));
    try!(write!(file_handle, "<date_time>{}</date_time>\n", &current_date_time));
    try!(write!(file_handle, "<data>\n"));
    try!(write!(file_handle, "{:?}\n", buffer));
    try!(write!(file_handle, "</data>\n"));
    try!(write!(file_handle, "</measure>\n\n"));

    Ok(())
}

fn handle_client(stream: &mut TcpStream, remote_addr: &SocketAddr,
    all_data_file: &Arc<Mutex<String>>, monthly_data_folder: &Arc<Mutex<String>>) -> Result<()> {
    info!("Client socket address: {}", remote_addr);

    let local_addr = try!(stream.local_addr());

    let local_port = match local_addr {
        SocketAddr::V4(local_addr) => local_addr.port(),
        SocketAddr::V6(local_addr) => local_addr.port()
    };

    info!("Port: {}", local_port);

    let mut buffer = Vec::new();

    let len = try!(stream.read_to_end(&mut buffer));
    info!("Number of bytes received: {}", len);

    // TODO: use str::from_utf8(buf) if data is sent in clear text
    // Otherwise parse binary data to floats

    // base16 2 byte floats:
    // https://en.wikipedia.org/wiki/Half-precision_floating-point_format
    // https://github.com/sgothel/jogl/blob/master/src/jogl/classes/com/jogamp/opengl/math/Binary16.java
    // https://books.google.de/books?id=FPlICAAAQBAJ&pg=PA84&lpg=PA84&dq=binary16&source=bl&ots=0FAzD4XOqn&sig=98h_pzPlLzUXjB4uY1T8MRIZOnA&hl=de&sa=X&ved=0ahUKEwjkpvXU5ZzLAhVD9HIKHQOfAxYQ6AEITzAH#v=onepage&q=binary16&f=false
    // http://www.gamedev.net/topic/557338-ieee-754-2008-binary-16-inaccuracy-in-wikipedia/

    if buffer.len() > HEADER_LENGTH {
        let (buffer_left, buffer_right) = buffer.split_at(HEADER_LENGTH);

        let str_header = String::from_utf8_lossy(buffer_left);
        let str_data = String::from_utf8_lossy(buffer_right);

        info!("Header: {:?}", buffer_left);
        info!("Data: {:?}", buffer_right);

        info!("Header (ASCII): '{}'", str_header);
        info!("Data (ASCII): '{}'", str_data);

        match all_data_file.lock() {
            Ok(all_data_file) => {
                let tm = now();
                let file_name = format!("{}/{}.txt", local_port, *all_data_file);
                try!(write_data(tm, local_port, buffer_right, &file_name));
            },
            Err(e) => info!("Mutex (poison) error (all_data_file): {}", e)
        }

        match monthly_data_folder.lock() {
            Ok(monthly_data_folder) => {
                let tm = now();
                let current_year = tm.strftime("%Y").unwrap();
                let current_month = tm.strftime("%m").unwrap();
                // TODO: create separate folder for year and month in Rust
                let file_name = format!("{}/{}/{}_{}.txt", *monthly_data_folder, local_port, current_year, current_month);
                try!(write_data(tm, local_port, buffer_right, &file_name));
            },
            Err(e) => info!("Mutex (poison) error (monthly_data_folder): {}", e)
        }
    } else if buffer.len() < HEADER_LENGTH {
        info!("Invalid header (less than {} bytes received)!", HEADER_LENGTH);
        info!("Bytes: {:?}", buffer);
        info!("Bytes (ASCII): '{}'", String::from_utf8_lossy(&buffer));
    } else { // buffer.len() == HEADER_LENGTH -> no data, only header
        info!("No data received, just header.");
        info!("Bytes: {:?}", buffer);
        info!("Bytes (ASCII): '{}'", String::from_utf8_lossy(&buffer));
    }

    Ok(())
}

pub fn start_service(config: Configuration) {
    let mut listeners = Vec::new();

    for port in config.ports {
        match TcpListener::bind(("0.0.0.0", port)) {
            Ok(listener) => {
                info!("Create listener for port {}", port);
                listeners.push(listener);
            },
            Err(e) => info!("Network error: {}", e)
        }
    }

    let all_data_file = Arc::new(Mutex::new(config.all_data_file.clone()));
    let monthly_data_folder = Arc::new(Mutex::new(config.monthly_data_folder.clone()));

    for listener in listeners {
        let all_data_file = all_data_file.clone();
        let monthly_data_folder = monthly_data_folder.clone();
        spawn(move|| {
            loop {
                let result = listener.accept();
                if let Ok(result) = result {
                    let (mut stream, addr) = result;
                    if let Err(io_error) = handle_client(&mut stream, &addr,
                            &all_data_file, &monthly_data_folder) {
                        info!("IOError: {}", io_error);
                    }
                }
            }
        });
    }
}
<|endoftext|>


<commit_before>use grain_capnp::{PowerboxCapability, UiView, UiSession};
use web_session_capnp::{WebSession};

use capnp::capability::{ClientHook, FromServer};
use capnp::AnyPointer;
use capnp_rpc::rpc::{RpcConnectionState, SturdyRefRestorer};
use capnp_rpc::capability::{LocalClient};

pub struct UiViewImpl;

impl PowerboxCapability::Server for UiViewImpl {
    fn get_powerbox_info(&mut self, context : PowerboxCapability::GetPowerboxInfoContext) {
        context.done()
    }
}

impl UiView::Server for UiViewImpl {
    fn get_view_info(&mut self, context : UiView::GetViewInfoContext) {
        context.done()
    }

    fn new_session(&mut self, mut context : UiView::NewSessionContext) {
        println!("asked for a new session!");
        let (_, results) = context.get();

        let client : WebSession::Client = FromServer::new(None::<LocalClient>, ~WebSessionImpl);
        // we need to do this dance to upcast.
        results.set_session(UiSession::Client { client : client.client});

        context.done()
    }
}

pub struct WebSessionImpl;

impl UiSession::Server for WebSessionImpl {

}

static main_css : &'static str =
    "body { font-family: Helvetica, Sans, Arial;
            font-size: medium;
             margin-left: auto;
             margin-right: auto;
             width: 600px;
     }";

static header : &'static str =
  r#"<title> acronomy </title><link rel="stylesheet" type="text/css" href="main.css" >
 <meta http-equiv="Content-Type" content="text/html;charset=utf-8" >
  "#;

impl WebSession::Server for WebSessionImpl {
    fn get(&mut self, mut context : WebSession::GetContext) {
        println!("GET");
        let (params, results) = context.get();
        println!("path = {}", params.get_path());
        let content = results.init_content();
        content.set_mime_type("text/html");
        if params.get_path() == "main.css" {
            content.get_body().set_bytes(main_css.as_bytes())
        } else {
            content.get_body().set_bytes(format!("{}hello world",
                                                 header).as_bytes());
        }
        context.done()
    }
    fn post(&mut self, context : WebSession::PostContext) {
        println!("POST");
        context.done()
    }
    fn put(&mut self, context : WebSession::PutContext) {
        println!("PUT");
        context.done()
    }
    fn delete(&mut self, context : WebSession::DeleteContext) {
        println!("DELETE");
        context.done()
    }
    fn open_web_socket(&mut self, context : WebSession::OpenWebSocketContext) {
        println!("OPEN WEB SOCKET");
        context.done()
    }
}

pub struct FdStream {
    inner : ~::std::rt::rtio::RtioFileStream:Send,
}

impl FdStream {
    pub fn new(fd : ::libc::c_int) -> ::std::io::IoResult<FdStream> {
        ::std::rt::rtio::LocalIo::maybe_raise(|io| {
            Ok (FdStream { inner : io.fs_from_raw_fd(fd, ::std::rt::rtio::DontClose) })
        })
    }
}

impl Reader for FdStream {
    fn read(&mut self, buf : &mut [u8]) -> ::std::io::IoResult<uint> {
        self.inner.read(buf).map(|i| i as uint)
    }
}

impl Writer for FdStream {
    fn write(&mut self, buf : &[u8]) -> ::std::io::IoResult<()> {
        self.inner.write(buf)
    }
}

pub struct Restorer;

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : AnyPointer::Reader) -> Option<~ClientHook:Send> {
        if obj_id.is_null() {
            let client : UiView::Client = FromServer::new(None::<LocalClient>, ~UiViewImpl);
            Some(client.client.hook)
        } else {
            None
        }
    }
}

pub fn main() -> ::std::io::IoResult<()> {
    let ifs = try!(FdStream::new(3));
    let ofs = try!(FdStream::new(3));

    let connection_state = RpcConnectionState::new();
    connection_state.run(ifs, ofs, Restorer);

    Ok(())
}

<commit_msg>making a form<commit_after>use grain_capnp::{PowerboxCapability, UiView, UiSession};
use web_session_capnp::{WebSession};

use capnp::capability::{ClientHook, FromServer};
use capnp::AnyPointer;
use capnp_rpc::rpc::{RpcConnectionState, SturdyRefRestorer};
use capnp_rpc::capability::{LocalClient};

pub struct UiViewImpl;

impl PowerboxCapability::Server for UiViewImpl {
    fn get_powerbox_info(&mut self, context : PowerboxCapability::GetPowerboxInfoContext) {
        context.done()
    }
}

impl UiView::Server for UiViewImpl {
    fn get_view_info(&mut self, context : UiView::GetViewInfoContext) {
        context.done()
    }

    fn new_session(&mut self, mut context : UiView::NewSessionContext) {
        println!("asked for a new session!");
        let (_, results) = context.get();

        let client : WebSession::Client = FromServer::new(None::<LocalClient>, ~WebSessionImpl);
        // we need to do this dance to upcast.
        results.set_session(UiSession::Client { client : client.client});

        context.done()
    }
}

pub struct WebSessionImpl;

impl UiSession::Server for WebSessionImpl {

}

static main_css : &'static str =
    "body { font-family: Helvetica, Sans, Arial;
            font-size: medium;
             margin-left: auto;
             margin-right: auto;
             width: 600px;
     }";

static header : &'static str =
  r#"<title> acronymy </title><link rel="stylesheet" type="text/css" href="main.css" >
 <meta http-equiv="Content-Type" content="text/html;charset=utf-8" >
  "#;

impl WebSession::Server for WebSessionImpl {
    fn get(&mut self, mut context : WebSession::GetContext) {
        println!("GET");
        let (params, results) = context.get();
        let path = params.get_path();
        println!("path = {}", path);
        let content = results.init_content();
        content.set_mime_type("text/html");
        if path == "main.css" {
            content.get_body().set_bytes(main_css.as_bytes())
        } else {
            content.get_body().set_bytes(
                format!("{}<form action=\"define\" method=\"get\"><input name=\"word\"/><button>go</button></form>",
                        header).as_bytes());
        }
        context.done()
    }
    fn post(&mut self, context : WebSession::PostContext) {
        println!("POST");
        context.done()
    }
    fn put(&mut self, context : WebSession::PutContext) {
        println!("PUT");
        context.done()
    }
    fn delete(&mut self, context : WebSession::DeleteContext) {
        println!("DELETE");
        context.done()
    }
    fn open_web_socket(&mut self, context : WebSession::OpenWebSocketContext) {
        println!("OPEN WEB SOCKET");
        context.done()
    }
}

pub struct FdStream {
    inner : ~::std::rt::rtio::RtioFileStream:Send,
}

impl FdStream {
    pub fn new(fd : ::libc::c_int) -> ::std::io::IoResult<FdStream> {
        ::std::rt::rtio::LocalIo::maybe_raise(|io| {
            Ok (FdStream { inner : io.fs_from_raw_fd(fd, ::std::rt::rtio::DontClose) })
        })
    }
}

impl Reader for FdStream {
    fn read(&mut self, buf : &mut [u8]) -> ::std::io::IoResult<uint> {
        self.inner.read(buf).map(|i| i as uint)
    }
}

impl Writer for FdStream {
    fn write(&mut self, buf : &[u8]) -> ::std::io::IoResult<()> {
        self.inner.write(buf)
    }
}

pub struct Restorer;

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : AnyPointer::Reader) -> Option<~ClientHook:Send> {
        if obj_id.is_null() {
            let client : UiView::Client = FromServer::new(None::<LocalClient>, ~UiViewImpl);
            Some(client.client.hook)
        } else {
            None
        }
    }
}

pub fn main() -> ::std::io::IoResult<()> {
    let ifs = try!(FdStream::new(3));
    let ofs = try!(FdStream::new(3));

    let connection_state = RpcConnectionState::new();
    connection_state.run(ifs, ofs, Restorer);

    Ok(())
}

<|endoftext|>