<commit_before>use clap::App as Clapp;
use std::env::home_dir;
use std::path::PathBuf;
use yaml_file_handler::yaml_handler::FileHandler as YamlFileHandler;
use io::read_prompted;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Options {
	pub name: String,
	pub server: String,
}

impl Options {
	/// Parses commandline arguments into an [`Options`](#) instance
	///
	/// Optionally reads from a config file in [YAML](http://yaml.org) format, however commandline arguments take preference thereover.
	/// The config file format is trivial: all root keys and values are of the same name and format as long commandline arguments,
	/// see `"example/config.yml"`.
	pub fn parse() -> Options {
		const USAGE: &'static str = "-c --config=[conf]   'Sets config file to load, values will be overriden by commandline args'
		                             -n --name   [name]   'Sets username, will prompt if not specified nor determined'
		                             -s --server [server] 'Sets the server to connect to'";

		let matches = Clapp::new("chattium-oxide-client").version(env!("CARGO_PKG_VERSION"))
		                                                 .author("nabijaczleweli <nabijaczleweli@gmail.com>")
		                                                 .about("Chat client for chattium-oxide-server")
		                                                 .args_from_usage(USAGE)
		                                                 .get_matches();
		let mut name: Option<String> = None;
		let mut server: Option<String> = None;
		if let Some(conf) = matches.value_of("conf") {
			let mut yaml = YamlFileHandler::new();
			if yaml.add_files(vec![conf]) {
				if let Some(yaml) = yaml.read_all_files().as_ref().map(|all| {
					let mut b = PathBuf::from(conf);
					b.set_extension("");
					&all[b.file_name().unwrap().to_str().unwrap()]
				}) {
					name = yaml["name"].as_str().map(|n| n.to_string());
					server = yaml["server"].as_str().map(|s| s.to_string());
				}
			}
		}

		if let Some(cname)   = matches.value_of("name")   {if cname.len()   > 0 {name   = Some(cname.to_string())}}
		if let Some(cserver) = matches.value_of("server") {if cserver.len() > 0 {server = Some(cserver.to_string())}}

		if name.is_none() {
			name = Some(match username() {
				Some(uname) =>
					if let Some("") = matches.value_of("name") {
						uname
					} else {
						match read_prompted(format_args!("Determined your username to {}.\nIf that's incorrect, type in your name now. Otherwise, hit <Return>: ", uname)) {
							Ok(rname) =>
								match rname {
									Some(rname) => rname,
									None => uname,
								},
							Err(_) => {
								println!("Failed to read custom name, assuming default OK.");
								uname
							}
						}
					},
				None => {
					let mut tname: Option<String> = None;
					while tname.is_none() {
						match read_prompted(format_args!("No username specified and none could be determined.\nPlease type in your username now: ")) {
							Ok(rname) => tname = rname,
							Err(error) => println!("Couldn't read username: {}", error),
						}
					}
					tname.unwrap()
				},
			});
		}

		if server.is_none() {
			let mut tserver: Option<String> = None;
			while tserver.is_none() {
				match read_prompted(format_args!("No server specified.\nPlease type in the server address now: ")) {
					Ok(rserver) => tserver = rserver,
					Err(error) => println!("Couldn't read server: {}", error),
				}
			}
			server = tserver;
		}

		assert!(name.is_some());
		assert!(server.is_some());
		Options{
			name: name.unwrap(),
			server: server.unwrap(),
		}
	}
}

/// Extract username from last segment of user's homedir
fn username() -> Option<String> {
	match home_dir() {
		Some(pbuf) =>
			match pbuf.as_path().file_name() {
				Some(fname) => fname.to_str().map(|string| string.to_string()),
				None => None,
			},
		None => None,
	}
}
<commit_msg>Display all prompts via BLT. Closes #1<commit_after>use yaml_file_handler::yaml_handler::FileHandler as YamlFileHandler;
use bear_lib_terminal::geometry::Point;
use bear_lib_terminal::terminal;
use clap::App as Clapp;
use std::env::home_dir;
use std::path::PathBuf;
use io;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Options {
	pub name: String,
	pub server: String,
}

impl Options {
	/// Parses commandline arguments into an [`Options`](#) instance
	///
	/// Optionally reads from a config file in [YAML](http://yaml.org) format, however commandline arguments take preference thereover.
	/// The config file format is trivial: all root keys and values are of the same name and format as long commandline arguments,
	/// see `"example/config.yml"`.
	pub fn parse() -> Options {
		const USAGE: &'static str = "-c --config=[conf]   'Sets config file to load, values will be overriden by commandline args'
		                             -n --name   [name]   'Sets username, will prompt if not specified nor determined'
		                             -s --server [server] 'Sets the server to connect to'";

		let matches = Clapp::new("chattium-oxide-client").version(env!("CARGO_PKG_VERSION"))
		                                                 .author("nabijaczleweli <nabijaczleweli@gmail.com>")
		                                                 .about("Chat client for chattium-oxide-server")
		                                                 .args_from_usage(USAGE)
		                                                 .get_matches();
		let mut name: Option<String> = None;
		let mut server: Option<String> = None;
		if let Some(conf) = matches.value_of("conf") {
			let mut yaml = YamlFileHandler::new();
			if yaml.add_files(vec![conf]) {
				if let Some(yaml) = yaml.read_all_files().as_ref().map(|all| {
					let mut b = PathBuf::from(conf);
					b.set_extension("");
					&all[b.file_name().unwrap().to_str().unwrap()]
				}) {
					name = yaml["name"].as_str().map(|n| n.to_string());
					server = yaml["server"].as_str().map(|s| s.to_string());
				}
			}
		}

		if let Some(cname)   = matches.value_of("name")   {if cname.len()   > 0 {name   = Some(cname.to_string())}}
		if let Some(cserver) = matches.value_of("server") {if cserver.len() > 0 {server = Some(cserver.to_string())}}

		if name.is_none() {
			name = Some(match username() {
				Some(uname) =>
					if let Some("") = matches.value_of("name") {
						uname
					} else {
						terminal::print_xy(0, 0, &*&format!("Determined your username to {}.
If that's incorrect, type in your name now. Otherwise, hit <Return>: ", uname));
						terminal::refresh();
						match terminal::read_str(Point::new(0, 2), terminal::state::size().width).into_iter().flat_map(io::maybe_trimmed).next() {
							Some(rname) => rname,
							None        => uname,
						}
					},
				None => {
					let mut tname: Option<String> = None;
					while tname.is_none() {
						terminal::print_xy(0, 0, "No username specified and none could be determined.");
						let second_line = "Please type in your username now: ";
						terminal::print_xy(0, 1, second_line);
						terminal::refresh();
						for rname in terminal::read_str(Point::new(second_line.len() as i32, 1), terminal::state::size().width).into_iter().flat_map(io::maybe_trimmed) {
							tname = Some(rname);
						}
					}
					tname.unwrap()
				},
			});
			terminal::clear(None);
		}

		if server.is_none() {
			terminal::print_xy(0, 0, "No server specified.\nPlease type in the server address now: ");
			terminal::refresh();
			loop {
				if let Some(rserver) = terminal::read_str(Point::new(0, 2), terminal::state::size().width).into_iter().flat_map(io::maybe_trimmed).next() {
					server = Some(rserver);
					break;
				}
			}
			terminal::clear(None);
			terminal::refresh();
		}

		assert!(name.is_some());
		assert!(server.is_some());
		Options{
			name: name.unwrap(),
			server: server.unwrap(),
		}
	}
}

/// Extract username from last segment of user's homedir
fn username() -> Option<String> {
	match home_dir() {
		Some(pbuf) =>
			match pbuf.as_path().file_name() {
				Some(fname) => fname.to_str().map(|string| string.to_string()),
				None => None,
			},
		None => None,
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Evil private APIs.
//!
//! These are liable to change at any time. Use with caution!

use geometry::CGRect;
use std::ptr;

pub struct CGSRegion {
    region: ffi::CGSRegion,
}

impl Drop for CGSRegion {
    fn drop(&mut self) {
        unsafe {
            ffi::CGSRegionRelease(self.region)
        }
    }
}

impl CGSRegion {
    #[inline]
    pub fn from_rect(rect: &CGRect) -> CGSRegion {
        unsafe {
            let mut region = ptr::null_mut();
            assert!(ffi::CGSNewRegionWithRect(rect, &mut region) == 0);
            CGSRegion {
                region: region,
            }
        }
    }
}

mod ffi {
    use geometry::CGRect;

    // This is an enum so that we can't easily make instances of this opaque type.
    enum CGSRegionObject {}

    pub type CGSRegion = *mut CGSRegionObject;
    pub type OSStatus = i32;

    #[link(name = "ApplicationServices", kind = "framework")]
    extern {
        pub fn CGSRegionRelease(region: CGSRegion);
        pub fn CGSNewRegionWithRect(rect: *const CGRect, region: *mut CGSRegion) -> OSStatus;
    }
}

<commit_msg>private: Rename `CGSRegion` types to match reverse engineered headers available elsewhere.<commit_after>// Copyright 2016 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Evil private APIs.
//!
//! These are liable to change at any time. Use with caution!

use geometry::CGRect;
use std::ptr;

pub struct CGSRegion {
    region: ffi::CGSRegionRef,
}

impl Drop for CGSRegion {
    fn drop(&mut self) {
        unsafe {
            ffi::CGSRegionRelease(self.region)
        }
    }
}

impl CGSRegion {
    #[inline]
    pub fn from_rect(rect: &CGRect) -> CGSRegion {
        unsafe {
            let mut region = ptr::null_mut();
            assert!(ffi::CGSNewRegionWithRect(rect, &mut region) == 0);
            CGSRegion {
                region: region,
            }
        }
    }
}

mod ffi {
    use geometry::CGRect;

    // This is an enum so that we can't easily make instances of this opaque type.
    enum CGSRegionObject {}

    pub type CGError = OSStatus;
    pub type CGSRegionRef = *mut CGSRegionObject;
    pub type OSStatus = i32;

    #[link(name = "ApplicationServices", kind = "framework")]
    extern {
        pub fn CGSRegionRelease(region: CGSRegionRef);
        pub fn CGSNewRegionWithRect(rect: *const CGRect, outRegion: *mut CGSRegionRef) -> CGError;
    }
}

<|endoftext|>


<commit_before>extern crate log;

pub use cli::CliConfig;
pub use configuration::Configuration as Cfg;

use std::io::stderr;
use std::io::Write;
use log::{LogRecord, LogLevel, LogLevelFilter, LogMetadata, SetLoggerError};

pub struct ImagLogger {
    lvl: LogLevel,
}

impl ImagLogger {

    pub fn new(lvl: LogLevel) -> ImagLogger {
        ImagLogger {
            lvl: lvl,
        }
    }

    pub fn init(cfg: &Cfg, config: &CliConfig) -> Result<(), SetLoggerError> {
        let lvl = if config.is_debugging() || cfg.is_debugging() {
            LogLevelFilter::Debug
        } else if config.is_verbose() || cfg.is_debugging() {
            LogLevelFilter::Info
        } else {
            LogLevelFilter::Error
        };

        log::set_logger(|max_log_lvl| {
            max_log_lvl.set(lvl);
            debug!("Init logger with: {}", lvl);
            Box::new(ImagLogger::new(lvl.to_log_level().unwrap()))
        })
    }

}

impl log::Log for ImagLogger {

    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.lvl
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            println!("[{}]: {}", record.level(), record.args());
        }
    }
}

pub struct Runtime<'a> {
    pub config : CliConfig<'a>,
    pub configuration : Cfg,
}

impl<'a> Runtime<'a> {

    pub fn new(cfg: Cfg, config : CliConfig<'a>) -> Runtime<'a> {
        Runtime {
            config: config,
            configuration: cfg,
        }
    }

    pub fn is_verbose(&self) -> bool {
        self.config.is_verbose() || self.configuration.is_verbose()
    }

    pub fn is_debugging(&self) -> bool {
        self.config.is_debugging() || self.configuration.is_verbose()
    }

    pub fn get_rtp(&self) -> String {
        if let Some(rtp) = self.config.get_rtp() {
            rtp
        } else {
            self.configuration.get_rtp()
        }
    }

}
<commit_msg>Fix: unused import<commit_after>extern crate log;

pub use cli::CliConfig;
pub use configuration::Configuration as Cfg;

use log::{LogRecord, LogLevel, LogLevelFilter, LogMetadata, SetLoggerError};

pub struct ImagLogger {
    lvl: LogLevel,
}

impl ImagLogger {

    pub fn new(lvl: LogLevel) -> ImagLogger {
        ImagLogger {
            lvl: lvl,
        }
    }

    pub fn init(cfg: &Cfg, config: &CliConfig) -> Result<(), SetLoggerError> {
        let lvl = if config.is_debugging() || cfg.is_debugging() {
            LogLevelFilter::Debug
        } else if config.is_verbose() || cfg.is_debugging() {
            LogLevelFilter::Info
        } else {
            LogLevelFilter::Error
        };

        log::set_logger(|max_log_lvl| {
            max_log_lvl.set(lvl);
            debug!("Init logger with: {}", lvl);
            Box::new(ImagLogger::new(lvl.to_log_level().unwrap()))
        })
    }

}

impl log::Log for ImagLogger {

    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= self.lvl
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            println!("[{}]: {}", record.level(), record.args());
        }
    }
}

pub struct Runtime<'a> {
    pub config : CliConfig<'a>,
    pub configuration : Cfg,
}

impl<'a> Runtime<'a> {

    pub fn new(cfg: Cfg, config : CliConfig<'a>) -> Runtime<'a> {
        Runtime {
            config: config,
            configuration: cfg,
        }
    }

    pub fn is_verbose(&self) -> bool {
        self.config.is_verbose() || self.configuration.is_verbose()
    }

    pub fn is_debugging(&self) -> bool {
        self.config.is_debugging() || self.configuration.is_verbose()
    }

    pub fn get_rtp(&self) -> String {
        if let Some(rtp) = self.config.get_rtp() {
            rtp
        } else {
            self.configuration.get_rtp()
        }
    }

}
<|endoftext|>


<commit_before>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
pub struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

pub type Lexeme = String;

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    pub lexeme: Lexeme,
    /// Position of the first character of the token
    pub position: Position,
}

#[derive(Debug,Clone)]
pub struct ScannerError {
    message: String,
}

impl ScannerError {
    fn from_message(message: String) -> ScannerError {
        ScannerError { message: message }
    }
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                } else {
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, ScannerError> {
        let initial_line = self.current_position.line;
        self.advance_while(&|c| c != '"');
        if !self.advance_if_match('"') {
            return Err(ScannerError::from_message(format!("Unterminated string at line {}",
                                                          initial_line)));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Option<Result<TokenWithContext, ScannerError>> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => return None,
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => {
                match self.string() {
                    Ok(token) => token,
                    Err(error) => return Some(Err(error)),
                }
            }
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Some(Err(ScannerError::from_message(format!("Unexpected character {} \
                                                                    at line {}, column {}",
                                                                   c,
                                                                   self.current_position.line,
                                                                   self.current_position.column -
                                                                   1))));
            }
        };
        Some(Ok(self.add_context(token, initial_position)))
    }
}

pub fn scan(source: &String) -> (Vec<TokenWithContext>, Vec<ScannerError>) {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = scanner.scan_next() {
        match result {
            Ok(token_with_context) => {
                match token_with_context.token {
                    // Ignoring tokens we don't care about
                    Token::Comment => {}
                    Token::Whitespace => {}
                    _ => tokens.push(token_with_context),
                };
            }
            Err(error) => errors.push(error),
        }
    }
    (tokens, errors)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let (tokens, _) = scan(&"+".into());
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let (tokens, _) = scan(&"1+2".into());
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn expression_with_whitespaces() {
        let (tokens, _) = scan(&"1 + 2".into());
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn assignement_with_comment() {
        let (tokens, _) = scan(&"var a = 1.0; // A comment".into());
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
    }

    #[test]
    fn multiline_statements() {
        let (tokens, _) = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
            .into());
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<commit_msg>Improved scanner errors<commit_after>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
pub struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

pub type Lexeme = String;

#[derive(Debug)]
pub struct TokenWithContext {
    pub token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    pub lexeme: Lexeme,
    /// Position of the first character of the token
    pub position: Position,
}

#[derive(Debug,Clone)]
pub enum ScannerError {
    MissingStringTerminator(Position),
    UnexpectedCharacter(char, Position),
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                } else {
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn advance_while(&mut self, condition: &Fn(char) -> bool) -> () {
        while self.peek_check(condition) {
            self.advance();
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, ScannerError> {
        self.advance_while(&|c| c != '"');
        if !self.advance_if_match('"') {
            return Err(ScannerError::MissingStringTerminator(self.current_position));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        self.advance_while(&is_digit);
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            self.advance_while(&is_digit);
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        self.advance_while(&is_alphanumeric);
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Option<Result<TokenWithContext, ScannerError>> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => return None,
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    self.advance_while(&|c| c != '\n');
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => {
                match self.string() {
                    Ok(token) => token,
                    Err(error) => return Some(Err(error)),
                }
            }
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                // TODO: check if position is off by one
                return Some(Err(ScannerError::UnexpectedCharacter(c, self.current_position)));
            }
        };
        Some(Ok(self.add_context(token, initial_position)))
    }
}

pub fn scan(source: &String) -> (Vec<TokenWithContext>, Vec<ScannerError>) {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut errors = Vec::new();
    while let Some(result) = scanner.scan_next() {
        match result {
            Ok(token_with_context) => {
                match token_with_context.token {
                    // Ignoring tokens we don't care about
                    Token::Comment => {}
                    Token::Whitespace => {}
                    _ => tokens.push(token_with_context),
                };
            }
            Err(error) => errors.push(error),
        }
    }
    (tokens, errors)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let (tokens, _) = scan(&"+".into());
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let (tokens, _) = scan(&"1+2".into());
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn expression_with_whitespaces() {
        let (tokens, _) = scan(&"1 + 2".into());
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
    }

    #[test]
    fn assignement_with_comment() {
        let (tokens, _) = scan(&"var a = 1.0; // A comment".into());
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
    }

    #[test]
    fn multiline_statements() {
        let (tokens, _) = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
            .into());
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<|endoftext|>


<commit_before>//! Learning algorithms implementing `Teacher` trait

use Teacher;
use Model;
use Cost;
use linear_algebra::Vector;

/// Calculates annealed learning rate
///
/// Smaller `t` will decrease the learning rate faster
/// After `t` events the start learning rate will be a half of `start`,
/// after two times `t` events the learning rate will be one third of `start`,
/// and so on.
fn annealed_learning_rate(num_events: usize, start: f64, t: f64) -> f64 {
    start / (1.0 + num_events as f64 / t)
}

/// Value of the gradient of the cost function (i.e. the cost function
/// derived by the n-th coefficient at x expressed in Error(x) and dY(x)/dx
///
/// This method is called by stochastic gradient descent (SGD)-based
/// training algorithm in order to determine the delta of the coefficients
fn gradient<T, P, C>(cost: &C, prediction: &P, truth: T, derivative_of_model: &P) -> f64
    where C: Cost<T, P>,
          P: Vector
{
    cost.outer_derivative(&prediction, truth).dot(derivative_of_model)
}

/// Gradient descent
///
/// Simplest possible implementation of gradient descent with fixed learning rate
pub struct GradientDescent {
    /// Defines how fast the coefficients of the trained `Model` will change
    pub learning_rate: f64,
}

impl<M> Teacher<M> for GradientDescent
    where M: Model,
          M::Target: Vector
{
    type Training = ();

    fn new_training(&self, _: &M) -> () {
        ()
    }

    fn teach_event<Y, C>(&self,
                         _training: &mut (),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let prediction = model.predict(features);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) =
                *model.coefficient(ci) -
                self.learning_rate *
                gradient(cost, &prediction, truth, &model.gradient(ci, features));
        }
    }
}

/// Gradient descent with annealing learning rate
///
/// For the i-th event the learning rate is `l = l0 * (1 + i/t)`
pub struct GradientDescentAl {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`
    /// and so on.
    pub t: f64,
}

impl<M> Teacher<M> for GradientDescentAl
    where M: Model,
          M::Target: Vector
{
    type Training = usize;

    fn new_training(&self, _: &M) -> usize {
        0
    }

    fn teach_event<Y, C>(&self,
                         num_events: &mut usize,
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) =
                *model.coefficient(ci) -
                learning_rate * gradient(cost, &prediction, truth, &model.gradient(ci, features));
        }
        *num_events += 1;
    }
}

/// Gradient descent with annealing learning rate and momentum
///
/// For the i-th event the learning rate is `l = l0 * (1 + i/t)`
pub struct Momentum {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`,
    /// and so on.
    pub t: f64,
    /// To simulate friction, please select a value smaller than 1 (recommended)
    pub inertia: f64,
}

impl<M> Teacher<M> for Momentum
    where M: Model,
          M::Target: Vector
{
    type Training = (usize, Vec<f64>);

    fn new_training(&self, model: &M) -> (usize, Vec<f64>) {

        let mut velocity = Vec::with_capacity(model.num_coefficients());
        velocity.resize(model.num_coefficients(), 0.0);

        (0, velocity)
    }

    fn teach_event<Y, C>(&self,
                         training: &mut (usize, Vec<f64>),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let (ref mut num_events, ref mut velocity) = *training;
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            velocity[ci] = self.inertia * velocity[ci] -
                           learning_rate *
                           gradient(cost, &prediction, truth, &model.gradient(ci, features));
            *model.coefficient(ci) = *model.coefficient(ci) + velocity[ci];
        }
        *num_events += 1;
    }
}

/// Nesterov accelerated gradient descent
///
/// Like accelerated gradient descent, Nesterov accelerated gradient descent
/// includes a momentum term. In contrast to regular gradient descent, the
/// acceleration is not calculated with respect to the current position, but to
/// the estimated new one.
/// Source:
/// [G. Hinton's lecture 6c]
/// (http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf)
pub struct Nesterov {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`,
    /// and so on.
    pub t: f64,
    /// To simulate friction, please select a value smaller than 1 (recommended)
    pub inertia: f64,
}

impl<M> Teacher<M> for Nesterov
    where M: Model,
          M::Target: Vector
{
    type Training = (usize, Vec<f64>);

    fn new_training(&self, model: &M) -> (usize, Vec<f64>) {

        let mut velocity = Vec::with_capacity(model.num_coefficients());
        velocity.resize(model.num_coefficients(), 0.0);

        (0, velocity)
    }

    fn teach_event<Y, C>(&self,
                         training: &mut (usize, Vec<f64>),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let (ref mut num_events, ref mut velocity) = *training;
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) = *model.coefficient(ci) + velocity[ci];
        }
        for ci in 0..model.num_coefficients() {
            let delta = -learning_rate *
                        gradient(cost, &prediction, truth, &model.gradient(ci, features));
            *model.coefficient(ci) = *model.coefficient(ci) + delta;
            velocity[ci] = self.inertia * velocity[ci] + delta;
        }
        *num_events += 1;
    }
}

/// Adagard learning algorithm
///
/// Adagard divides the learning rate through the square root of the square sum of gradients for
/// each coefficient. In effect the learning rate is smaller for frequent and larger for infrequent
/// features.
/// See [this paper](http://jmlr.org/papers/v12/duchi11a.html) for more information.
pub struct Adagard {
    /// The larger this parameter is, the more the coefficients will change with each iteration
    pub learning_rate: f64,
    /// Small smoothing term, to avoid division by zero in first iteration
    pub epsilon: f64,
}

impl<M> Teacher<M> for Adagard
    where M: Model,
          M::Target: Vector
{
    type Training = Vec<f64>;

    fn new_training(&self, model: &M) -> Vec<f64> {

        let mut squared_gradients = Vec::with_capacity(model.num_coefficients());
        squared_gradients.resize(model.num_coefficients(), self.epsilon);
        squared_gradients
    }

    fn teach_event<Y, C>(&self,
                         squared_gradients: &mut Vec<f64>,
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {

        let prediction = model.predict(features);
        for ci in 0..model.num_coefficients() {
            let gradient = gradient(cost, &prediction, truth, &model.gradient(ci, features));
            let delta = -self.learning_rate * gradient / squared_gradients[ci].sqrt();
            *model.coefficient(ci) += delta;
            squared_gradients[ci] += gradient.powi(2);
        }
    }
}<commit_msg>simplified allocation of new vectors<commit_after>//! Learning algorithms implementing `Teacher` trait

use Teacher;
use Model;
use Cost;
use linear_algebra::Vector;

/// Calculates annealed learning rate
///
/// Smaller `t` will decrease the learning rate faster
/// After `t` events the start learning rate will be a half of `start`,
/// after two times `t` events the learning rate will be one third of `start`,
/// and so on.
fn annealed_learning_rate(num_events: usize, start: f64, t: f64) -> f64 {
    start / (1.0 + num_events as f64 / t)
}

/// Value of the gradient of the cost function (i.e. the cost function
/// derived by the n-th coefficient at x expressed in Error(x) and dY(x)/dx
///
/// This method is called by stochastic gradient descent (SGD)-based
/// training algorithm in order to determine the delta of the coefficients
fn gradient<T, P, C>(cost: &C, prediction: &P, truth: T, derivative_of_model: &P) -> f64
    where C: Cost<T, P>,
          P: Vector
{
    cost.outer_derivative(&prediction, truth).dot(derivative_of_model)
}

/// Gradient descent
///
/// Simplest possible implementation of gradient descent with fixed learning rate
pub struct GradientDescent {
    /// Defines how fast the coefficients of the trained `Model` will change
    pub learning_rate: f64,
}

impl<M> Teacher<M> for GradientDescent
    where M: Model,
          M::Target: Vector
{
    type Training = ();

    fn new_training(&self, _: &M) -> () {
        ()
    }

    fn teach_event<Y, C>(&self,
                         _training: &mut (),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let prediction = model.predict(features);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) =
                *model.coefficient(ci) -
                self.learning_rate *
                gradient(cost, &prediction, truth, &model.gradient(ci, features));
        }
    }
}

/// Gradient descent with annealing learning rate
///
/// For the i-th event the learning rate is `l = l0 * (1 + i/t)`
pub struct GradientDescentAl {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`
    /// and so on.
    pub t: f64,
}

impl<M> Teacher<M> for GradientDescentAl
    where M: Model,
          M::Target: Vector
{
    type Training = usize;

    fn new_training(&self, _: &M) -> usize {
        0
    }

    fn teach_event<Y, C>(&self,
                         num_events: &mut usize,
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) =
                *model.coefficient(ci) -
                learning_rate * gradient(cost, &prediction, truth, &model.gradient(ci, features));
        }
        *num_events += 1;
    }
}

/// Gradient descent with annealing learning rate and momentum
///
/// For the i-th event the learning rate is `l = l0 * (1 + i/t)`
pub struct Momentum {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`,
    /// and so on.
    pub t: f64,
    /// To simulate friction, please select a value smaller than 1 (recommended)
    pub inertia: f64,
}

impl<M> Teacher<M> for Momentum
    where M: Model,
          M::Target: Vector
{
    type Training = (usize, Vec<f64>);

    fn new_training(&self, model: &M) -> (usize, Vec<f64>) {
        (0, (0..model.num_coefficients()).map(|_| 0.0).collect::<Vec<_>>())
    }

    fn teach_event<Y, C>(&self,
                         training: &mut (usize, Vec<f64>),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let (ref mut num_events, ref mut velocity) = *training;
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            velocity[ci] = self.inertia * velocity[ci] -
                           learning_rate *
                           gradient(cost, &prediction, truth, &model.gradient(ci, features));
            *model.coefficient(ci) = *model.coefficient(ci) + velocity[ci];
        }
        *num_events += 1;
    }
}

/// Nesterov accelerated gradient descent
///
/// Like accelerated gradient descent, Nesterov accelerated gradient descent
/// includes a momentum term. In contrast to regular gradient descent, the
/// acceleration is not calculated with respect to the current position, but to
/// the estimated new one.
/// Source:
/// [G. Hinton's lecture 6c]
/// (http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf)
pub struct Nesterov {
    /// Start learning rate
    pub l0: f64,
    /// Smaller t will decrease the learning rate faster
    ///
    /// After t events the start learning rate will be a half `l0`,
    /// after two t events the learning rate will be one third `l0`,
    /// and so on.
    pub t: f64,
    /// To simulate friction, please select a value smaller than 1 (recommended)
    pub inertia: f64,
}

impl<M> Teacher<M> for Nesterov
    where M: Model,
          M::Target: Vector
{
    type Training = (usize, Vec<f64>);

    fn new_training(&self, model: &M) -> (usize, Vec<f64>) {
        (0, (0..model.num_coefficients()).map(|_| 0.0).collect::<Vec<_>>())
    }

    fn teach_event<Y, C>(&self,
                         training: &mut (usize, Vec<f64>),
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {
        let (ref mut num_events, ref mut velocity) = *training;
        let prediction = model.predict(features);
        let learning_rate = annealed_learning_rate(*num_events, self.l0, self.t);

        for ci in 0..model.num_coefficients() {
            *model.coefficient(ci) = *model.coefficient(ci) + velocity[ci];
        }
        for ci in 0..model.num_coefficients() {
            let delta = -learning_rate *
                        gradient(cost, &prediction, truth, &model.gradient(ci, features));
            *model.coefficient(ci) = *model.coefficient(ci) + delta;
            velocity[ci] = self.inertia * velocity[ci] + delta;
        }
        *num_events += 1;
    }
}

/// Adagard learning algorithm
///
/// Adagard divides the learning rate through the square root of the square sum of gradients for
/// each coefficient. In effect the learning rate is smaller for frequent and larger for infrequent
/// features.
/// See [this paper](http://jmlr.org/papers/v12/duchi11a.html) for more information.
pub struct Adagard {
    /// The larger this parameter is, the more the coefficients will change with each iteration
    pub learning_rate: f64,
    /// Small smoothing term, to avoid division by zero in first iteration
    pub epsilon: f64,
}

impl<M> Teacher<M> for Adagard
    where M: Model,
          M::Target: Vector
{
    type Training = Vec<f64>;

    fn new_training(&self, model: &M) -> Vec<f64> {
        (0..model.num_coefficients()).map(|_| self.epsilon).collect::<Vec<_>>()
    }

    fn teach_event<Y, C>(&self,
                         squared_gradients: &mut Vec<f64>,
                         model: &mut M,
                         cost: &C,
                         features: &M::Features,
                         truth: Y)
        where C: Cost<Y, M::Target>,
              Y: Copy
    {

        let prediction = model.predict(features);
        for ci in 0..model.num_coefficients() {
            let gradient = gradient(cost, &prediction, truth, &model.gradient(ci, features));
            let delta = -self.learning_rate * gradient / squared_gradients[ci].sqrt();
            *model.coefficient(ci) += delta;
            squared_gradients[ci] += gradient.powi(2);
        }
    }
}<|endoftext|>


<commit_before>#![warn(rust_2018_idioms)]

use proc_macro::TokenStream;
use quote::quote;
use syn::{Lit, Meta, MetaNameValue};

#[proc_macro_derive(Partial, attributes(location))]
pub fn derive_partial(tokens: TokenStream) -> TokenStream {
    let ast: syn::DeriveInput = syn::parse(tokens).unwrap();
    let mut location: Option<String> = None;
    let name = &ast.ident;

    // Iterate over the struct's #[...] attributes
    for option in ast.attrs.into_iter() {
        let option = option.parse_meta().unwrap();
        match option {
            // Match '#[ident = lit]' attributes. Match guard makes it '#[prefix = lit]'
            Meta::NameValue(MetaNameValue {
                ref path, ref lit, ..
            }) if path.is_ident("location") => {
                if let Lit::Str(lit) = lit {
                    location = Some(lit.value());
                }
            }
            _ => {} // ...
        }
    }

    let location = location.unwrap();

    let gen = quote! {
        impl<'a> ::toml_query::read::Partial<'a> for #name {
            const LOCATION : &'static str = #location;
            type  Output                  = Self;
        }
    };

    gen.into()
}
<commit_msg>Allow collapsible match for better readability<commit_after>#![warn(rust_2018_idioms)]
#![allow(clippy::collapsible_match)]

use proc_macro::TokenStream;
use quote::quote;
use syn::{Lit, Meta, MetaNameValue};

#[proc_macro_derive(Partial, attributes(location))]
pub fn derive_partial(tokens: TokenStream) -> TokenStream {
    let ast: syn::DeriveInput = syn::parse(tokens).unwrap();
    let mut location: Option<String> = None;
    let name = &ast.ident;

    // Iterate over the struct's #[...] attributes
    for option in ast.attrs.into_iter() {
        let option = option.parse_meta().unwrap();
        match option {
            // Match '#[ident = lit]' attributes. Match guard makes it '#[prefix = lit]'
            Meta::NameValue(MetaNameValue { ref path, lit, .. }) if path.is_ident("location") => {
                if let Lit::Str(lit) = lit {
                    location = Some(lit.value());
                }
            }
            _ => {} // ...
        }
    }

    let location = location.unwrap();

    let gen = quote! {
        impl<'a> ::toml_query::read::Partial<'a> for #name {
            const LOCATION : &'static str = #location;
            type  Output                  = Self;
        }
    };

    gen.into()
}
<|endoftext|>


<commit_before>use std::cmp::min;
use std::f64;

use markup5ever::{expanded_name, local_name, namespace_url, ns};
use nalgebra::{DMatrix, Dynamic, VecStorage};

use crate::document::AcquiredNodes;
use crate::drawing_ctx::DrawingCtx;
use crate::element::{ElementResult, SetAttributes};
use crate::node::{CascadedValues, Node};
use crate::parsers::{NonNegative, NumberOptionalNumber, ParseValue};
use crate::properties::ColorInterpolationFilters;
use crate::rect::IRect;
use crate::session::Session;
use crate::surface_utils::{
    shared_surface::{BlurDirection, Horizontal, SharedImageSurface, Vertical},
    EdgeMode,
};
use crate::xml::Attributes;

use super::bounds::BoundsBuilder;
use super::context::{FilterContext, FilterOutput};
use super::{
    FilterEffect, FilterError, FilterResolveError, Input, Primitive, PrimitiveParams,
    ResolvedPrimitive,
};

/// The maximum gaussian blur kernel size.
///
/// The value of 500 is used in webkit.
const MAXIMUM_KERNEL_SIZE: usize = 500;

/// The `feGaussianBlur` filter primitive.
#[derive(Default)]
pub struct FeGaussianBlur {
    base: Primitive,
    params: GaussianBlur,
}

/// Resolved `feGaussianBlur` primitive for rendering.
#[derive(Clone)]
pub struct GaussianBlur {
    pub in1: Input,
    pub std_deviation: NumberOptionalNumber<f64>,
    pub color_interpolation_filters: ColorInterpolationFilters,
}

// We need this because NumberOptionalNumber doesn't impl Default
impl Default for GaussianBlur {
    fn default() -> GaussianBlur {
        GaussianBlur {
            in1: Default::default(),
            std_deviation: NumberOptionalNumber(0.0, 0.0),
            color_interpolation_filters: Default::default(),
        }
    }
}

impl SetAttributes for FeGaussianBlur {
    fn set_attributes(&mut self, attrs: &Attributes, session: &Session) -> ElementResult {
        self.params.in1 = self.base.parse_one_input(attrs, session);

        for (attr, value) in attrs.iter() {
            if let expanded_name!("", "stdDeviation") = attr.expanded() {
                let NumberOptionalNumber(NonNegative(x), NonNegative(y)) = attr.parse(value)?;
                self.params.std_deviation = NumberOptionalNumber(x, y);
            }
        }

        Ok(())
    }
}

/// Computes a gaussian kernel line for the given standard deviation.
fn gaussian_kernel(std_deviation: f64) -> Vec<f64> {
    assert!(std_deviation > 0.0);

    // Make sure there aren't any infinities.
    let maximal_deviation = (MAXIMUM_KERNEL_SIZE / 2) as f64 / 3.0;

    // Values further away than std_deviation * 3 are too small to contribute anything meaningful.
    let radius = ((std_deviation.min(maximal_deviation) * 3.0) + 0.5) as usize;
    // Clamp the radius rather than diameter because `MAXIMUM_KERNEL_SIZE` might be even and we
    // want an odd-sized kernel.
    let radius = min(radius, (MAXIMUM_KERNEL_SIZE - 1) / 2);
    let diameter = radius * 2 + 1;

    let mut kernel = Vec::with_capacity(diameter);

    let gauss_point = |x: f64| (-x.powi(2) / (2.0 * std_deviation.powi(2))).exp();

    // Fill the matrix by doing numerical integration approximation from -2*std_dev to 2*std_dev,
    // sampling 50 points per pixel. We do the bottom half, mirror it to the top half, then compute
    // the center point. Otherwise asymmetric quantization errors will occur. The formula to
    // integrate is e^-(x^2/2s^2).
    for i in 0..diameter / 2 {
        let base_x = (diameter / 2 + 1 - i) as f64 - 0.5;

        let mut sum = 0.0;
        for j in 1..=50 {
            let r = base_x + 0.02 * f64::from(j);
            sum += gauss_point(r);
        }

        kernel.push(sum / 50.0);
    }

    // We'll compute the middle point later.
    kernel.push(0.0);

    // Mirror the bottom half to the top half.
    for i in 0..diameter / 2 {
        let x = kernel[diameter / 2 - 1 - i];
        kernel.push(x);
    }

    // Find center val -- calculate an odd number of quanta to make it symmetric, even if the
    // center point is weighted slightly higher than others.
    let mut sum = 0.0;
    for j in 0..=50 {
        let r = -0.5 + 0.02 * f64::from(j);
        sum += gauss_point(r);
    }
    kernel[diameter / 2] = sum / 51.0;

    // Normalize the distribution by scaling the total sum to 1.
    let sum = kernel.iter().sum::<f64>();
    kernel.iter_mut().for_each(|x| *x /= sum);

    kernel
}

/// Returns a size of the box blur kernel to approximate the gaussian blur.
fn box_blur_kernel_size(std_deviation: f64) -> usize {
    let d = (std_deviation * 3.0 * (2.0 * f64::consts::PI).sqrt() / 4.0 + 0.5).floor();
    let d = d.min(MAXIMUM_KERNEL_SIZE as f64);
    d as usize
}

/// Applies three box blurs to approximate the gaussian blur.
///
/// This is intended to be used in two steps, horizontal and vertical.
fn three_box_blurs<B: BlurDirection>(
    surface: &SharedImageSurface,
    bounds: IRect,
    std_deviation: f64,
) -> Result<SharedImageSurface, FilterError> {
    let d = box_blur_kernel_size(std_deviation);
    if d == 0 {
        return Ok(surface.clone());
    }

    let surface = if d % 2 == 1 {
        // Odd kernel sizes just get three successive box blurs.
        let mut surface = surface.clone();

        for _ in 0..3 {
            surface = surface.box_blur::<B>(bounds, d, d / 2)?;
        }

        surface
    } else {
        // Even kernel sizes have a more interesting scheme.
        let surface = surface.box_blur::<B>(bounds, d, d / 2)?;
        let surface = surface.box_blur::<B>(bounds, d, d / 2 - 1)?;

        let d = d + 1;
        surface.box_blur::<B>(bounds, d, d / 2)?
    };

    Ok(surface)
}

/// Applies the gaussian blur.
///
/// This is intended to be used in two steps, horizontal and vertical.
fn gaussian_blur(
    input_surface: &SharedImageSurface,
    bounds: IRect,
    std_deviation: f64,
    vertical: bool,
) -> Result<SharedImageSurface, FilterError> {
    let kernel = gaussian_kernel(std_deviation);
    let (rows, cols) = if vertical {
        (kernel.len(), 1)
    } else {
        (1, kernel.len())
    };
    let kernel = DMatrix::from_data(VecStorage::new(
        Dynamic::new(rows),
        Dynamic::new(cols),
        kernel,
    ));

    Ok(input_surface.convolve(
        bounds,
        ((cols / 2) as i32, (rows / 2) as i32),
        &kernel,
        EdgeMode::None,
    )?)
}

impl GaussianBlur {
    pub fn render(
        &self,
        bounds_builder: BoundsBuilder,
        ctx: &FilterContext,
        acquired_nodes: &mut AcquiredNodes<'_>,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterOutput, FilterError> {
        let input_1 = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in1,
            self.color_interpolation_filters,
        )?;
        let bounds: IRect = bounds_builder
            .add_input(&input_1)
            .compute(ctx)
            .clipped
            .into();

        let NumberOptionalNumber(std_x, std_y) = self.std_deviation;

        // "A negative value or a value of zero disables the effect of
        // the given filter primitive (i.e., the result is the filter
        // input image)."
        if std_x <= 0.0 && std_y <= 0.0 {
            return Ok(FilterOutput {
                surface: input_1.surface().clone(),
                bounds,
            });
        }

        let (std_x, std_y) = ctx.paffine().transform_distance(std_x, std_y);

        // The deviation can become negative here due to the transform.
        let std_x = std_x.abs();
        let std_y = std_y.abs();

        // Performance TODO: gaussian blur is frequently used for shadows, operating on SourceAlpha
        // (so the image is alpha-only). We can use this to not waste time processing the other
        // channels.

        // Horizontal convolution.
        let horiz_result_surface = if std_x >= 2.0 {
            // The spec says for deviation >= 2.0 three box blurs can be used as an optimization.
            three_box_blurs::<Horizontal>(input_1.surface(), bounds, std_x)?
        } else if std_x != 0.0 {
            gaussian_blur(input_1.surface(), bounds, std_x, false)?
        } else {
            input_1.surface().clone()
        };

        // Vertical convolution.
        let output_surface = if std_y >= 2.0 {
            // The spec says for deviation >= 2.0 three box blurs can be used as an optimization.
            three_box_blurs::<Vertical>(&horiz_result_surface, bounds, std_y)?
        } else if std_y != 0.0 {
            gaussian_blur(&horiz_result_surface, bounds, std_y, true)?
        } else {
            horiz_result_surface
        };

        Ok(FilterOutput {
            surface: output_surface,
            bounds,
        })
    }
}

impl FilterEffect for FeGaussianBlur {
    fn resolve(
        &self,
        _acquired_nodes: &mut AcquiredNodes<'_>,
        node: &Node,
    ) -> Result<ResolvedPrimitive, FilterResolveError> {
        let cascaded = CascadedValues::new_from_node(node);
        let values = cascaded.get();

        let mut params = self.params.clone();
        params.color_interpolation_filters = values.color_interpolation_filters();

        Ok(ResolvedPrimitive {
            primitive: self.base.clone(),
            params: PrimitiveParams::GaussianBlur(params),
        })
    }
}
<commit_msg>filters/gaussian_blur.rs: use set_attribute()<commit_after>use std::cmp::min;
use std::f64;

use markup5ever::{expanded_name, local_name, namespace_url, ns};
use nalgebra::{DMatrix, Dynamic, VecStorage};

use crate::document::AcquiredNodes;
use crate::drawing_ctx::DrawingCtx;
use crate::element::{set_attribute, ElementResult, SetAttributes};
use crate::node::{CascadedValues, Node};
use crate::parsers::{NumberOptionalNumber, ParseValue};
use crate::properties::ColorInterpolationFilters;
use crate::rect::IRect;
use crate::session::Session;
use crate::surface_utils::{
    shared_surface::{BlurDirection, Horizontal, SharedImageSurface, Vertical},
    EdgeMode,
};
use crate::xml::Attributes;

use super::bounds::BoundsBuilder;
use super::context::{FilterContext, FilterOutput};
use super::{
    FilterEffect, FilterError, FilterResolveError, Input, Primitive, PrimitiveParams,
    ResolvedPrimitive,
};

/// The maximum gaussian blur kernel size.
///
/// The value of 500 is used in webkit.
const MAXIMUM_KERNEL_SIZE: usize = 500;

/// The `feGaussianBlur` filter primitive.
#[derive(Default)]
pub struct FeGaussianBlur {
    base: Primitive,
    params: GaussianBlur,
}

/// Resolved `feGaussianBlur` primitive for rendering.
#[derive(Clone)]
pub struct GaussianBlur {
    pub in1: Input,
    pub std_deviation: NumberOptionalNumber<f64>,
    pub color_interpolation_filters: ColorInterpolationFilters,
}

// We need this because NumberOptionalNumber doesn't impl Default
impl Default for GaussianBlur {
    fn default() -> GaussianBlur {
        GaussianBlur {
            in1: Default::default(),
            std_deviation: NumberOptionalNumber(0.0, 0.0),
            color_interpolation_filters: Default::default(),
        }
    }
}

impl SetAttributes for FeGaussianBlur {
    fn set_attributes(&mut self, attrs: &Attributes, session: &Session) -> ElementResult {
        self.params.in1 = self.base.parse_one_input(attrs, session);

        for (attr, value) in attrs.iter() {
            if let expanded_name!("", "stdDeviation") = attr.expanded() {
                set_attribute(&mut self.params.std_deviation, attr.parse(value), session);
            }
        }

        Ok(())
    }
}

/// Computes a gaussian kernel line for the given standard deviation.
fn gaussian_kernel(std_deviation: f64) -> Vec<f64> {
    assert!(std_deviation > 0.0);

    // Make sure there aren't any infinities.
    let maximal_deviation = (MAXIMUM_KERNEL_SIZE / 2) as f64 / 3.0;

    // Values further away than std_deviation * 3 are too small to contribute anything meaningful.
    let radius = ((std_deviation.min(maximal_deviation) * 3.0) + 0.5) as usize;
    // Clamp the radius rather than diameter because `MAXIMUM_KERNEL_SIZE` might be even and we
    // want an odd-sized kernel.
    let radius = min(radius, (MAXIMUM_KERNEL_SIZE - 1) / 2);
    let diameter = radius * 2 + 1;

    let mut kernel = Vec::with_capacity(diameter);

    let gauss_point = |x: f64| (-x.powi(2) / (2.0 * std_deviation.powi(2))).exp();

    // Fill the matrix by doing numerical integration approximation from -2*std_dev to 2*std_dev,
    // sampling 50 points per pixel. We do the bottom half, mirror it to the top half, then compute
    // the center point. Otherwise asymmetric quantization errors will occur. The formula to
    // integrate is e^-(x^2/2s^2).
    for i in 0..diameter / 2 {
        let base_x = (diameter / 2 + 1 - i) as f64 - 0.5;

        let mut sum = 0.0;
        for j in 1..=50 {
            let r = base_x + 0.02 * f64::from(j);
            sum += gauss_point(r);
        }

        kernel.push(sum / 50.0);
    }

    // We'll compute the middle point later.
    kernel.push(0.0);

    // Mirror the bottom half to the top half.
    for i in 0..diameter / 2 {
        let x = kernel[diameter / 2 - 1 - i];
        kernel.push(x);
    }

    // Find center val -- calculate an odd number of quanta to make it symmetric, even if the
    // center point is weighted slightly higher than others.
    let mut sum = 0.0;
    for j in 0..=50 {
        let r = -0.5 + 0.02 * f64::from(j);
        sum += gauss_point(r);
    }
    kernel[diameter / 2] = sum / 51.0;

    // Normalize the distribution by scaling the total sum to 1.
    let sum = kernel.iter().sum::<f64>();
    kernel.iter_mut().for_each(|x| *x /= sum);

    kernel
}

/// Returns a size of the box blur kernel to approximate the gaussian blur.
fn box_blur_kernel_size(std_deviation: f64) -> usize {
    let d = (std_deviation * 3.0 * (2.0 * f64::consts::PI).sqrt() / 4.0 + 0.5).floor();
    let d = d.min(MAXIMUM_KERNEL_SIZE as f64);
    d as usize
}

/// Applies three box blurs to approximate the gaussian blur.
///
/// This is intended to be used in two steps, horizontal and vertical.
fn three_box_blurs<B: BlurDirection>(
    surface: &SharedImageSurface,
    bounds: IRect,
    std_deviation: f64,
) -> Result<SharedImageSurface, FilterError> {
    let d = box_blur_kernel_size(std_deviation);
    if d == 0 {
        return Ok(surface.clone());
    }

    let surface = if d % 2 == 1 {
        // Odd kernel sizes just get three successive box blurs.
        let mut surface = surface.clone();

        for _ in 0..3 {
            surface = surface.box_blur::<B>(bounds, d, d / 2)?;
        }

        surface
    } else {
        // Even kernel sizes have a more interesting scheme.
        let surface = surface.box_blur::<B>(bounds, d, d / 2)?;
        let surface = surface.box_blur::<B>(bounds, d, d / 2 - 1)?;

        let d = d + 1;
        surface.box_blur::<B>(bounds, d, d / 2)?
    };

    Ok(surface)
}

/// Applies the gaussian blur.
///
/// This is intended to be used in two steps, horizontal and vertical.
fn gaussian_blur(
    input_surface: &SharedImageSurface,
    bounds: IRect,
    std_deviation: f64,
    vertical: bool,
) -> Result<SharedImageSurface, FilterError> {
    let kernel = gaussian_kernel(std_deviation);
    let (rows, cols) = if vertical {
        (kernel.len(), 1)
    } else {
        (1, kernel.len())
    };
    let kernel = DMatrix::from_data(VecStorage::new(
        Dynamic::new(rows),
        Dynamic::new(cols),
        kernel,
    ));

    Ok(input_surface.convolve(
        bounds,
        ((cols / 2) as i32, (rows / 2) as i32),
        &kernel,
        EdgeMode::None,
    )?)
}

impl GaussianBlur {
    pub fn render(
        &self,
        bounds_builder: BoundsBuilder,
        ctx: &FilterContext,
        acquired_nodes: &mut AcquiredNodes<'_>,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterOutput, FilterError> {
        let input_1 = ctx.get_input(
            acquired_nodes,
            draw_ctx,
            &self.in1,
            self.color_interpolation_filters,
        )?;
        let bounds: IRect = bounds_builder
            .add_input(&input_1)
            .compute(ctx)
            .clipped
            .into();

        let NumberOptionalNumber(std_x, std_y) = self.std_deviation;

        // "A negative value or a value of zero disables the effect of
        // the given filter primitive (i.e., the result is the filter
        // input image)."
        if std_x <= 0.0 && std_y <= 0.0 {
            return Ok(FilterOutput {
                surface: input_1.surface().clone(),
                bounds,
            });
        }

        let (std_x, std_y) = ctx.paffine().transform_distance(std_x, std_y);

        // The deviation can become negative here due to the transform.
        let std_x = std_x.abs();
        let std_y = std_y.abs();

        // Performance TODO: gaussian blur is frequently used for shadows, operating on SourceAlpha
        // (so the image is alpha-only). We can use this to not waste time processing the other
        // channels.

        // Horizontal convolution.
        let horiz_result_surface = if std_x >= 2.0 {
            // The spec says for deviation >= 2.0 three box blurs can be used as an optimization.
            three_box_blurs::<Horizontal>(input_1.surface(), bounds, std_x)?
        } else if std_x != 0.0 {
            gaussian_blur(input_1.surface(), bounds, std_x, false)?
        } else {
            input_1.surface().clone()
        };

        // Vertical convolution.
        let output_surface = if std_y >= 2.0 {
            // The spec says for deviation >= 2.0 three box blurs can be used as an optimization.
            three_box_blurs::<Vertical>(&horiz_result_surface, bounds, std_y)?
        } else if std_y != 0.0 {
            gaussian_blur(&horiz_result_surface, bounds, std_y, true)?
        } else {
            horiz_result_surface
        };

        Ok(FilterOutput {
            surface: output_surface,
            bounds,
        })
    }
}

impl FilterEffect for FeGaussianBlur {
    fn resolve(
        &self,
        _acquired_nodes: &mut AcquiredNodes<'_>,
        node: &Node,
    ) -> Result<ResolvedPrimitive, FilterResolveError> {
        let cascaded = CascadedValues::new_from_node(node);
        let values = cascaded.get();

        let mut params = self.params.clone();
        params.color_interpolation_filters = values.color_interpolation_filters();

        Ok(ResolvedPrimitive {
            primitive: self.base.clone(),
            params: PrimitiveParams::GaussianBlur(params),
        })
    }
}
<|endoftext|>


<commit_before>extern crate chrono;

use flowgger::Decoder;
use flowgger::config::Config;
use flowgger::record::{Record, StructuredData};
use self::chrono::DateTime;

#[derive(Clone)]
pub struct LTSVDecoder;

impl LTSVDecoder {
    pub fn new(config: &Config) -> LTSVDecoder {
        let _ = config;
        LTSVDecoder
    }
}

impl Decoder for LTSVDecoder {
    fn decode(&self, line: &str) -> Result<Record, &'static str> {
        let mut sd = StructuredData::new(None);
        let mut ts = None;
        let mut hostname = None;
        for part in line.split('\t') {
            let mut pair = part.splitn(2, ':');
            let name = try!(pair.next().ok_or("Missing name in an LTSV record"));
            let value = try!(pair.next().ok_or("Missing value in an LTSV record"));
            match name {
                "time" => {
                    let ts_s = if value.starts_with('[') && value.ends_with(']') {
                        &value[1..(value.len() -1)]
                    } else {
                        value
                    };
                    ts = Some(try!(parse_ts(ts_s)));
                },
                "host" => {
                    hostname = Some(value.to_owned())
                },
                name @ _ => {
                    sd.pairs.push((format!("_{}", name), value.to_owned()));
                }
            };
        }
        let record = Record {
            pri: None,
            ts: try!(ts.ok_or("Missing timestamp")),
            hostname: try!(hostname.ok_or("Missing hostname")),
            appname: None,
            procid: None,
            msgid: None,
            sd: if sd.pairs.is_empty() {
                    None
                } else {
                    Some(sd)
                },
            msg: None
        };
        Ok(record)
    }
}

fn rfc3339_to_unix(rfc3339: &str) -> Result<i64, &'static str> {
    match DateTime::parse_from_rfc3339(rfc3339) {
        Ok(date) => Ok(date.timestamp()),
        Err(_) => Err("Unable to parse the date")
    }
}

fn english_time_to_unix(et: &str) -> Result<i64, &'static str> {
    match DateTime::parse_from_str(et, "%e/%b/%Y:%H:%M:%S %z") {
        Ok(date) => Ok(date.timestamp()),
        Err(_) => Err("Unable to parse the date")
    }
}

fn parse_ts(line: &str) -> Result<i64, &'static str> {
    rfc3339_to_unix(line).or(english_time_to_unix(line))
}

#[test]
fn test_ltsv() {
    let msg = "time:[2015-08-05T15:53:45.637824Z]\thost:testhostname\tname1:value1\tname 2: value 2\tn3:v3";
    let res = LTSVDecoder.decode(msg).unwrap();
    assert!(res.ts == 1438790025);

    let msg = "time:[10/Oct/2000:13:55:36 -0700]\thost:testhostname\tname1:value1\tname 2: value 2\tn3:v3";
    let res = LTSVDecoder.decode(msg).unwrap();
    assert!(res.ts == 971211336);

    assert!(res.hostname == "testhostname");
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_name1" && v == "value1"));
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_name 2" && v == " value 2"));
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_n3" && v == "v3"));
}
<commit_msg>LTSV decoder: make "message" a special, non-sd property<commit_after>extern crate chrono;

use flowgger::Decoder;
use flowgger::config::Config;
use flowgger::record::{Record, StructuredData};
use self::chrono::DateTime;

#[derive(Clone)]
pub struct LTSVDecoder;

impl LTSVDecoder {
    pub fn new(config: &Config) -> LTSVDecoder {
        let _ = config;
        LTSVDecoder
    }
}

impl Decoder for LTSVDecoder {
    fn decode(&self, line: &str) -> Result<Record, &'static str> {
        let mut sd = StructuredData::new(None);
        let mut ts = None;
        let mut hostname = None;
        let mut msg = None;
        for part in line.split('\t') {
            let mut pair = part.splitn(2, ':');
            let name = try!(pair.next().ok_or("Missing name in an LTSV record"));
            let value = try!(pair.next().ok_or("Missing value in an LTSV record"));
            match name {
                "time" => {
                    let ts_s = if value.starts_with('[') && value.ends_with(']') {
                        &value[1..(value.len() -1)]
                    } else {
                        value
                    };
                    ts = Some(try!(parse_ts(ts_s)));
                },
                "host" => {
                    hostname = Some(value.to_owned())
                },
                "message" => {
                    msg = Some(value.to_owned());
                },
                name @ _ => {
                    sd.pairs.push((format!("_{}", name), value.to_owned()));
                }
            };
        }
        let record = Record {
            pri: None,
            ts: try!(ts.ok_or("Missing timestamp")),
            hostname: try!(hostname.ok_or("Missing hostname")),
            appname: None,
            procid: None,
            msgid: None,
            sd: if sd.pairs.is_empty() {
                    None
                } else {
                    Some(sd)
                },
            msg: msg
        };
        Ok(record)
    }
}

fn rfc3339_to_unix(rfc3339: &str) -> Result<i64, &'static str> {
    match DateTime::parse_from_rfc3339(rfc3339) {
        Ok(date) => Ok(date.timestamp()),
        Err(_) => Err("Unable to parse the date")
    }
}

fn english_time_to_unix(et: &str) -> Result<i64, &'static str> {
    match DateTime::parse_from_str(et, "%e/%b/%Y:%H:%M:%S %z") {
        Ok(date) => Ok(date.timestamp()),
        Err(_) => Err("Unable to parse the date")
    }
}

fn parse_ts(line: &str) -> Result<i64, &'static str> {
    rfc3339_to_unix(line).or(english_time_to_unix(line))
}

#[test]
fn test_ltsv() {
    let msg = "time:[2015-08-05T15:53:45.637824Z]\thost:testhostname\tname1:value1\tname 2: value 2\tn3:v3";
    let res = LTSVDecoder.decode(msg).unwrap();
    assert!(res.ts == 1438790025);

    let msg = "time:[10/Oct/2000:13:55:36 -0700]\thost:testhostname\tname1:value1\tname 2: value 2\tn3:v3";
    let res = LTSVDecoder.decode(msg).unwrap();
    assert!(res.ts == 971211336);

    assert!(res.hostname == "testhostname");
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_name1" && v == "value1"));
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_name 2" && v == " value 2"));
    assert!(pairs.iter().cloned().any(|(k, v)| k == "_n3" && v == "v3"));
}
<|endoftext|>