<commit_before>// This lexer is based on the grammar I found here: https://github.com/antlr/grammars-v4/blob/master/asm6502/asm6502.g4
// It looks like it matches the various 6502 assembly examples I have seen online.

use std;
use std::error::Error;
use std::fs::File;
use std::io::{self, Read};
use std::str;

use assembler::token::Token;
use ::opcodes::OpCode;

#[derive(Debug, PartialEq)]
pub struct LexerError {
    pub message: String,
}

impl LexerError {
    fn unexpected_ident<A, B>(expected: A, found: B) -> LexerError
        where A: std::fmt::Display,
              B: std::fmt::Display
    {
        LexerError {
            message: format!("ERR: Unexpected identifier. Found '{}', expected '{}'",
                             found,
                             expected),
        }
    }
}

impl From<std::io::Error> for LexerError {
    fn from(error: std::io::Error) -> LexerError {
        LexerError { message: error.description().into() }
    }
}

impl From<String> for LexerError {
    fn from(error: String) -> LexerError {
        LexerError { message: error }
    }
}

impl<'a> From<&'a str> for LexerError {
    fn from(error: &str) -> LexerError {
        LexerError { message: error.into() }
    }
}

/// Lexer accepts the program code as a string
/// and converts it to a list of Tokens
pub struct Lexer;

impl Lexer {
    /// Returns a vector of Tokens given an input of
    /// 6502 assembly code
    pub fn lex_string<S>(input: S) -> Result<Vec<Vec<Token>>, LexerError>
        where S: Into<String>
    {
        Ok(Self::lex(input.into())?)
    }

    /// Returns a vector of Tokens given a file
    /// to load 6502 assembly code from
    pub fn lex_file<P>(path: P) -> Result<Vec<Vec<Token>>, LexerError>
        where P: AsRef<std::path::Path>
    {
        let mut file = File::open(&path)?;
        let mut contents = String::new();

        file.read_to_string(&mut contents)?;

        Ok(Self::lex(contents)?)
    }

    /// Performs the bulk of the lexing logic
    fn lex(source: String) -> Result<Vec<Vec<Token>>, LexerError> {

        let mut result = Vec::new();

        for line in source.lines() {
            if line.trim().len() == 0 {
                continue;
            }
            let mut tokens = Vec::new();
            let mut idx = 0;
            let line = &line[..].as_bytes();
            while idx < line.len() {
                let c = line[idx] as char;
                if c.is_alphanumeric() {
                    let token = Self::consume_alphanumeric(&mut idx, line)?;
                    tokens.push(token);
                } else if c == '#' {
                    let token = Self::consume_immediate(&mut idx, line)?;
                    tokens.push(token);
                } else if c == '.' {
                    idx += 1;
                    let token = Self::consume_alphanumeric(&mut idx, line)?;
                    if let Token::Label(label) = token {
                        tokens.push(Token::Directive(label));
                    }
                } else {
                    idx += 1;
                }
            }

            result.push(tokens);
        }

        Ok(result)
    }

    fn consume_alphanumeric(idx: &mut usize, line: &[u8]) -> Result<Token, LexerError> {
        let mut tok = String::new();

        while *idx < line.len() {
            let c = line[*idx] as char;
            if c.is_whitespace() || c == ';' || c == ':' || c == '\n' {
                return Ok(Self::classify(&tok));
            } else {
                tok.push(c);
            }

            *idx += 1;
        }

        Err("ERR: An error occurred while consuming an alphanumeric identifier".into())
    }

    fn consume_immediate(idx: &mut usize, line: &[u8]) -> Result<Token, LexerError> {
        let mut tok = String::new();
        // Very next character should be a dollar sign
        let c = line[*idx + 0x01] as char;
        if c != '$' {
            return Err(LexerError::unexpected_ident('$', c));
        }

        let c = line[*idx + 0x02] as char;
        if c.is_digit(16) {
            tok.push(c as char);
        } else {
            return Err(LexerError::unexpected_ident("{hex_digit}", c));
        }

        let c = line[*idx + 0x03] as char;
        if c.is_digit(16) {
            let val = &line[*idx..*idx + 0x04];
            *idx += 0x04;
            return Ok(Self::classify(str::from_utf8(val).unwrap().into()));
        } else {
            return Err(LexerError::unexpected_ident("{hex_digit}", c));
        }

        Err("An error occurred while consuming an immediate identifier".into())
    }

    fn classify(input: &str) -> Token {
        let mut tok = String::from(input);
        if let Some(opcode) = OpCode::from_mnemonic(tok.clone()) {
            Token::OpCode(tok.clone())
        } else if tok.as_bytes()[0] == b'#' {
            Token::Immediate((&tok[2..]).into())
        } else {
            Token::Label(input.into())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ::assembler::token::Token;

    #[test]
    fn can_classify_simple_labels_and_opcodes() {
        let tokens = Lexer::lex_string("
            LDA #$20
        ")
            .unwrap();

        assert_eq!(&[Token::OpCode("LDA".into()), Token::Immediate("20".into())],
                   &tokens[0][..]);
    }

    #[test]
    fn does_not_classify_unknown_labels_and_opcodes() {
        let tokens = Lexer::lex_string("
            .LOL LDA #$20
        ")
            .unwrap();

        assert_eq!(&[Token::Directive("LOL".into()),
                     Token::OpCode("LDA".into()),
                     Token::Immediate("20".into())],
                   &tokens[0][..]);
    }

    #[test]
    fn immediate_values_throw_errors_when_invalid() {
        assert_eq!(Lexer::lex_string("
            LDA #$INVALID20
        "),
                   Err(LexerError::unexpected_ident("{hex_digit}", "I")));
    }

    #[test]
    fn immediate_values_throw_errors_when_invalid_hex_code() {
        assert_eq!(Lexer::lex_string("
            LDA #$2Z
        "),
                   Err(LexerError::unexpected_ident("{hex_digit}", "Z")));
    }

    #[test]
    fn immediate_values_throw_errors_when_not_even_a_hex_code() {
        assert_eq!(Lexer::lex_string("
            LDA #@hotmail.com
        "),
                   Err(LexerError::unexpected_ident("$", "@")));
    }
}
<commit_msg>Add Label test<commit_after>// This lexer is based on the grammar I found here: https://github.com/antlr/grammars-v4/blob/master/asm6502/asm6502.g4
// It looks like it matches the various 6502 assembly examples I have seen online.

use std;
use std::error::Error;
use std::fs::File;
use std::io::{self, Read};
use std::str;

use assembler::token::Token;
use ::opcodes::OpCode;

#[derive(Debug, PartialEq)]
pub struct LexerError {
    pub message: String,
}

impl LexerError {
    fn unexpected_ident<A, B>(expected: A, found: B) -> LexerError
        where A: std::fmt::Display,
              B: std::fmt::Display
    {
        LexerError {
            message: format!("ERR: Unexpected identifier. Found '{}', expected '{}'",
                             found,
                             expected),
        }
    }
}

impl From<std::io::Error> for LexerError {
    fn from(error: std::io::Error) -> LexerError {
        LexerError { message: error.description().into() }
    }
}

impl From<String> for LexerError {
    fn from(error: String) -> LexerError {
        LexerError { message: error }
    }
}

impl<'a> From<&'a str> for LexerError {
    fn from(error: &str) -> LexerError {
        LexerError { message: error.into() }
    }
}

/// Lexer accepts the program code as a string
/// and converts it to a list of Tokens
pub struct Lexer;

impl Lexer {
    /// Returns a vector of Tokens given an input of
    /// 6502 assembly code
    pub fn lex_string<S>(input: S) -> Result<Vec<Vec<Token>>, LexerError>
        where S: Into<String>
    {
        Ok(Self::lex(input.into())?)
    }

    /// Returns a vector of Tokens given a file
    /// to load 6502 assembly code from
    pub fn lex_file<P>(path: P) -> Result<Vec<Vec<Token>>, LexerError>
        where P: AsRef<std::path::Path>
    {
        let mut file = File::open(&path)?;
        let mut contents = String::new();

        file.read_to_string(&mut contents)?;

        Ok(Self::lex(contents)?)
    }

    /// Performs the bulk of the lexing logic
    fn lex(source: String) -> Result<Vec<Vec<Token>>, LexerError> {

        let mut result = Vec::new();

        for line in source.lines() {
            if line.trim().len() == 0 {
                continue;
            }
            let mut tokens = Vec::new();
            let mut idx = 0;
            let line = &line[..].as_bytes();
            while idx < line.len() {
                let c = line[idx] as char;
                if c.is_alphanumeric() {
                    let token = Self::consume_alphanumeric(&mut idx, line)?;
                    tokens.push(token);
                } else if c == '#' {
                    let token = Self::consume_immediate(&mut idx, line)?;
                    tokens.push(token);
                } else if c == '.' {
                    idx += 1;
                    let token = Self::consume_alphanumeric(&mut idx, line)?;
                    if let Token::Label(label) = token {
                        tokens.push(Token::Directive(label));
                    }
                } else {
                    idx += 1;
                }
            }

            result.push(tokens);
        }

        Ok(result)
    }

    fn consume_alphanumeric(idx: &mut usize, line: &[u8]) -> Result<Token, LexerError> {
        let mut tok = String::new();

        while *idx < line.len() {
            let c = line[*idx] as char;
            if c.is_whitespace() || c == ';' || c == ':' || c == '\n' {
                return Ok(Self::classify(&tok));
            } else {
                tok.push(c);
            }

            *idx += 1;
        }

        Err("ERR: An error occurred while consuming an alphanumeric identifier".into())
    }

    fn consume_immediate(idx: &mut usize, line: &[u8]) -> Result<Token, LexerError> {
        let mut tok = String::new();
        // Very next character should be a dollar sign
        let c = line[*idx + 0x01] as char;
        if c != '$' {
            return Err(LexerError::unexpected_ident('$', c));
        }

        let c = line[*idx + 0x02] as char;
        if c.is_digit(16) {
            tok.push(c as char);
        } else {
            return Err(LexerError::unexpected_ident("{hex_digit}", c));
        }

        let c = line[*idx + 0x03] as char;
        if c.is_digit(16) {
            let val = &line[*idx..*idx + 0x04];
            *idx += 0x04;
            return Ok(Self::classify(str::from_utf8(val).unwrap().into()));
        } else {
            return Err(LexerError::unexpected_ident("{hex_digit}", c));
        }

        Err("An error occurred while consuming an immediate identifier".into())
    }

    fn classify(input: &str) -> Token {
        let mut tok = String::from(input);
        if let Some(opcode) = OpCode::from_mnemonic(tok.clone()) {
            Token::OpCode(tok.clone())
        } else if tok.as_bytes()[0] == b'#' {
            Token::Immediate((&tok[2..]).into())
        } else {
            Token::Label(input.into())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ::assembler::token::Token;

    #[test]
    fn can_classify_simple_labels_and_opcodes() {
        let tokens = Lexer::lex_string("
            LDA #$20
        ")
            .unwrap();

        assert_eq!(&[Token::OpCode("LDA".into()), Token::Immediate("20".into())],
                   &tokens[0][..]);
    }

    #[test]
    fn does_not_classify_unknown_labels_and_opcodes() {
        let tokens = Lexer::lex_string("
            .LOL LDA #$20
        ")
            .unwrap();

        assert_eq!(&[Token::Directive("LOL".into()),
                     Token::OpCode("LDA".into()),
                     Token::Immediate("20".into())],
                   &tokens[0][..]);
    }

    #[test]
    fn can_classify_labels() {
        let tokens = Lexer::lex_string("MAIN: LDA #$20").unwrap();

        assert_eq!(&[Token::Label("MAIN".into()),
                     Token::OpCode("LDA".into()),
                     Token::Immediate("20".into())],
                   &tokens[0][..]);
    }

    #[test]
    fn immediate_values_throw_errors_when_invalid() {
        assert_eq!(Lexer::lex_string("
            LDA #$INVALID20
        "),
                   Err(LexerError::unexpected_ident("{hex_digit}", "I")));
    }

    #[test]
    fn immediate_values_throw_errors_when_invalid_hex_code() {
        assert_eq!(Lexer::lex_string("
            LDA #$2Z
        "),
                   Err(LexerError::unexpected_ident("{hex_digit}", "Z")));
    }

    #[test]
    fn immediate_values_throw_errors_when_not_even_a_hex_code() {
        assert_eq!(Lexer::lex_string("
            LDA #@hotmail.com
        "),
                   Err(LexerError::unexpected_ident("$", "@")));
    }
}
<|endoftext|>


<commit_before>use std::ffi::{OsStr, OsString};
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{SystemTime, UNIX_EPOCH};
use std::{env, fs};

/// A helper macro to `unwrap` a result except also print out details like:
///
/// * The file/line of the panic
/// * The expression that failed
/// * The error itself
///
/// This is currently used judiciously throughout the build system rather than
/// using a `Result` with `try!`, but this may change one day...
#[macro_export]
macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {}", stringify!($e), e),
        }
    };
    // it can show extra info in the second parameter
    ($e:expr, $extra:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra),
        }
    };
}

/// Reads an environment variable and adds it to dependencies.
/// Supposed to be used for all variables except those set for build scripts by cargo
/// <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts>
pub fn tracked_env_var_os<K: AsRef<OsStr> + Display>(key: K) -> Option<OsString> {
    println!("cargo:rerun-if-env-changed={}", key);
    env::var_os(key)
}

// Because Cargo adds the compiler's dylib path to our library search path, llvm-config may
// break: the dylib path for the compiler, as of this writing, contains a copy of the LLVM
// shared library, which means that when our freshly built llvm-config goes to load it's
// associated LLVM, it actually loads the compiler's LLVM. In particular when building the first
// compiler (i.e., in stage 0) that's a problem, as the compiler's LLVM is likely different from
// the one we want to use. As such, we restore the environment to what bootstrap saw. This isn't
// perfect -- we might actually want to see something from Cargo's added library paths -- but
// for now it works.
pub fn restore_library_path() {
    let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR");
    if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") {
        env::set_var(&key, &env);
    } else {
        env::remove_var(&key);
    }
}

/// Run the command, printing what we are running.
pub fn run_verbose(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    run(cmd);
}

pub fn run(cmd: &mut Command) {
    if !try_run(cmd) {
        std::process::exit(1);
    }
}

pub fn try_run(cmd: &mut Command) -> bool {
    let status = match cmd.status() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !status.success() {
        println!(
            "\n\ncommand did not execute successfully: {:?}\n\
             expected success, got: {}\n\n",
            cmd, status
        );
    }
    status.success()
}

pub fn run_suppressed(cmd: &mut Command) {
    if !try_run_suppressed(cmd) {
        std::process::exit(1);
    }
}

pub fn try_run_suppressed(cmd: &mut Command) -> bool {
    let output = match cmd.output() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !output.status.success() {
        println!(
            "\n\ncommand did not execute successfully: {:?}\n\
             expected success, got: {}\n\n\
             stdout ----\n{}\n\
             stderr ----\n{}\n\n",
            cmd,
            output.status,
            String::from_utf8_lossy(&output.stdout),
            String::from_utf8_lossy(&output.stderr)
        );
    }
    output.status.success()
}

pub fn gnu_target(target: &str) -> &str {
    match target {
        "i686-pc-windows-msvc" => "i686-pc-win32",
        "x86_64-pc-windows-msvc" => "x86_64-pc-win32",
        "i686-pc-windows-gnu" => "i686-w64-mingw32",
        "x86_64-pc-windows-gnu" => "x86_64-w64-mingw32",
        s => s,
    }
}

pub fn make(host: &str) -> PathBuf {
    if host.contains("dragonfly")
        || host.contains("freebsd")
        || host.contains("netbsd")
        || host.contains("openbsd")
    {
        PathBuf::from("gmake")
    } else {
        PathBuf::from("make")
    }
}

pub fn output(cmd: &mut Command) -> String {
    let output = match cmd.stderr(Stdio::inherit()).output() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !output.status.success() {
        panic!(
            "command did not execute successfully: {:?}\n\
             expected success, got: {}",
            cmd, output.status
        );
    }
    String::from_utf8(output.stdout).unwrap()
}

pub fn rerun_if_changed_anything_in_dir(dir: &Path) {
    let mut stack = dir
        .read_dir()
        .unwrap()
        .map(|e| e.unwrap())
        .filter(|e| &*e.file_name() != ".git")
        .collect::<Vec<_>>();
    while let Some(entry) = stack.pop() {
        let path = entry.path();
        if entry.file_type().unwrap().is_dir() {
            stack.extend(path.read_dir().unwrap().map(|e| e.unwrap()));
        } else {
            println!("cargo:rerun-if-changed={}", path.display());
        }
    }
}

/// Returns the last-modified time for `path`, or zero if it doesn't exist.
pub fn mtime(path: &Path) -> SystemTime {
    fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH)
}

/// Returns `true` if `dst` is up to date given that the file or files in `src`
/// are used to generate it.
///
/// Uses last-modified time checks to verify this.
pub fn up_to_date(src: &Path, dst: &Path) -> bool {
    if !dst.exists() {
        return false;
    }
    let threshold = mtime(dst);
    let meta = match fs::metadata(src) {
        Ok(meta) => meta,
        Err(e) => panic!("source {:?} failed to get metadata: {}", src, e),
    };
    if meta.is_dir() {
        dir_up_to_date(src, threshold)
    } else {
        meta.modified().unwrap_or(UNIX_EPOCH) <= threshold
    }
}

fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool {
    t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
        let meta = t!(e.metadata());
        if meta.is_dir() {
            dir_up_to_date(&e.path(), threshold)
        } else {
            meta.modified().unwrap_or(UNIX_EPOCH) < threshold
        }
    })
}

fn fail(s: &str) -> ! {
    println!("\n\n{}\n\n", s);
    std::process::exit(1);
}
<commit_msg>Add track_caller to `builder_helper::output`<commit_after>use std::ffi::{OsStr, OsString};
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{SystemTime, UNIX_EPOCH};
use std::{env, fs};

/// A helper macro to `unwrap` a result except also print out details like:
///
/// * The file/line of the panic
/// * The expression that failed
/// * The error itself
///
/// This is currently used judiciously throughout the build system rather than
/// using a `Result` with `try!`, but this may change one day...
#[macro_export]
macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {}", stringify!($e), e),
        }
    };
    // it can show extra info in the second parameter
    ($e:expr, $extra:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra),
        }
    };
}

/// Reads an environment variable and adds it to dependencies.
/// Supposed to be used for all variables except those set for build scripts by cargo
/// <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts>
pub fn tracked_env_var_os<K: AsRef<OsStr> + Display>(key: K) -> Option<OsString> {
    println!("cargo:rerun-if-env-changed={}", key);
    env::var_os(key)
}

// Because Cargo adds the compiler's dylib path to our library search path, llvm-config may
// break: the dylib path for the compiler, as of this writing, contains a copy of the LLVM
// shared library, which means that when our freshly built llvm-config goes to load it's
// associated LLVM, it actually loads the compiler's LLVM. In particular when building the first
// compiler (i.e., in stage 0) that's a problem, as the compiler's LLVM is likely different from
// the one we want to use. As such, we restore the environment to what bootstrap saw. This isn't
// perfect -- we might actually want to see something from Cargo's added library paths -- but
// for now it works.
pub fn restore_library_path() {
    let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR");
    if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") {
        env::set_var(&key, &env);
    } else {
        env::remove_var(&key);
    }
}

/// Run the command, printing what we are running.
pub fn run_verbose(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    run(cmd);
}

pub fn run(cmd: &mut Command) {
    if !try_run(cmd) {
        std::process::exit(1);
    }
}

pub fn try_run(cmd: &mut Command) -> bool {
    let status = match cmd.status() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !status.success() {
        println!(
            "\n\ncommand did not execute successfully: {:?}\n\
             expected success, got: {}\n\n",
            cmd, status
        );
    }
    status.success()
}

pub fn run_suppressed(cmd: &mut Command) {
    if !try_run_suppressed(cmd) {
        std::process::exit(1);
    }
}

pub fn try_run_suppressed(cmd: &mut Command) -> bool {
    let output = match cmd.output() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !output.status.success() {
        println!(
            "\n\ncommand did not execute successfully: {:?}\n\
             expected success, got: {}\n\n\
             stdout ----\n{}\n\
             stderr ----\n{}\n\n",
            cmd,
            output.status,
            String::from_utf8_lossy(&output.stdout),
            String::from_utf8_lossy(&output.stderr)
        );
    }
    output.status.success()
}

pub fn gnu_target(target: &str) -> &str {
    match target {
        "i686-pc-windows-msvc" => "i686-pc-win32",
        "x86_64-pc-windows-msvc" => "x86_64-pc-win32",
        "i686-pc-windows-gnu" => "i686-w64-mingw32",
        "x86_64-pc-windows-gnu" => "x86_64-w64-mingw32",
        s => s,
    }
}

pub fn make(host: &str) -> PathBuf {
    if host.contains("dragonfly")
        || host.contains("freebsd")
        || host.contains("netbsd")
        || host.contains("openbsd")
    {
        PathBuf::from("gmake")
    } else {
        PathBuf::from("make")
    }
}

#[track_caller]
pub fn output(cmd: &mut Command) -> String {
    let output = match cmd.stderr(Stdio::inherit()).output() {
        Ok(status) => status,
        Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)),
    };
    if !output.status.success() {
        panic!(
            "command did not execute successfully: {:?}\n\
             expected success, got: {}",
            cmd, output.status
        );
    }
    String::from_utf8(output.stdout).unwrap()
}

pub fn rerun_if_changed_anything_in_dir(dir: &Path) {
    let mut stack = dir
        .read_dir()
        .unwrap()
        .map(|e| e.unwrap())
        .filter(|e| &*e.file_name() != ".git")
        .collect::<Vec<_>>();
    while let Some(entry) = stack.pop() {
        let path = entry.path();
        if entry.file_type().unwrap().is_dir() {
            stack.extend(path.read_dir().unwrap().map(|e| e.unwrap()));
        } else {
            println!("cargo:rerun-if-changed={}", path.display());
        }
    }
}

/// Returns the last-modified time for `path`, or zero if it doesn't exist.
pub fn mtime(path: &Path) -> SystemTime {
    fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH)
}

/// Returns `true` if `dst` is up to date given that the file or files in `src`
/// are used to generate it.
///
/// Uses last-modified time checks to verify this.
pub fn up_to_date(src: &Path, dst: &Path) -> bool {
    if !dst.exists() {
        return false;
    }
    let threshold = mtime(dst);
    let meta = match fs::metadata(src) {
        Ok(meta) => meta,
        Err(e) => panic!("source {:?} failed to get metadata: {}", src, e),
    };
    if meta.is_dir() {
        dir_up_to_date(src, threshold)
    } else {
        meta.modified().unwrap_or(UNIX_EPOCH) <= threshold
    }
}

fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool {
    t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
        let meta = t!(e.metadata());
        if meta.is_dir() {
            dir_up_to_date(&e.path(), threshold)
        } else {
            meta.modified().unwrap_or(UNIX_EPOCH) < threshold
        }
    })
}

fn fail(s: &str) -> ! {
    println!("\n\n{}\n\n", s);
    std::process::exit(1);
}
<|endoftext|>


<commit_before>//! byt - os_unix
//!
//! Implement functions by targeting the terminal functionality
//! commonly offered by unix terminals.

// EXTERNS

// LIBRARY INCLUDES
use std::io::{self, Write};
use byt::libc::{
    self,
    ioctl,
    c_void,
    TIOCGWINSZ,
    winsize
};
use std::mem;

// SUBMODULES

// LOCAL INCLUDES
use byt::envs::*;

// Constants for manipulating the terminal.
const CMD_LEAD           : &str = "\x1B";  // Leader for all commands.
const CMD_CLEAR          : &str = "[2J";     // Clears the display.
const CMD_TI             : &str = "7\x1B[?47h";   // Change to termcap mode.
const CMD_TE             : &str = "[2J\x1B[?47l\x1B8";   // Change to normal mode.

pub struct Term {
    saved_config : libc::termios,
}

impl Term {
    // #################################
    // P R I V A T E  F U N C T I O N S
    // #################################
    /// Print a command to the terminal.
    fn cmd(&self, command : &str) {
        let mut handle = io::stdout();
        handle.write(CMD_LEAD.as_bytes())
              .expect("Writing command leader to stdout failed");
        handle.write(command.as_bytes())
              .expect("Writing command to stdout failed");
        handle.flush()
              .expect("Flushing stdout failed");
    }

    /// Set the terminal to the alternate full screen buffer.
    fn set_alternate(&self) {
        self.cmd(CMD_TI);
    }

    /// Set the terminal back to the normal buffer.
    fn set_normal(&self) {
        self.cmd(CMD_TE);
    }

    /// Get the current termios config.
    fn get_mode() -> libc::termios {
        unsafe {
            let mut temp : libc::termios = mem::zeroed();
            libc::tcgetattr(0, &mut temp);
            temp
        }
    }

    // ###############################
    // P U B L I C  F U N C T I O N S
    // ###############################
    pub fn new() -> Term {
        Term {
            saved_config : Term::get_mode(),
        }
    }

    /// Open the application.
    pub fn start(&self) {
        self.set_alternate();
    }

    /// Close the application.
    pub fn stop(&self) {
        self.set_normal();
    }

    /// Get the size of the terminal in rows and columns.
    pub fn get_size(&self) -> (u16, u16) {
        let (rows, cols);

        // Get the size of the terminal by calling ioctl.
        unsafe {
            let mut ws = winsize {
                ws_row: 0,
                ws_col: 0,
                ws_xpixel: 0,
                ws_ypixel: 0,
            };

            // Grab a pointer to the struct.
            let ptr : *mut c_void = &mut ws as *mut _ as *mut c_void;

            // Run ioctl to check the window size.
            // TODO: If this fails, try TIOCGSIZE.
            let result = ioctl(1, TIOCGWINSZ, ptr);

            match result {
                0 => {rows = ws.ws_row; cols = ws.ws_col},
                _ => panic!("Grabbing terminal size from ioctl failed")
            }
        }

        (rows, cols)
    }

    /// Move the terminal's cursor to a particular location.
    ///
    /// Takes input in (row, col).
    pub fn move_cursor(&self, row : u16, col : u16) {
        self.cmd(format!("{};{}H", row, col).as_str());
    }

    /// Clear the terminal's screen.
    pub fn clear_screen(&self) {
        self.cmd(CMD_CLEAR);
    }

    /// Write a string to the terminal at the cursor.
    pub fn write(&self, out : &str) {
        let mut handle = io::stdout();
        handle.write(out.as_bytes())
              .expect("Failed to write to stdout");
        handle.flush()
              .expect("Flushing to stdout failed");
    }
    /// Set the terminal's mode between `Raw` and `Cooked`
    /// modes.
    pub fn set_mode(&self, mode : TermMode) {
        let mut copy = self.saved_config;
        match mode {
            TermMode::Cooked => {
                // Don't do anything, just set it back to normal
            },
            TermMode::Raw => {
                // Most of this is borrowed from:
                // http://www.cs.uleth.ca/~holzmann/C/system/ttyraw.c
                copy.c_iflag &=
                    !(libc::BRKINT | // Turn off line break
                      libc::ICRNL  | // Don't check for parity
                      libc::INPCK  | // Don't strip characters
                      libc::ISTRIP |
                      libc::IXON);

                // give full 8 bits
                copy.c_cflag |= libc::CS8;

                // Also fix local modes
                copy.c_lflag &= !(libc::ECHO |
                                  libc::ICANON |
                                  libc::IEXTEN |
                                  libc::ISIG);

                // We only want to interpret one byte at a time
                copy.c_cc[libc::VMIN] = 1;

                // Don't wait for anything
                copy.c_cc[libc::VTIME] = 0;
            }
        }

        unsafe {
            libc::tcsetattr(0, libc::TCSAFLUSH, &copy);
        }
    }
}
<commit_msg>fix(os_unix): fix cursor move issue<commit_after>//! byt - os_unix
//!
//! Implement functions by targeting the terminal functionality
//! commonly offered by unix terminals.

// EXTERNS

// LIBRARY INCLUDES
use std::io::{self, Write};
use byt::libc::{
    self,
    ioctl,
    c_void,
    TIOCGWINSZ,
    winsize
};
use std::mem;

// SUBMODULES

// LOCAL INCLUDES
use byt::envs::*;

// Constants for manipulating the terminal.
const CMD_LEAD           : &str = "\x1B";  // Leader for all commands.
const CMD_CLEAR          : &str = "[2J";     // Clears the display.
const CMD_TI             : &str = "7\x1B[?47h";   // Change to termcap mode.
const CMD_TE             : &str = "[2J\x1B[?47l\x1B8";   // Change to normal mode.

pub struct Term {
    saved_config : libc::termios,
}

impl Term {
    // #################################
    // P R I V A T E  F U N C T I O N S
    // #################################
    /// Print a command to the terminal.
    fn cmd(&self, command : &str) {
        let mut handle = io::stdout();
        handle.write(CMD_LEAD.as_bytes())
              .expect("Writing command leader to stdout failed");
        handle.write(command.as_bytes())
              .expect("Writing command to stdout failed");
        handle.flush()
              .expect("Flushing stdout failed");
    }

    /// Set the terminal to the alternate full screen buffer.
    fn set_alternate(&self) {
        self.cmd(CMD_TI);
    }

    /// Set the terminal back to the normal buffer.
    fn set_normal(&self) {
        self.cmd(CMD_TE);
    }

    /// Get the current termios config.
    fn get_mode() -> libc::termios {
        unsafe {
            let mut temp : libc::termios = mem::zeroed();
            libc::tcgetattr(0, &mut temp);
            temp
        }
    }

    // ###############################
    // P U B L I C  F U N C T I O N S
    // ###############################
    pub fn new() -> Term {
        Term {
            saved_config : Term::get_mode(),
        }
    }

    /// Open the application.
    pub fn start(&self) {
        self.set_alternate();
    }

    /// Close the application.
    pub fn stop(&self) {
        self.set_normal();
    }

    /// Get the size of the terminal in rows and columns.
    pub fn get_size(&self) -> (u16, u16) {
        let (rows, cols);

        // Get the size of the terminal by calling ioctl.
        unsafe {
            let mut ws = winsize {
                ws_row: 0,
                ws_col: 0,
                ws_xpixel: 0,
                ws_ypixel: 0,
            };

            // Grab a pointer to the struct.
            let ptr : *mut c_void = &mut ws as *mut _ as *mut c_void;

            // Run ioctl to check the window size.
            // TODO: If this fails, try TIOCGSIZE.
            let result = ioctl(1, TIOCGWINSZ, ptr);

            match result {
                0 => {rows = ws.ws_row; cols = ws.ws_col},
                _ => panic!("Grabbing terminal size from ioctl failed")
            }
        }

        (rows, cols)
    }

    /// Move the terminal's cursor to a particular location.
    ///
    /// Takes input in (row, col).
    pub fn move_cursor(&self, row : u16, col : u16) {
        self.cmd(format!("[{};{}H", row, col).as_str());
    }

    /// Clear the terminal's screen.
    pub fn clear_screen(&self) {
        self.cmd(CMD_CLEAR);
    }

    /// Write a string to the terminal at the cursor.
    pub fn write(&self, out : &str) {
        let mut handle = io::stdout();
        handle.write(out.as_bytes())
              .expect("Failed to write to stdout");
        handle.flush()
              .expect("Flushing to stdout failed");
    }
    /// Set the terminal's mode between `Raw` and `Cooked`
    /// modes.
    pub fn set_mode(&self, mode : TermMode) {
        let mut copy = self.saved_config;
        match mode {
            TermMode::Cooked => {
                // Don't do anything, just set it back to normal
            },
            TermMode::Raw => {
                // Most of this is borrowed from:
                // http://www.cs.uleth.ca/~holzmann/C/system/ttyraw.c
                copy.c_iflag &=
                    !(libc::BRKINT | // Turn off line break
                      libc::ICRNL  | // Don't check for parity
                      libc::INPCK  | // Don't strip characters
                      libc::ISTRIP |
                      libc::IXON);

                // give full 8 bits
                copy.c_cflag |= libc::CS8;

                // Also fix local modes
                copy.c_lflag &= !(libc::ECHO |
                                  libc::ICANON |
                                  libc::IEXTEN |
                                  libc::ISIG);

                // We only want to interpret one byte at a time
                copy.c_cc[libc::VMIN] = 1;

                // Don't wait for anything
                copy.c_cc[libc::VTIME] = 0;
            }
        }

        unsafe {
            libc::tcsetattr(0, libc::TCSAFLUSH, &copy);
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

//! minigbm: implements swapchain allocation using ChromeOS's minigbm library.
//!
//! External code found at <https://chromium.googlesource.com/chromiumos/platform/minigbm>.

#![cfg(feature = "minigbm")]

use std::ffi::CStr;
use std::fs::File;
use std::io::Seek;
use std::io::SeekFrom;
use std::os::raw::c_char;
use std::sync::Arc;

use base::AsRawDescriptor;
use base::Error as BaseError;
use base::FromRawDescriptor;

use crate::rutabaga_gralloc::formats::DrmFormat;
use crate::rutabaga_gralloc::gralloc::Gralloc;
use crate::rutabaga_gralloc::gralloc::ImageAllocationInfo;
use crate::rutabaga_gralloc::gralloc::ImageMemoryRequirements;
use crate::rutabaga_gralloc::minigbm_bindings::*;
use crate::rutabaga_gralloc::rendernode;
use crate::rutabaga_utils::*;

struct MinigbmDeviceInner {
    _fd: File,
    gbm: *mut gbm_device,
}

// Safe because minigbm handles synchronization internally.
unsafe impl Send for MinigbmDeviceInner {}
unsafe impl Sync for MinigbmDeviceInner {}

impl Drop for MinigbmDeviceInner {
    fn drop(&mut self) {
        // Safe because MinigbmDeviceInner is only constructed with a valid minigbm_device.
        unsafe {
            gbm_device_destroy(self.gbm);
        }
    }
}

/// A device capable of allocating `MinigbmBuffer`.
#[derive(Clone)]
pub struct MinigbmDevice {
    minigbm_device: Arc<MinigbmDeviceInner>,
    last_buffer: Option<Arc<MinigbmBuffer>>,
    device_name: &'static str,
}

impl MinigbmDevice {
    /// Returns a new `MinigbmDevice` if there is a rendernode in `/dev/dri/` that is accepted by
    /// the minigbm library.
    pub fn init() -> RutabagaResult<Box<dyn Gralloc>> {
        let undesired: &[&str] = &["vgem", "pvr"];
        let fd = rendernode::open_device(undesired)?;

        // gbm_create_device is safe to call with a valid fd, and we check that a valid one is
        // returned.  If the fd does not refer to a DRM device, gbm_create_device will reject it.
        let gbm = unsafe { gbm_create_device(fd.as_raw_descriptor()) };
        if gbm.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        // Safe because a valid minigbm device has a statically allocated string associated with
        // it, which is valid for the lifetime of the process.
        let backend_name: *const c_char = unsafe { gbm_device_get_backend_name(gbm) };
        let c_str: &CStr = unsafe { CStr::from_ptr(backend_name) };
        let device_name: &str = c_str.to_str()?;

        Ok(Box::new(MinigbmDevice {
            minigbm_device: Arc::new(MinigbmDeviceInner { _fd: fd, gbm }),
            last_buffer: None,
            device_name,
        }))
    }
}

impl Gralloc for MinigbmDevice {
    fn supports_external_gpu_memory(&self) -> bool {
        true
    }

    fn supports_dmabuf(&self) -> bool {
        true
    }

    fn get_image_memory_requirements(
        &mut self,
        info: ImageAllocationInfo,
    ) -> RutabagaResult<ImageMemoryRequirements> {
        let bo = unsafe {
            gbm_bo_create(
                self.minigbm_device.gbm,
                info.width,
                info.height,
                info.drm_format.0,
                info.flags.0,
            )
        };
        if bo.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        let mut reqs: ImageMemoryRequirements = Default::default();
        let gbm_buffer = MinigbmBuffer(bo, self.clone());

        // Intel GPUs typically only use cached memory buffers.  This will change with dGPUs, but
        // perhaps minigbm will be deprecated by then.  Other display drivers (rockchip, mediatek,
        // amdgpu) typically use write combine memory.  We can also consider use flags too if this
        // heuristic proves insufficient.
        //
        // Existing qcom devices use a mix of cached and WC buffers.  We don't *yet* have a good
        // way to differentiate, but https://patchwork.freedesktop.org/series/106847/ is a proposal
        // to fix that.  But existing devices without the FWB feature, the resulting mapping attrs
        // are the more restrictive of the combination for S1 and S2 mappings.  So if we map as
        // cached in S2 pgtables, but S1 has WC, then the result will be WC.
        if self.device_name == "i915" || self.device_name == "msm" {
            reqs.map_info = RUTABAGA_MAP_CACHE_CACHED;
        } else {
            reqs.map_info = RUTABAGA_MAP_CACHE_WC;
        }

        reqs.modifier = gbm_buffer.format_modifier();
        for plane in 0..gbm_buffer.num_planes() {
            reqs.strides[plane] = gbm_buffer.plane_stride(plane);
            reqs.offsets[plane] = gbm_buffer.plane_offset(plane);
        }

        let mut fd = gbm_buffer.export()?;
        let size = fd.seek(SeekFrom::End(0))?;

        // minigbm does have the ability to query image requirements without allocating memory
        // via the TEST_ALLOC flag.  However, support has only been added in i915.  Until this
        // flag is supported everywhere, do the actual allocation here and stash it away.
        if self.last_buffer.is_some() {
            return Err(RutabagaError::AlreadyInUse);
        }

        self.last_buffer = Some(Arc::new(gbm_buffer));
        reqs.info = info;
        reqs.size = size;
        Ok(reqs)
    }

    fn allocate_memory(&mut self, reqs: ImageMemoryRequirements) -> RutabagaResult<RutabagaHandle> {
        let last_buffer = self.last_buffer.take();
        if let Some(gbm_buffer) = last_buffer {
            if gbm_buffer.width() != reqs.info.width
                || gbm_buffer.height() != reqs.info.height
                || gbm_buffer.format() != reqs.info.drm_format
            {
                return Err(RutabagaError::InvalidGrallocDimensions);
            }

            let dmabuf = gbm_buffer.export()?.into();
            return Ok(RutabagaHandle {
                os_handle: dmabuf,
                handle_type: RUTABAGA_MEM_HANDLE_TYPE_DMABUF,
            });
        }

        let bo = unsafe {
            gbm_bo_create(
                self.minigbm_device.gbm,
                reqs.info.width,
                reqs.info.height,
                reqs.info.drm_format.0,
                reqs.info.flags.0,
            )
        };

        if bo.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        let gbm_buffer = MinigbmBuffer(bo, self.clone());
        let dmabuf = gbm_buffer.export()?.into();
        Ok(RutabagaHandle {
            os_handle: dmabuf,
            handle_type: RUTABAGA_MEM_HANDLE_TYPE_DMABUF,
        })
    }
}

/// An allocation from a `MinigbmDevice`.
pub struct MinigbmBuffer(*mut gbm_bo, MinigbmDevice);

// Safe because minigbm handles synchronization internally.
unsafe impl Send for MinigbmBuffer {}
unsafe impl Sync for MinigbmBuffer {}

impl MinigbmBuffer {
    /// Width in pixels.
    pub fn width(&self) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_width(self.0) }
    }

    /// Height in pixels.
    pub fn height(&self) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_height(self.0) }
    }

    /// `DrmFormat` of the buffer.
    pub fn format(&self) -> DrmFormat {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { DrmFormat(gbm_bo_get_format(self.0)) }
    }

    /// DrmFormat modifier flags for the buffer.
    pub fn format_modifier(&self) -> u64 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_modifier(self.0) }
    }

    /// Number of planes present in this buffer.
    pub fn num_planes(&self) -> usize {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_plane_count(self.0) as usize }
    }

    /// Offset in bytes for the given plane.
    pub fn plane_offset(&self, plane: usize) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_offset(self.0, plane) }
    }

    /// Length in bytes of one row for the given plane.
    pub fn plane_stride(&self, plane: usize) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_stride_for_plane(self.0, plane) }
    }

    /// Exports a new dmabuf/prime file descriptor.
    pub fn export(&self) -> RutabagaResult<File> {
        // This is always safe to call with a valid gbm_bo pointer.
        match unsafe { gbm_bo_get_fd(self.0) } {
            fd if fd >= 0 => {
                let dmabuf = unsafe { File::from_raw_descriptor(fd) };
                Ok(dmabuf)
            }
            ret => Err(RutabagaError::ComponentError(ret)),
        }
    }
}

impl Drop for MinigbmBuffer {
    fn drop(&mut self) {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_destroy(self.0) }
    }
}
<commit_msg>Revert "rutabaga_gfx: Support for cached mappings on msm"<commit_after>// Copyright 2018 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

//! minigbm: implements swapchain allocation using ChromeOS's minigbm library.
//!
//! External code found at <https://chromium.googlesource.com/chromiumos/platform/minigbm>.

#![cfg(feature = "minigbm")]

use std::ffi::CStr;
use std::fs::File;
use std::io::Seek;
use std::io::SeekFrom;
use std::os::raw::c_char;
use std::sync::Arc;

use base::AsRawDescriptor;
use base::Error as BaseError;
use base::FromRawDescriptor;

use crate::rutabaga_gralloc::formats::DrmFormat;
use crate::rutabaga_gralloc::gralloc::Gralloc;
use crate::rutabaga_gralloc::gralloc::ImageAllocationInfo;
use crate::rutabaga_gralloc::gralloc::ImageMemoryRequirements;
use crate::rutabaga_gralloc::minigbm_bindings::*;
use crate::rutabaga_gralloc::rendernode;
use crate::rutabaga_utils::*;

struct MinigbmDeviceInner {
    _fd: File,
    gbm: *mut gbm_device,
}

// Safe because minigbm handles synchronization internally.
unsafe impl Send for MinigbmDeviceInner {}
unsafe impl Sync for MinigbmDeviceInner {}

impl Drop for MinigbmDeviceInner {
    fn drop(&mut self) {
        // Safe because MinigbmDeviceInner is only constructed with a valid minigbm_device.
        unsafe {
            gbm_device_destroy(self.gbm);
        }
    }
}

/// A device capable of allocating `MinigbmBuffer`.
#[derive(Clone)]
pub struct MinigbmDevice {
    minigbm_device: Arc<MinigbmDeviceInner>,
    last_buffer: Option<Arc<MinigbmBuffer>>,
    device_name: &'static str,
}

impl MinigbmDevice {
    /// Returns a new `MinigbmDevice` if there is a rendernode in `/dev/dri/` that is accepted by
    /// the minigbm library.
    pub fn init() -> RutabagaResult<Box<dyn Gralloc>> {
        let undesired: &[&str] = &["vgem", "pvr"];
        let fd = rendernode::open_device(undesired)?;

        // gbm_create_device is safe to call with a valid fd, and we check that a valid one is
        // returned.  If the fd does not refer to a DRM device, gbm_create_device will reject it.
        let gbm = unsafe { gbm_create_device(fd.as_raw_descriptor()) };
        if gbm.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        // Safe because a valid minigbm device has a statically allocated string associated with
        // it, which is valid for the lifetime of the process.
        let backend_name: *const c_char = unsafe { gbm_device_get_backend_name(gbm) };
        let c_str: &CStr = unsafe { CStr::from_ptr(backend_name) };
        let device_name: &str = c_str.to_str()?;

        Ok(Box::new(MinigbmDevice {
            minigbm_device: Arc::new(MinigbmDeviceInner { _fd: fd, gbm }),
            last_buffer: None,
            device_name,
        }))
    }
}

impl Gralloc for MinigbmDevice {
    fn supports_external_gpu_memory(&self) -> bool {
        true
    }

    fn supports_dmabuf(&self) -> bool {
        true
    }

    fn get_image_memory_requirements(
        &mut self,
        info: ImageAllocationInfo,
    ) -> RutabagaResult<ImageMemoryRequirements> {
        let bo = unsafe {
            gbm_bo_create(
                self.minigbm_device.gbm,
                info.width,
                info.height,
                info.drm_format.0,
                info.flags.0,
            )
        };
        if bo.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        let mut reqs: ImageMemoryRequirements = Default::default();
        let gbm_buffer = MinigbmBuffer(bo, self.clone());

        // Intel GPUs typically only use cached memory buffers.  This will change with dGPUs, but
        // perhaps minigbm will be deprecated by then.  Other display drivers (rockchip, mediatek,
        // amdgpu) typically use write combine memory.  We can also consider use flags too if this
        // heuristic proves insufficient.
        if self.device_name == "i915" {
            reqs.map_info = RUTABAGA_MAP_CACHE_CACHED;
        } else {
            reqs.map_info = RUTABAGA_MAP_CACHE_WC;
        }

        reqs.modifier = gbm_buffer.format_modifier();
        for plane in 0..gbm_buffer.num_planes() {
            reqs.strides[plane] = gbm_buffer.plane_stride(plane);
            reqs.offsets[plane] = gbm_buffer.plane_offset(plane);
        }

        let mut fd = gbm_buffer.export()?;
        let size = fd.seek(SeekFrom::End(0))?;

        // minigbm does have the ability to query image requirements without allocating memory
        // via the TEST_ALLOC flag.  However, support has only been added in i915.  Until this
        // flag is supported everywhere, do the actual allocation here and stash it away.
        if self.last_buffer.is_some() {
            return Err(RutabagaError::AlreadyInUse);
        }

        self.last_buffer = Some(Arc::new(gbm_buffer));
        reqs.info = info;
        reqs.size = size;
        Ok(reqs)
    }

    fn allocate_memory(&mut self, reqs: ImageMemoryRequirements) -> RutabagaResult<RutabagaHandle> {
        let last_buffer = self.last_buffer.take();
        if let Some(gbm_buffer) = last_buffer {
            if gbm_buffer.width() != reqs.info.width
                || gbm_buffer.height() != reqs.info.height
                || gbm_buffer.format() != reqs.info.drm_format
            {
                return Err(RutabagaError::InvalidGrallocDimensions);
            }

            let dmabuf = gbm_buffer.export()?.into();
            return Ok(RutabagaHandle {
                os_handle: dmabuf,
                handle_type: RUTABAGA_MEM_HANDLE_TYPE_DMABUF,
            });
        }

        let bo = unsafe {
            gbm_bo_create(
                self.minigbm_device.gbm,
                reqs.info.width,
                reqs.info.height,
                reqs.info.drm_format.0,
                reqs.info.flags.0,
            )
        };

        if bo.is_null() {
            return Err(RutabagaError::BaseError(BaseError::last()));
        }

        let gbm_buffer = MinigbmBuffer(bo, self.clone());
        let dmabuf = gbm_buffer.export()?.into();
        Ok(RutabagaHandle {
            os_handle: dmabuf,
            handle_type: RUTABAGA_MEM_HANDLE_TYPE_DMABUF,
        })
    }
}

/// An allocation from a `MinigbmDevice`.
pub struct MinigbmBuffer(*mut gbm_bo, MinigbmDevice);

// Safe because minigbm handles synchronization internally.
unsafe impl Send for MinigbmBuffer {}
unsafe impl Sync for MinigbmBuffer {}

impl MinigbmBuffer {
    /// Width in pixels.
    pub fn width(&self) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_width(self.0) }
    }

    /// Height in pixels.
    pub fn height(&self) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_height(self.0) }
    }

    /// `DrmFormat` of the buffer.
    pub fn format(&self) -> DrmFormat {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { DrmFormat(gbm_bo_get_format(self.0)) }
    }

    /// DrmFormat modifier flags for the buffer.
    pub fn format_modifier(&self) -> u64 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_modifier(self.0) }
    }

    /// Number of planes present in this buffer.
    pub fn num_planes(&self) -> usize {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_plane_count(self.0) as usize }
    }

    /// Offset in bytes for the given plane.
    pub fn plane_offset(&self, plane: usize) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_offset(self.0, plane) }
    }

    /// Length in bytes of one row for the given plane.
    pub fn plane_stride(&self, plane: usize) -> u32 {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_get_stride_for_plane(self.0, plane) }
    }

    /// Exports a new dmabuf/prime file descriptor.
    pub fn export(&self) -> RutabagaResult<File> {
        // This is always safe to call with a valid gbm_bo pointer.
        match unsafe { gbm_bo_get_fd(self.0) } {
            fd if fd >= 0 => {
                let dmabuf = unsafe { File::from_raw_descriptor(fd) };
                Ok(dmabuf)
            }
            ret => Err(RutabagaError::ComponentError(ret)),
        }
    }
}

impl Drop for MinigbmBuffer {
    fn drop(&mut self) {
        // This is always safe to call with a valid gbm_bo pointer.
        unsafe { gbm_bo_destroy(self.0) }
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate rustc_serialize;
extern crate docopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use docopt::Docopt;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;

const USAGE: &'static str = "
Usage:
    stops2mimir --help
    stops2mimir --input=<file> \
     [--connection-string=<connection-string>] [--dataset=<dataset>] [--city-level=<level>]

Options:
    -h, --help                Show this message.
    -i, --input=<file>        NTFS stops.txt file.
    -c, --connection-string=<connection-string>   \
                              Elasticsearch parameters [default: http://localhost:9200/munin].
    -d, --dataset=<dataset>   Name of the dataset [default: fr].
    -C, --city-level=<level>  City level to calculate weight [default: 8]
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_input: String,
    flag_dataset: String,
    flag_connection_string: String,
    flag_city_level: u32,
}

struct StopPointIter<'a, R: std::io::Read + 'a> {
    iter: csv::StringRecords<'a, R>,
    stop_id_pos: usize,
    stop_lat_pos: usize,
    stop_lon_pos: usize,
    stop_name_pos: usize,
    location_type_pos: Option<usize>,
    stop_visible_pos: Option<usize>,
    parent_station_pos: Option<usize>,
    nb_stop_points: &'a mut HashMap<String, u32>
}

impl<'a, R: std::io::Read + 'a> StopPointIter<'a, R> {
    fn new(r: &'a mut csv::Reader<R>, nb_stop_points: &'a mut HashMap<String, u32>) -> csv::Result<Self> {
        let headers = try!(r.headers());
        let get_optional_pos = |name| headers.iter().position(|s| s == name);

        let get_pos = |field| {
            get_optional_pos(field).ok_or_else(|| {
                csv::Error::Decode(format!("Invalid file, cannot find column '{}'", field))
            })
        };

        Ok(StopPointIter {
            iter: r.records(),
            stop_id_pos: try!(get_pos("stop_id")),
            stop_lat_pos: try!(get_pos("stop_lat")),
            stop_lon_pos: try!(get_pos("stop_lon")),
            stop_name_pos: try!(get_pos("stop_name")),
            location_type_pos: get_optional_pos("location_type"),
            stop_visible_pos: get_optional_pos("visible"),
            parent_station_pos: get_optional_pos("parent_station"),
            nb_stop_points: nb_stop_points
        })
    }
    fn get_location_type(&self, record: &[String]) -> Option<u8> {
        self.location_type_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }
    fn get_visible(&self, record: &[String]) -> Option<u8> {
        self.stop_visible_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }

    fn get_parent_station<'b>(&self, record: &'b[String]) -> Option<&'b String> {
        self.parent_station_pos.and_then(|pos| record.get(pos))
    }
}

impl<'a, R: std::io::Read + 'a> Iterator for StopPointIter<'a, R> {
    type Item = csv::Result<mimir::Stop>;
    fn next(&mut self) -> Option<Self::Item> {
        fn get(record: &[String], pos: usize) -> csv::Result<&str> {
            record.get(pos)
                .map(|s| s.as_str())
                .ok_or_else(|| csv::Error::Decode(format!("Failed accessing record '{}'.", pos)))
        }
        fn parse_f64(s: &str) -> csv::Result<f64> {
            s.parse()
                .map_err(|_| csv::Error::Decode(format!("Failed converting '{}' from str.", s)))
        }

        fn is_valid_stop_area(location_type: &Option<u8>, visible: &Option<u8>) -> csv::Result<()> {
            if *location_type != Some(1) {
                Err(csv::Error::Decode("not a stop_area.".to_string()))
            } else if *visible == Some(0) {
                Err(csv::Error::Decode("stop_area invisible.".to_string()))
            } else {
                Ok(())
            }
        }

        self.iter.next().map(|r| {
            r.and_then(|r| {
                let location_type = self.get_location_type(&r);
                let visible = self.get_visible(&r);
                let parent_station = self.get_parent_station(&r);
                if let (Some(0), Some(id)) = (location_type, parent_station) {
                	if  !id.is_empty() {
                		//Increment the count by one
                		*self.nb_stop_points.entry(format!("stop_area:{}", id)).or_insert(0) +=1;
                	}
            	}
                try!(is_valid_stop_area(&location_type, &visible));
                let stop_id = try!(get(&r, self.stop_id_pos));
                let stop_lat = try!(get(&r, self.stop_lat_pos));
                let stop_lat = try!(parse_f64(stop_lat));
                let stop_lon = try!(get(&r, self.stop_lon_pos));
                let stop_lon = try!(parse_f64(stop_lon));
                let stop_name = try!(get(&r, self.stop_name_pos));
                Ok(mimir::Stop {
                    id: format!("stop_area:{}", stop_id), // prefix to match navitia's id
                    coord: mimir::Coord::new(stop_lat, stop_lon),
                    label: stop_name.to_string(),
                    weight: 1,
                    zip_codes: vec![],
                    administrative_regions: vec![],
                    name: stop_name.to_string(),
                })
            })
        })
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
                                                                        rubber: &mut Rubber,
                                                                        city_level: u32) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!("there is {}/{} stops without any admin",
          nb_unmatched,
          nb_matched + nb_unmatched);
}

//Update weight value for each stop_area from HashMap.
fn finalize_stop_area_weight<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It, nb_stop_points: &HashMap<String, u32>){
    for stop in stops {
    	if let Some(weight) = nb_stop_points.get(&stop.id) {
        	stop.weight = *weight;
    	}
    }
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args: Args = Docopt::new(USAGE).and_then(|dopt| dopt.decode()).unwrap_or_else(|e| e.exit());

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);
    let mut rdr = csv::Reader::from_file(args.flag_input).unwrap().double_quote(true);

	let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| rc.map_err(|e| debug!("skip csv line because: {}", e)).ok())
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.flag_city_level);

    finalize_stop_area_weight(stops.iter_mut(), &nb_stop_points);

    info!("Importing stops into Mimir");
    let nb_stops = rubber.index(&args.flag_dataset, stops.iter()).unwrap();

    info!("Nb of indexed stops: {}", nb_stops);

}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    // stops.txt:
    // SP:main_station : StopPoint object
    // SA:main_station: StopArea valid
    // SA:second_station: StopArea valid with visible is empty
    // SA:invisible_station: invisible StopArea
    // SA:without_lat: StopArea object without lattitude coord
    // SA:witout_lon: StopArea object without longitude coord
    // SA:station_no_city: StopArea far away, we won't be able to attach it to a city
    let mut rdr = csv::Reader::from_file("./tests/fixtures/stops.txt".to_string())
        .unwrap()
        .double_quote(true);

    let stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr)
        .unwrap()
        .filter_map(|rc| rc.map_err(|e| println!("error at csv line decoding : {}", e)).ok())
        .collect();
    assert_eq!(stops.len(), 3);
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(ids,
               vec!["stop_area:SA:main_station",
                    "stop_area:SA:second_station",
                    "stop_area:SA:station_no_city"]);
}
<commit_msg>mimir: modification in test<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate rustc_serialize;
extern crate docopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use docopt::Docopt;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;

const USAGE: &'static str = "
Usage:
    stops2mimir --help
    stops2mimir --input=<file> \
     [--connection-string=<connection-string>] [--dataset=<dataset>] [--city-level=<level>]

Options:
    -h, --help                Show this message.
    -i, --input=<file>        NTFS stops.txt file.
    -c, --connection-string=<connection-string>   \
                              Elasticsearch parameters [default: http://localhost:9200/munin].
    -d, --dataset=<dataset>   Name of the dataset [default: fr].
    -C, --city-level=<level>  City level to calculate weight [default: 8]
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_input: String,
    flag_dataset: String,
    flag_connection_string: String,
    flag_city_level: u32,
}

struct StopPointIter<'a, R: std::io::Read + 'a> {
    iter: csv::StringRecords<'a, R>,
    stop_id_pos: usize,
    stop_lat_pos: usize,
    stop_lon_pos: usize,
    stop_name_pos: usize,
    location_type_pos: Option<usize>,
    stop_visible_pos: Option<usize>,
    parent_station_pos: Option<usize>,
    nb_stop_points: &'a mut HashMap<String, u32>
}

impl<'a, R: std::io::Read + 'a> StopPointIter<'a, R> {
    fn new(r: &'a mut csv::Reader<R>, nb_stop_points: &'a mut HashMap<String, u32>) -> csv::Result<Self> {
        let headers = try!(r.headers());
        let get_optional_pos = |name| headers.iter().position(|s| s == name);

        let get_pos = |field| {
            get_optional_pos(field).ok_or_else(|| {
                csv::Error::Decode(format!("Invalid file, cannot find column '{}'", field))
            })
        };

        Ok(StopPointIter {
            iter: r.records(),
            stop_id_pos: try!(get_pos("stop_id")),
            stop_lat_pos: try!(get_pos("stop_lat")),
            stop_lon_pos: try!(get_pos("stop_lon")),
            stop_name_pos: try!(get_pos("stop_name")),
            location_type_pos: get_optional_pos("location_type"),
            stop_visible_pos: get_optional_pos("visible"),
            parent_station_pos: get_optional_pos("parent_station"),
            nb_stop_points: nb_stop_points
        })
    }
    fn get_location_type(&self, record: &[String]) -> Option<u8> {
        self.location_type_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }
    fn get_visible(&self, record: &[String]) -> Option<u8> {
        self.stop_visible_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }

    fn get_parent_station<'b>(&self, record: &'b[String]) -> Option<&'b String> {
        self.parent_station_pos.and_then(|pos| record.get(pos))
    }
}

impl<'a, R: std::io::Read + 'a> Iterator for StopPointIter<'a, R> {
    type Item = csv::Result<mimir::Stop>;
    fn next(&mut self) -> Option<Self::Item> {
        fn get(record: &[String], pos: usize) -> csv::Result<&str> {
            record.get(pos)
                .map(|s| s.as_str())
                .ok_or_else(|| csv::Error::Decode(format!("Failed accessing record '{}'.", pos)))
        }
        fn parse_f64(s: &str) -> csv::Result<f64> {
            s.parse()
                .map_err(|_| csv::Error::Decode(format!("Failed converting '{}' from str.", s)))
        }

        fn is_valid_stop_area(location_type: &Option<u8>, visible: &Option<u8>) -> csv::Result<()> {
            if *location_type != Some(1) {
                Err(csv::Error::Decode("not a stop_area.".to_string()))
            } else if *visible == Some(0) {
                Err(csv::Error::Decode("stop_area invisible.".to_string()))
            } else {
                Ok(())
            }
        }

        self.iter.next().map(|r| {
            r.and_then(|r| {
                let location_type = self.get_location_type(&r);
                let visible = self.get_visible(&r);
                let parent_station = self.get_parent_station(&r);
                if let (Some(0), Some(id)) = (location_type, parent_station) {
                	if  !id.is_empty() {
                		//Increment the count by one
                		*self.nb_stop_points.entry(format!("stop_area:{}", id)).or_insert(0) +=1;
                	}
            	}
                try!(is_valid_stop_area(&location_type, &visible));
                let stop_id = try!(get(&r, self.stop_id_pos));
                let stop_lat = try!(get(&r, self.stop_lat_pos));
                let stop_lat = try!(parse_f64(stop_lat));
                let stop_lon = try!(get(&r, self.stop_lon_pos));
                let stop_lon = try!(parse_f64(stop_lon));
                let stop_name = try!(get(&r, self.stop_name_pos));
                Ok(mimir::Stop {
                    id: format!("stop_area:{}", stop_id), // prefix to match navitia's id
                    coord: mimir::Coord::new(stop_lat, stop_lon),
                    label: stop_name.to_string(),
                    weight: 1,
                    zip_codes: vec![],
                    administrative_regions: vec![],
                    name: stop_name.to_string(),
                })
            })
        })
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
                                                                        rubber: &mut Rubber,
                                                                        city_level: u32) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!("there is {}/{} stops without any admin",
          nb_unmatched,
          nb_matched + nb_unmatched);
}

//Update weight value for each stop_area from HashMap.
fn finalize_stop_area_weight<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It, nb_stop_points: &HashMap<String, u32>){
    for stop in stops {
    	if let Some(weight) = nb_stop_points.get(&stop.id) {
        	stop.weight = *weight;
    	}
    }
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args: Args = Docopt::new(USAGE).and_then(|dopt| dopt.decode()).unwrap_or_else(|e| e.exit());

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);
    let mut rdr = csv::Reader::from_file(args.flag_input).unwrap().double_quote(true);

	let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| rc.map_err(|e| debug!("skip csv line because: {}", e)).ok())
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.flag_city_level);

    finalize_stop_area_weight(stops.iter_mut(), &nb_stop_points);

    info!("Importing stops into Mimir");
    let nb_stops = rubber.index(&args.flag_dataset, stops.iter()).unwrap();

    info!("Nb of indexed stops: {}", nb_stops);

}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    // stops.txt:
    // SP:main_station : StopPoint object
    // SA:main_station: StopArea valid
    // SA:second_station: StopArea valid with visible is empty
    // SA:invisible_station: invisible StopArea
    // SA:without_lat: StopArea object without lattitude coord
    // SA:witout_lon: StopArea object without longitude coord
    // SA:station_no_city: StopArea far away, we won't be able to attach it to a city
    let mut rdr = csv::Reader::from_file("./tests/fixtures/stops.txt".to_string())
        .unwrap()
        .double_quote(true);

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| rc.map_err(|e| println!("error at csv line decoding : {}", e)).ok())
        .collect();
    assert_eq!(stops.len(), 3);
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(ids,
               vec!["stop_area:SA:main_station",
                    "stop_area:SA:second_station",
                    "stop_area:SA:station_no_city"]);
}
<|endoftext|>


<commit_before>// (C)opyleft 2013 Frank Denis

/*!
 * Bloom filter for Rust
 *
 * This is a simple but fast Bloom filter implementation, that requires only
 * 2 hash functions, generated with SipHash-2-4 using randomized keys.
 */

#[desc = "A fast Bloom filter implementation."];
#[license = "BSD"];
#[crate_id="bloomfilter#0.1"];
#[crate_type = "rlib"];
#[warn(non_camel_case_types, non_uppercase_statics, non_uppercase_statics, unnecessary_qualification, managed_heap_memory)];

extern mod collections;
extern mod extra;

use std::num;
use std::rand;
use std::rand::Rng;
use collections::bitv;

struct SipHashKey {
    k1: u64,
    k2: u64
}

impl SipHashKey {
    fn new_random() -> SipHashKey {
        let mut rng = rand::task_rng();
        SipHashKey {
            k1: rand::Rand::rand(& mut rng),
            k2: rand::Rand::rand(& mut rng)
        }
    }
}

/// Bloom filter structure
pub struct Bloom {
    priv bitmap: bitv::Bitv,
    priv bitmap_bits: u64,
    priv k_num: uint,
    priv skeys: [SipHashKey, ..2]
}

impl Bloom {

/// Create a new bloom filter structure.
/// bitmap_size is the size in bytes (not bits) that will be allocated in memory
/// items_count is an estimation of the maximum number of items to store.
    pub fn new(bitmap_size: uint, items_count: uint) -> Bloom {
        assert!(bitmap_size > 0u && items_count > 0u);
        let bitmap_bits = (bitmap_size as u64) * 8u64;
        let k_num = Bloom::optimal_k_num(bitmap_bits, items_count);
        let bitmap = bitv::Bitv::new(bitmap_bits as uint, false);
        let skeys = [ SipHashKey::new_random(), SipHashKey::new_random() ];
        Bloom {
            bitmap: bitmap,
            bitmap_bits: bitmap_bits,
            k_num: k_num,
            skeys: skeys
        }
    }

/// Create a new bloom filter structure.
/// items_count is an estimation of the maximum number of items to store.
/// fp_p is the wanted rate of false positives, in ]0.0, 1.0[
    pub fn new_for_fp_rate(items_count: uint, fp_p: f64) -> Bloom {
        let bitmap_size = Bloom::compute_bitmap_size(items_count, fp_p);
        Bloom::new(bitmap_size, items_count)
    }

/// Compute a recommended bitmap size for items_count items
/// and a fp_p rate of false positives.
/// fp_p obviously has to be within the ]0.0, 1.0[ range.
    pub fn compute_bitmap_size(items_count: uint, fp_p: f64) -> uint {
        assert!(items_count > 0u);
        assert!(fp_p > 0.0 && fp_p < 1.0);
        let log2 = num::ln(2.0f64);
        let log2_2 = log2 * log2;
        ((items_count as f64) * num::ln(fp_p) / (-8.0 * log2_2)).ceil() as uint
    }

/// Record the presence of an item.
    pub fn set<T: Hash>(& mut self, item: T) {
        let mut hashes = [ 0u64, 0u64 ];
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            self.bitmap.set(bit_offset, true);
        }
    }

/// Check if an item is present in the set.
/// There can be false positives, but no false negatives.
    pub fn check<T: Hash>(&self, item: T) -> bool {
        let mut hashes = [ 0u64, 0u64 ];
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            if self.bitmap.get(bit_offset) == false {
                return false;
            }
        }
        true
    }

/// Record the presence of an item in the set,
/// and return the previous state of this item.
    pub fn check_and_set<T: Hash>(&mut self, item: T) -> bool {
        let mut hashes = [ 0u64, 0u64 ];
        let mut found = true;
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            if self.bitmap.get(bit_offset) == false {
                found = false;
                self.bitmap.set(bit_offset, true);
            }
        }
        found
    }

    fn optimal_k_num(bitmap_bits: u64, items_count: uint) -> uint {
        let m = bitmap_bits as f64;
        let n = items_count as f64;
        let k_num = (m / n * num::ln(2.0f64).ceil()) as uint;
        num::max(k_num, 1)
    }

    fn bloom_hash<T: Hash>(&self, hashes: & mut [u64, ..2],
                  item: &T, k_i: uint) -> u64 {
        if k_i < 2 {
            let skey = self.skeys[k_i];
            let hash = (*item).hash_keyed(skey.k1, skey.k2);
            hashes[k_i] = hash;
            hash
        } else {
            hashes[0] + (((k_i as u64) * hashes[1]) % 0xffffffffffffffc5)
        }
    }
}

#[test]
fn bloom_test_set() {
    let mut bloom = Bloom::new(10, 80);
    let key = &rand::task_rng().gen_ascii_str(16u);
    assert!(bloom.check(key) == false);
    bloom.set(&key);
    assert!(bloom.check(key.clone()) == true);
}

#[test]
fn bloom_test_check_and_set() {
    let mut bloom = Bloom::new(10, 80);
    let key = &rand::task_rng().gen_ascii_str(16u);
    assert!(bloom.check_and_set(key) == false);
    assert!(bloom.check_and_set(key.clone()) == true);
}
<commit_msg>extern mod => extern crate<commit_after>// (C)opyleft 2013 Frank Denis

/*!
 * Bloom filter for Rust
 *
 * This is a simple but fast Bloom filter implementation, that requires only
 * 2 hash functions, generated with SipHash-2-4 using randomized keys.
 */

#[desc = "A fast Bloom filter implementation."];
#[license = "BSD"];
#[crate_id="bloomfilter#0.1"];
#[crate_type = "rlib"];
#[warn(non_camel_case_types, non_uppercase_statics, non_uppercase_statics, unnecessary_qualification, managed_heap_memory)];

extern crate collections;
extern crate extra;

use std::num;
use std::rand;
use std::rand::Rng;
use collections::bitv;

struct SipHashKey {
    k1: u64,
    k2: u64
}

impl SipHashKey {
    fn new_random() -> SipHashKey {
        let mut rng = rand::task_rng();
        SipHashKey {
            k1: rand::Rand::rand(& mut rng),
            k2: rand::Rand::rand(& mut rng)
        }
    }
}

/// Bloom filter structure
pub struct Bloom {
    priv bitmap: bitv::Bitv,
    priv bitmap_bits: u64,
    priv k_num: uint,
    priv skeys: [SipHashKey, ..2]
}

impl Bloom {

/// Create a new bloom filter structure.
/// bitmap_size is the size in bytes (not bits) that will be allocated in memory
/// items_count is an estimation of the maximum number of items to store.
    pub fn new(bitmap_size: uint, items_count: uint) -> Bloom {
        assert!(bitmap_size > 0u && items_count > 0u);
        let bitmap_bits = (bitmap_size as u64) * 8u64;
        let k_num = Bloom::optimal_k_num(bitmap_bits, items_count);
        let bitmap = bitv::Bitv::new(bitmap_bits as uint, false);
        let skeys = [ SipHashKey::new_random(), SipHashKey::new_random() ];
        Bloom {
            bitmap: bitmap,
            bitmap_bits: bitmap_bits,
            k_num: k_num,
            skeys: skeys
        }
    }

/// Create a new bloom filter structure.
/// items_count is an estimation of the maximum number of items to store.
/// fp_p is the wanted rate of false positives, in ]0.0, 1.0[
    pub fn new_for_fp_rate(items_count: uint, fp_p: f64) -> Bloom {
        let bitmap_size = Bloom::compute_bitmap_size(items_count, fp_p);
        Bloom::new(bitmap_size, items_count)
    }

/// Compute a recommended bitmap size for items_count items
/// and a fp_p rate of false positives.
/// fp_p obviously has to be within the ]0.0, 1.0[ range.
    pub fn compute_bitmap_size(items_count: uint, fp_p: f64) -> uint {
        assert!(items_count > 0u);
        assert!(fp_p > 0.0 && fp_p < 1.0);
        let log2 = num::ln(2.0f64);
        let log2_2 = log2 * log2;
        ((items_count as f64) * num::ln(fp_p) / (-8.0 * log2_2)).ceil() as uint
    }

/// Record the presence of an item.
    pub fn set<T: Hash>(& mut self, item: T) {
        let mut hashes = [ 0u64, 0u64 ];
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            self.bitmap.set(bit_offset, true);
        }
    }

/// Check if an item is present in the set.
/// There can be false positives, but no false negatives.
    pub fn check<T: Hash>(&self, item: T) -> bool {
        let mut hashes = [ 0u64, 0u64 ];
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            if self.bitmap.get(bit_offset) == false {
                return false;
            }
        }
        true
    }

/// Record the presence of an item in the set,
/// and return the previous state of this item.
    pub fn check_and_set<T: Hash>(&mut self, item: T) -> bool {
        let mut hashes = [ 0u64, 0u64 ];
        let mut found = true;
        for k_i in range(0u, self.k_num) {
            let bit_offset = (self.bloom_hash(& mut hashes, &item, k_i)
                              % self.bitmap_bits) as uint;
            if self.bitmap.get(bit_offset) == false {
                found = false;
                self.bitmap.set(bit_offset, true);
            }
        }
        found
    }

    fn optimal_k_num(bitmap_bits: u64, items_count: uint) -> uint {
        let m = bitmap_bits as f64;
        let n = items_count as f64;
        let k_num = (m / n * num::ln(2.0f64).ceil()) as uint;
        num::max(k_num, 1)
    }

    fn bloom_hash<T: Hash>(&self, hashes: & mut [u64, ..2],
                  item: &T, k_i: uint) -> u64 {
        if k_i < 2 {
            let skey = self.skeys[k_i];
            let hash = (*item).hash_keyed(skey.k1, skey.k2);
            hashes[k_i] = hash;
            hash
        } else {
            hashes[0] + (((k_i as u64) * hashes[1]) % 0xffffffffffffffc5)
        }
    }
}

#[test]
fn bloom_test_set() {
    let mut bloom = Bloom::new(10, 80);
    let key = &rand::task_rng().gen_ascii_str(16u);
    assert!(bloom.check(key) == false);
    bloom.set(&key);
    assert!(bloom.check(key.clone()) == true);
}

#[test]
fn bloom_test_check_and_set() {
    let mut bloom = Bloom::new(10, 80);
    let key = &rand::task_rng().gen_ascii_str(16u);
    assert!(bloom.check_and_set(key) == false);
    assert!(bloom.check_and_set(key.clone()) == true);
}
<|endoftext|>