<commit_before>// System tests for compiling C code.
//
// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![deny(rust_2018_idioms)]
#![allow(dead_code, unused_imports)]

#[macro_use]
extern crate log;
use crate::harness::{
    get_stats, sccache_client_cfg, sccache_command, start_local_daemon, stop_local_daemon,
    write_json_cfg, write_source, zero_stats,
};
use assert_cmd::prelude::*;
use escargot::CargoBuild;
use log::Level::Trace;
use predicates::prelude::*;
use std::collections::HashMap;
use std::env;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::{self, File};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::str;
use tempdir::TempDir;
use which::which_in;

mod harness;

#[derive(Clone)]
struct Compiler {
    pub name: &'static str,
    pub exe: OsString,
    pub env_vars: Vec<(OsString, OsString)>,
}

// Test GCC + clang on non-OS X platforms.
#[cfg(all(unix, not(target_os = "macos")))]
const COMPILERS: &'static [&'static str] = &["gcc", "clang"];

// OS X ships a `gcc` that's just a clang wrapper, so only test clang there.
#[cfg(target_os = "macos")]
const COMPILERS: &'static [&'static str] = &["clang"];

//TODO: could test gcc when targeting mingw.

macro_rules! vec_from {
    ( $t:ty, $( $x:expr ),* ) => {
        vec!($( Into::<$t>::into(&$x), )*)
    };
}

fn compile_cmdline<T: AsRef<OsStr>>(
    compiler: &str,
    exe: T,
    input: &str,
    output: &str,
) -> Vec<OsString> {
    match compiler {
        "gcc" | "clang" => vec_from!(OsString, exe.as_ref(), "-c", input, "-o", output),
        "cl.exe" => vec_from!(OsString, exe, "-c", input, format!("-Fo{}", output)),
        _ => panic!("Unsupported compiler: {}", compiler),
    }
}

const INPUT: &'static str = "test.c";
const INPUT_ERR: &'static str = "test_err.c";
const INPUT_MACRO_EXPANSION: &'static str = "test_macro_expansion.c";
const OUTPUT: &'static str = "test.o";

// Copy the source files into the tempdir so we can compile with relative paths, since the commandline winds up in the hash key.
fn copy_to_tempdir(inputs: &[&str], tempdir: &Path) {
    for f in inputs {
        let original_source_file = Path::new(file!()).parent().unwrap().join(f.clone());
        let source_file = tempdir.join(f);
        trace!("fs::copy({:?}, {:?})", original_source_file, source_file);
        fs::copy(&original_source_file, &source_file).unwrap();
    }
}

fn test_basic_compile(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("run_sccache_command_test: {}", name);
    // Compile a source file.
    copy_to_tempdir(&[INPUT, INPUT_ERR], tempdir);

    let out_file = tempdir.join("test.o");
    trace!("compile");
    sccache_command()
        .args(&compile_cmdline(name, &exe, INPUT, OUTPUT))
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    assert_eq!(
        true,
        fs::metadata(&out_file)
            .and_then(|m| Ok(m.len() > 0))
            .unwrap()
    );
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(1, info.stats.compile_requests);
        assert_eq!(1, info.stats.requests_executed);
        assert_eq!(0, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    trace!("compile");
    fs::remove_file(&out_file).unwrap();
    sccache_command()
        .args(&compile_cmdline(name, &exe, INPUT, OUTPUT))
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    assert_eq!(
        true,
        fs::metadata(&out_file)
            .and_then(|m| Ok(m.len() > 0))
            .unwrap()
    );
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(2, info.stats.compile_requests);
        assert_eq!(2, info.stats.requests_executed);
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
}

fn test_noncacheable_stats(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test_noncacheable_stats: {}", name);
    copy_to_tempdir(&[INPUT], tempdir);

    trace!("compile");
    Command::main_binary()
        .unwrap()
        .arg(&exe)
        .arg("-E")
        .arg(INPUT)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(1, info.stats.compile_requests);
        assert_eq!(0, info.stats.requests_executed);
        assert_eq!(1, info.stats.not_cached.len());
        assert_eq!(Some(&1), info.stats.not_cached.get("-E"));
    });
}

fn test_msvc_deps(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    // Check that -deps works.
    trace!("compile with -deps");
    let mut args = compile_cmdline(name, &exe, INPUT, OUTPUT);
    args.push("-depstest.d".into());
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    // Check the contents
    let mut f = File::open(tempdir.join("test.d")).expect("Failed to open dep file");
    let mut buf = String::new();
    // read_to_string should be safe because we're supplying all the filenames here,
    // and there are no absolute paths.
    f.read_to_string(&mut buf).expect("Failed to read dep file");
    let lines: Vec<_> = buf.lines().map(|l| l.trim_end()).collect();
    let expected = format!(
        "{output}: {input}\n{input}:\n",
        output = OUTPUT,
        input = INPUT
    );
    let expected_lines: Vec<_> = expected.lines().collect();
    assert_eq!(lines, expected_lines);
}

fn test_gcc_mp_werror(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test -MP with -Werror");
    let mut args = compile_cmdline(name, &exe, INPUT_ERR, OUTPUT);
    args.extend(vec_from!(
        OsString, "-MD", "-MP", "-MF", "foo.pp", "-Werror"
    ));
    // This should fail, but the error should be from the #error!
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .failure()
        .stderr(
            predicates::str::contains("to generate dependencies you must specify either -M or -MM")
                .from_utf8()
                .not(),
        );
}

fn test_gcc_fprofile_generate_source_changes(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test -fprofile-generate with different source inputs");
    zero_stats();
    const SRC: &str = "source.c";
    write_source(
        &tempdir,
        SRC,
        "/*line 1*/
#ifndef UNDEFINED
/*unused line 1*/
#endif

int main(int argc, char** argv) {
  return 0;
}
",
    );
    let mut args = compile_cmdline(name, &exe, SRC, OUTPUT);
    args.extend(vec_from!(OsString, "-fprofile-generate"));
    trace!("compile source.c (1)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(0, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    // Compile the same source again to ensure we can get a cache hit.
    trace!("compile source.c (2)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    // Now write out a slightly different source file that will preprocess to the same thing,
    // modulo line numbers. This should not be a cache hit because line numbers are important
    // with -fprofile-generate.
    write_source(
        &tempdir,
        SRC,
        "/*line 1*/
#ifndef UNDEFINED
/*unused line 1*/
/*unused line 2*/
#endif

int main(int argc, char** argv) {
  return 0;
}
",
    );
    trace!("compile source.c (3)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(2, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&2, info.stats.cache_misses.get("C/C++").unwrap());
    });
}

fn test_gcc_clang_no_warnings_from_macro_expansion(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test_gcc_clang_no_warnings_from_macro_expansion: {}", name);
    // Compile a source file.
    copy_to_tempdir(&[INPUT_MACRO_EXPANSION], tempdir);

    trace!("compile");
    sccache_command()
        .args(
            [
                &compile_cmdline(name, &exe, INPUT_MACRO_EXPANSION, OUTPUT)[..],
                &vec_from!(OsString, "-Wunreachable-code")[..],
            ]
            .concat(),
        )
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success()
        .stderr(predicates::str::contains("warning:").from_utf8().not());
}

fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) {
    test_basic_compile(compiler.clone(), tempdir);
    if compiler.name == "cl.exe" {
        test_msvc_deps(compiler.clone(), tempdir);
    }
    if compiler.name == "gcc" {
        test_gcc_mp_werror(compiler.clone(), tempdir);
        test_gcc_fprofile_generate_source_changes(compiler.clone(), tempdir);
    }
    if compiler.name == "clang" || compiler.name == "gcc" {
        test_gcc_clang_no_warnings_from_macro_expansion(compiler.clone(), tempdir);
    }
}

#[cfg(unix)]
fn find_compilers() -> Vec<Compiler> {
    let cwd = env::current_dir().unwrap();
    COMPILERS
        .iter()
        .filter_map(|c| match which_in(c, env::var_os("PATH"), &cwd) {
            Ok(full_path) => match full_path.canonicalize() {
                Ok(full_path_canon) => Some(Compiler {
                    name: *c,
                    exe: full_path_canon.into_os_string(),
                    env_vars: vec![],
                }),
                Err(_) => None,
            },
            Err(_) => None,
        })
        .collect::<Vec<_>>()
}

#[cfg(target_env = "msvc")]
fn find_compilers() -> Vec<Compiler> {
    let tool = cc::Build::new()
        .opt_level(1)
        .host("x86_64-pc-windows-msvc")
        .target("x86_64-pc-windows-msvc")
        .debug(false)
        .get_compiler();
    vec![Compiler {
        name: "cl.exe",
        exe: tool.path().as_os_str().to_os_string(),
        env_vars: tool.env().to_vec(),
    }]
}

#[test]
#[cfg(any(unix, target_env = "msvc"))]
fn test_sccache_command() {
    match env_logger::try_init() {
        Ok(_) => {}
        Err(_) => {}
    }
    let tempdir = TempDir::new("sccache_system_test").unwrap();
    let compilers = find_compilers();
    if compilers.is_empty() {
        warn!("No compilers found, skipping test");
    } else {
        // Ensure there's no existing sccache server running.
        stop_local_daemon();
        // Create the configurations
        let sccache_cfg = sccache_client_cfg(tempdir.path());
        write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg);
        let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg");
        // Start a server.
        trace!("start server");
        start_local_daemon(
            &tempdir.path().join("sccache-cfg.json"),
            &sccache_cached_cfg_path,
        );
        for compiler in compilers {
            run_sccache_command_tests(compiler, tempdir.path());
            zero_stats();
        }
        stop_local_daemon();
    }
}
<commit_msg>Added comment on splitting up test cases<commit_after>// System tests for compiling C code.
//
// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![deny(rust_2018_idioms)]
#![allow(dead_code, unused_imports)]

#[macro_use]
extern crate log;
use crate::harness::{
    get_stats, sccache_client_cfg, sccache_command, start_local_daemon, stop_local_daemon,
    write_json_cfg, write_source, zero_stats,
};
use assert_cmd::prelude::*;
use escargot::CargoBuild;
use log::Level::Trace;
use predicates::prelude::*;
use std::collections::HashMap;
use std::env;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::{self, File};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::str;
use tempdir::TempDir;
use which::which_in;

mod harness;

#[derive(Clone)]
struct Compiler {
    pub name: &'static str,
    pub exe: OsString,
    pub env_vars: Vec<(OsString, OsString)>,
}

// Test GCC + clang on non-OS X platforms.
#[cfg(all(unix, not(target_os = "macos")))]
const COMPILERS: &'static [&'static str] = &["gcc", "clang"];

// OS X ships a `gcc` that's just a clang wrapper, so only test clang there.
#[cfg(target_os = "macos")]
const COMPILERS: &'static [&'static str] = &["clang"];

//TODO: could test gcc when targeting mingw.

macro_rules! vec_from {
    ( $t:ty, $( $x:expr ),* ) => {
        vec!($( Into::<$t>::into(&$x), )*)
    };
}

fn compile_cmdline<T: AsRef<OsStr>>(
    compiler: &str,
    exe: T,
    input: &str,
    output: &str,
) -> Vec<OsString> {
    match compiler {
        "gcc" | "clang" => vec_from!(OsString, exe.as_ref(), "-c", input, "-o", output),
        "cl.exe" => vec_from!(OsString, exe, "-c", input, format!("-Fo{}", output)),
        _ => panic!("Unsupported compiler: {}", compiler),
    }
}

const INPUT: &'static str = "test.c";
const INPUT_ERR: &'static str = "test_err.c";
const INPUT_MACRO_EXPANSION: &'static str = "test_macro_expansion.c";
const OUTPUT: &'static str = "test.o";

// Copy the source files into the tempdir so we can compile with relative paths, since the commandline winds up in the hash key.
fn copy_to_tempdir(inputs: &[&str], tempdir: &Path) {
    for f in inputs {
        let original_source_file = Path::new(file!()).parent().unwrap().join(f.clone());
        let source_file = tempdir.join(f);
        trace!("fs::copy({:?}, {:?})", original_source_file, source_file);
        fs::copy(&original_source_file, &source_file).unwrap();
    }
}

fn test_basic_compile(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("run_sccache_command_test: {}", name);
    // Compile a source file.
    copy_to_tempdir(&[INPUT, INPUT_ERR], tempdir);

    let out_file = tempdir.join("test.o");
    trace!("compile");
    sccache_command()
        .args(&compile_cmdline(name, &exe, INPUT, OUTPUT))
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    assert_eq!(
        true,
        fs::metadata(&out_file)
            .and_then(|m| Ok(m.len() > 0))
            .unwrap()
    );
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(1, info.stats.compile_requests);
        assert_eq!(1, info.stats.requests_executed);
        assert_eq!(0, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    trace!("compile");
    fs::remove_file(&out_file).unwrap();
    sccache_command()
        .args(&compile_cmdline(name, &exe, INPUT, OUTPUT))
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    assert_eq!(
        true,
        fs::metadata(&out_file)
            .and_then(|m| Ok(m.len() > 0))
            .unwrap()
    );
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(2, info.stats.compile_requests);
        assert_eq!(2, info.stats.requests_executed);
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
}

fn test_noncacheable_stats(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test_noncacheable_stats: {}", name);
    copy_to_tempdir(&[INPUT], tempdir);

    trace!("compile");
    Command::main_binary()
        .unwrap()
        .arg(&exe)
        .arg("-E")
        .arg(INPUT)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    trace!("request stats");
    get_stats(|info| {
        assert_eq!(1, info.stats.compile_requests);
        assert_eq!(0, info.stats.requests_executed);
        assert_eq!(1, info.stats.not_cached.len());
        assert_eq!(Some(&1), info.stats.not_cached.get("-E"));
    });
}

fn test_msvc_deps(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    // Check that -deps works.
    trace!("compile with -deps");
    let mut args = compile_cmdline(name, &exe, INPUT, OUTPUT);
    args.push("-depstest.d".into());
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    // Check the contents
    let mut f = File::open(tempdir.join("test.d")).expect("Failed to open dep file");
    let mut buf = String::new();
    // read_to_string should be safe because we're supplying all the filenames here,
    // and there are no absolute paths.
    f.read_to_string(&mut buf).expect("Failed to read dep file");
    let lines: Vec<_> = buf.lines().map(|l| l.trim_end()).collect();
    let expected = format!(
        "{output}: {input}\n{input}:\n",
        output = OUTPUT,
        input = INPUT
    );
    let expected_lines: Vec<_> = expected.lines().collect();
    assert_eq!(lines, expected_lines);
}

fn test_gcc_mp_werror(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test -MP with -Werror");
    let mut args = compile_cmdline(name, &exe, INPUT_ERR, OUTPUT);
    args.extend(vec_from!(
        OsString, "-MD", "-MP", "-MF", "foo.pp", "-Werror"
    ));
    // This should fail, but the error should be from the #error!
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .failure()
        .stderr(
            predicates::str::contains("to generate dependencies you must specify either -M or -MM")
                .from_utf8()
                .not(),
        );
}

fn test_gcc_fprofile_generate_source_changes(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test -fprofile-generate with different source inputs");
    zero_stats();
    const SRC: &str = "source.c";
    write_source(
        &tempdir,
        SRC,
        "/*line 1*/
#ifndef UNDEFINED
/*unused line 1*/
#endif

int main(int argc, char** argv) {
  return 0;
}
",
    );
    let mut args = compile_cmdline(name, &exe, SRC, OUTPUT);
    args.extend(vec_from!(OsString, "-fprofile-generate"));
    trace!("compile source.c (1)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(0, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    // Compile the same source again to ensure we can get a cache hit.
    trace!("compile source.c (2)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(1, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap());
    });
    // Now write out a slightly different source file that will preprocess to the same thing,
    // modulo line numbers. This should not be a cache hit because line numbers are important
    // with -fprofile-generate.
    write_source(
        &tempdir,
        SRC,
        "/*line 1*/
#ifndef UNDEFINED
/*unused line 1*/
/*unused line 2*/
#endif

int main(int argc, char** argv) {
  return 0;
}
",
    );
    trace!("compile source.c (3)");
    sccache_command()
        .args(&args)
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success();
    get_stats(|info| {
        assert_eq!(1, info.stats.cache_hits.all());
        assert_eq!(2, info.stats.cache_misses.all());
        assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap());
        assert_eq!(&2, info.stats.cache_misses.get("C/C++").unwrap());
    });
}

fn test_gcc_clang_no_warnings_from_macro_expansion(compiler: Compiler, tempdir: &Path) {
    let Compiler {
        name,
        exe,
        env_vars,
    } = compiler;
    trace!("test_gcc_clang_no_warnings_from_macro_expansion: {}", name);
    // Compile a source file.
    copy_to_tempdir(&[INPUT_MACRO_EXPANSION], tempdir);

    trace!("compile");
    sccache_command()
        .args(
            [
                &compile_cmdline(name, &exe, INPUT_MACRO_EXPANSION, OUTPUT)[..],
                &vec_from!(OsString, "-Wunreachable-code")[..],
            ]
            .concat(),
        )
        .current_dir(tempdir)
        .envs(env_vars.clone())
        .assert()
        .success()
        .stderr(predicates::str::contains("warning:").from_utf8().not());
}

fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) {
    test_basic_compile(compiler.clone(), tempdir);
    if compiler.name == "cl.exe" {
        test_msvc_deps(compiler.clone(), tempdir);
    }
    if compiler.name == "gcc" {
        test_gcc_mp_werror(compiler.clone(), tempdir);
        test_gcc_fprofile_generate_source_changes(compiler.clone(), tempdir);
    }
    if compiler.name == "clang" || compiler.name == "gcc" {
        test_gcc_clang_no_warnings_from_macro_expansion(compiler.clone(), tempdir);
    }
}

#[cfg(unix)]
fn find_compilers() -> Vec<Compiler> {
    let cwd = env::current_dir().unwrap();
    COMPILERS
        .iter()
        .filter_map(|c| match which_in(c, env::var_os("PATH"), &cwd) {
            Ok(full_path) => match full_path.canonicalize() {
                Ok(full_path_canon) => Some(Compiler {
                    name: *c,
                    exe: full_path_canon.into_os_string(),
                    env_vars: vec![],
                }),
                Err(_) => None,
            },
            Err(_) => None,
        })
        .collect::<Vec<_>>()
}

#[cfg(target_env = "msvc")]
fn find_compilers() -> Vec<Compiler> {
    let tool = cc::Build::new()
        .opt_level(1)
        .host("x86_64-pc-windows-msvc")
        .target("x86_64-pc-windows-msvc")
        .debug(false)
        .get_compiler();
    vec![Compiler {
        name: "cl.exe",
        exe: tool.path().as_os_str().to_os_string(),
        env_vars: tool.env().to_vec(),
    }]
}

// TODO: This runs multiple test cases, for multiple compilers. It should be
// split up to run them individually. In the current form, it is hard to see
// which sub test cases are executed, and if one fails, the remaining tests
// are not run.
#[test]
#[cfg(any(unix, target_env = "msvc"))]
fn test_sccache_command() {
    match env_logger::try_init() {
        Ok(_) => {}
        Err(_) => {}
    }
    let tempdir = TempDir::new("sccache_system_test").unwrap();
    let compilers = find_compilers();
    if compilers.is_empty() {
        warn!("No compilers found, skipping test");
    } else {
        // Ensure there's no existing sccache server running.
        stop_local_daemon();
        // Create the configurations
        let sccache_cfg = sccache_client_cfg(tempdir.path());
        write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg);
        let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg");
        // Start a server.
        trace!("start server");
        start_local_daemon(
            &tempdir.path().join("sccache-cfg.json"),
            &sccache_cached_cfg_path,
        );
        for compiler in compilers {
            run_sccache_command_tests(compiler, tempdir.path());
            zero_stats();
        }
        stop_local_daemon();
    }
}
<|endoftext|>


<commit_before>use structs::*;

const NAME: &str = "Ebenezer";
const AGE: u32 = 89;
const WEIGHT: f32 = 131.6;

#[test]
fn test_name() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert_eq!(user.name(), NAME);
}

#[test]
#[ignore]
fn test_age() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert_eq!(user.age(), AGE);
}

#[test]
#[ignore]
fn test_weight() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert_eq!(user.weight(), WEIGHT);
}

#[test]
#[ignore]
fn test_set_age() {
    let new_age: u32 = 90;
    let mut user = User::new(NAME.into(), AGE, WEIGHT);
    user.set_age(new_age);
    assert_eq!(user.age(), new_age);
}

#[test]
#[ignore]
fn test_set_weight() {
    let new_weight: f32 = 129.4;
    let mut user = User::new(NAME.into(), AGE, WEIGHT);
    user.set_weight(new_weight);
    assert!((user.weight() - new_weight).abs() < f32::EPSILON);
}
<commit_msg>another structs clippy fix<commit_after>use structs::*;

const NAME: &str = "Ebenezer";
const AGE: u32 = 89;
const WEIGHT: f32 = 131.6;

#[test]
fn test_name() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert_eq!(user.name(), NAME);
}

#[test]
#[ignore]
fn test_age() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert_eq!(user.age(), AGE);
}

#[test]
#[ignore]
fn test_weight() {
    let user = User::new(NAME.into(), AGE, WEIGHT);
    assert!((user.weight() - WEIGHT).abs() < f32::EPSILON);
}

#[test]
#[ignore]
fn test_set_age() {
    let new_age: u32 = 90;
    let mut user = User::new(NAME.into(), AGE, WEIGHT);
    user.set_age(new_age);
    assert_eq!(user.age(), new_age);
}

#[test]
#[ignore]
fn test_set_weight() {
    let new_weight: f32 = 129.4;
    let mut user = User::new(NAME.into(), AGE, WEIGHT);
    user.set_weight(new_weight);
    assert!((user.weight() - new_weight).abs() < f32::EPSILON);
}
<|endoftext|>


<commit_before>/// The Toml Insert extensions

use toml::Value;

use tokenizer::Token;
use tokenizer::tokenize_with_seperator;
use error::*;

pub trait TomlValueInsertExt<'doc> {

    /// Extension function for inserting a value in the current toml::Value document
    /// using a custom seperator.
    ///
    /// For difference to TomlSetExt::set() and friends, read [#semantics].
    ///
    /// # Semantics
    ///
    /// The function automatically creates intermediate data structures based on the query string.
    /// That means, if the query string is `"a.b.c.[0]"`, but only a table `"a"` exists in the
    /// document, the function automatically creates a table `"b"` inside `"a"` and `"c"` inside
    /// `"b"`, and an array in `"c"`. The array index is ignored if the array is created.
    ///
    /// If an Array exists, but the specified index is larger than the last index, the array will
    /// be expanded by one element: If the array has a length of 3, but the query string specifies
    /// that the element should be put at 1000, the function ignores the large index and simply
    /// appends the value to the index.
    ///
    /// # Return value
    ///
    /// If the insert operation worked correctly, `Ok(None)` is returned.
    /// If the insert operation replaced an existing value `Ok(Some(old_value))` is returned
    /// On failure, `Err(e)` is returned
    ///
    fn insert_with_seperator(&mut self, query: &String, sep: char, value: Value) -> Result<Option<Value>>;

    /// Extension function for inserting a value from the current toml::Value document
    ///
    /// See documentation of `TomlValueinsertExt::insert_with_seperator`
    fn insert(&mut self, query: &String, value: Value) -> Result<Option<Value>> {
        self.insert_with_seperator(query, '.', value)
    }

}

impl<'doc> TomlValueInsertExt<'doc> for Value {

    fn insert_with_seperator(&mut self, query: &String, sep: char, value: Value) -> Result<Option<Value>> {
        use resolver::mut_resolver::resolve;

        let mut tokens = try!(tokenize_with_seperator(query, sep));
        let last       = tokens.pop_last().unwrap();
        let mut val    = try!(resolve(self, &tokens));

        match *last {
            Token::Identifier { ident, .. } => {
                match val {
                    &mut Value::Table(ref mut t) => {
                        Ok(t.insert(ident, value))
                    },
                    _ => unimplemented!()
                }
            },

            Token::Index { idx , .. } => {
                match val {
                    &mut Value::Array(ref mut a) => {
                        if a.len() > idx {
                            unimplemented!()
                        } else {
                            a.push(value);
                            Ok(None)
                        }
                    },
                    _ => unimplemented!()
                }
            },
        }
    }

}

#[cfg(test)]
mod test {
    use super::*;
    use error::*;
    use toml::Value;
    use toml::from_str as toml_from_str;

    #[test]
    fn test_insert_with_seperator_into_table() {
        let mut toml : Value = toml_from_str(r#"
        [table]
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("table.a"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref t) => {
                assert!(!t.is_empty());

                let table = t.get("table");
                assert!(table.is_some());

                let table = table.unwrap();
                assert!(is_match!(table, &Value::Table(_)));
                match table {
                    &Value::Table(ref t) => {
                        assert!(!t.is_empty());

                        let a = t.get("a");
                        assert!(a.is_some());

                        let a = a.unwrap();
                        assert!(is_match!(a, &Value::Integer(1)));
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happenend?"),
        }
    }

    #[test]
    fn test_insert_with_seperator_into_array() {
        use std::ops::Index;

        let mut toml : Value = toml_from_str(r#"
        array = []
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("array.[0]"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref t) => {
                assert!(!t.is_empty());

                let array = t.get("array");
                assert!(array.is_some());

                let array = array.unwrap();
                assert!(is_match!(array, &Value::Array(_)));
                match array {
                    &Value::Array(ref a) => {
                        assert!(!a.is_empty());
                        assert!(is_match!(a.index(0), &Value::Integer(1)));
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happenend?"),
        }
    }

    #[test]
    fn test_insert_with_seperator_into_nested_table() {
        let mut toml : Value = toml_from_str(r#"
        [a.b.c]
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("a.b.c.d"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref outer) => {
                assert!(!outer.is_empty());
                let a_tab = outer.get("a");
                assert!(a_tab.is_some());

                let a_tab = a_tab.unwrap();
                assert!(is_match!(a_tab, &Value::Table(_)));
                match a_tab {
                    &Value::Table(ref a) => {
                        assert!(!a.is_empty());

                        let b_tab = a.get("b");
                        assert!(b_tab.is_some());

                        let b_tab = b_tab.unwrap();
                        assert!(is_match!(b_tab, &Value::Table(_)));
                        match b_tab {
                            &Value::Table(ref b) => {
                                assert!(!b.is_empty());

                                let c_tab = b.get("c");
                                assert!(c_tab.is_some());

                                let c_tab = c_tab.unwrap();
                                assert!(is_match!(c_tab, &Value::Table(_)));
                                match c_tab {
                                    &Value::Table(ref c) => {
                                        assert!(!c.is_empty());

                                        let d = c.get("d");
                                        assert!(d.is_some());

                                        let d = d.unwrap();
                                        assert!(is_match!(d, &Value::Integer(1)));
                                    },
                                    _ => panic!("What just happenend?"),
                                }
                            },
                            _ => panic!("What just happenend?"),
                        }
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happened?"),
        }
    }

}

<commit_msg>Remove unneeded lifetimes<commit_after>/// The Toml Insert extensions

use toml::Value;

use tokenizer::Token;
use tokenizer::tokenize_with_seperator;
use error::*;

pub trait TomlValueInsertExt {

    /// Extension function for inserting a value in the current toml::Value document
    /// using a custom seperator.
    ///
    /// For difference to TomlSetExt::set() and friends, read [#semantics].
    ///
    /// # Semantics
    ///
    /// The function automatically creates intermediate data structures based on the query string.
    /// That means, if the query string is `"a.b.c.[0]"`, but only a table `"a"` exists in the
    /// document, the function automatically creates a table `"b"` inside `"a"` and `"c"` inside
    /// `"b"`, and an array in `"c"`. The array index is ignored if the array is created.
    ///
    /// If an Array exists, but the specified index is larger than the last index, the array will
    /// be expanded by one element: If the array has a length of 3, but the query string specifies
    /// that the element should be put at 1000, the function ignores the large index and simply
    /// appends the value to the index.
    ///
    /// # Return value
    ///
    /// If the insert operation worked correctly, `Ok(None)` is returned.
    /// If the insert operation replaced an existing value `Ok(Some(old_value))` is returned
    /// On failure, `Err(e)` is returned
    ///
    fn insert_with_seperator(&mut self, query: &String, sep: char, value: Value) -> Result<Option<Value>>;

    /// Extension function for inserting a value from the current toml::Value document
    ///
    /// See documentation of `TomlValueinsertExt::insert_with_seperator`
    fn insert(&mut self, query: &String, value: Value) -> Result<Option<Value>> {
        self.insert_with_seperator(query, '.', value)
    }

}

impl TomlValueInsertExt for Value {

    fn insert_with_seperator(&mut self, query: &String, sep: char, value: Value) -> Result<Option<Value>> {
        use resolver::mut_resolver::resolve;

        let mut tokens = try!(tokenize_with_seperator(query, sep));
        let last       = tokens.pop_last().unwrap();
        let mut val    = try!(resolve(self, &tokens));

        match *last {
            Token::Identifier { ident, .. } => {
                match val {
                    &mut Value::Table(ref mut t) => {
                        Ok(t.insert(ident, value))
                    },
                    _ => unimplemented!()
                }
            },

            Token::Index { idx , .. } => {
                match val {
                    &mut Value::Array(ref mut a) => {
                        if a.len() > idx {
                            unimplemented!()
                        } else {
                            a.push(value);
                            Ok(None)
                        }
                    },
                    _ => unimplemented!()
                }
            },
        }
    }

}

#[cfg(test)]
mod test {
    use super::*;
    use error::*;
    use toml::Value;
    use toml::from_str as toml_from_str;

    #[test]
    fn test_insert_with_seperator_into_table() {
        let mut toml : Value = toml_from_str(r#"
        [table]
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("table.a"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref t) => {
                assert!(!t.is_empty());

                let table = t.get("table");
                assert!(table.is_some());

                let table = table.unwrap();
                assert!(is_match!(table, &Value::Table(_)));
                match table {
                    &Value::Table(ref t) => {
                        assert!(!t.is_empty());

                        let a = t.get("a");
                        assert!(a.is_some());

                        let a = a.unwrap();
                        assert!(is_match!(a, &Value::Integer(1)));
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happenend?"),
        }
    }

    #[test]
    fn test_insert_with_seperator_into_array() {
        use std::ops::Index;

        let mut toml : Value = toml_from_str(r#"
        array = []
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("array.[0]"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref t) => {
                assert!(!t.is_empty());

                let array = t.get("array");
                assert!(array.is_some());

                let array = array.unwrap();
                assert!(is_match!(array, &Value::Array(_)));
                match array {
                    &Value::Array(ref a) => {
                        assert!(!a.is_empty());
                        assert!(is_match!(a.index(0), &Value::Integer(1)));
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happenend?"),
        }
    }

    #[test]
    fn test_insert_with_seperator_into_nested_table() {
        let mut toml : Value = toml_from_str(r#"
        [a.b.c]
        "#).unwrap();

        let res = toml.insert_with_seperator(&String::from("a.b.c.d"), '.', Value::Integer(1));

        assert!(res.is_ok());

        let res = res.unwrap();
        assert!(res.is_none());

        assert!(is_match!(toml, Value::Table(_)));
        match toml {
            Value::Table(ref outer) => {
                assert!(!outer.is_empty());
                let a_tab = outer.get("a");
                assert!(a_tab.is_some());

                let a_tab = a_tab.unwrap();
                assert!(is_match!(a_tab, &Value::Table(_)));
                match a_tab {
                    &Value::Table(ref a) => {
                        assert!(!a.is_empty());

                        let b_tab = a.get("b");
                        assert!(b_tab.is_some());

                        let b_tab = b_tab.unwrap();
                        assert!(is_match!(b_tab, &Value::Table(_)));
                        match b_tab {
                            &Value::Table(ref b) => {
                                assert!(!b.is_empty());

                                let c_tab = b.get("c");
                                assert!(c_tab.is_some());

                                let c_tab = c_tab.unwrap();
                                assert!(is_match!(c_tab, &Value::Table(_)));
                                match c_tab {
                                    &Value::Table(ref c) => {
                                        assert!(!c.is_empty());

                                        let d = c.get("d");
                                        assert!(d.is_some());

                                        let d = d.unwrap();
                                        assert!(is_match!(d, &Value::Integer(1)));
                                    },
                                    _ => panic!("What just happenend?"),
                                }
                            },
                            _ => panic!("What just happenend?"),
                        }
                    },
                    _ => panic!("What just happenend?"),
                }
            },
            _ => panic!("What just happened?"),
        }
    }

}

<|endoftext|>


<commit_before>use rustler::types::truthy::Truthy;
use rustler::Encoder;
use rustler::{NifMap, NifRecord, NifStruct, NifTuple, NifUnitEnum, NifUntaggedEnum};

#[derive(NifTuple)]
pub struct AddTuple {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn tuple_echo(tuple: AddTuple) -> AddTuple {
    tuple
}

#[derive(NifRecord)]
#[rustler(encode, decode)] // Added to check encode/decode attribute, #180
#[must_use] // Added to check attribute order (see similar issue #152)
#[tag = "record"]
pub struct AddRecord {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn record_echo(record: AddRecord) -> AddRecord {
    record
}

#[derive(NifMap)]
pub struct AddMap {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn map_echo(map: AddMap) -> AddMap {
    map
}

#[derive(Debug, NifStruct)]
#[must_use] // Added to test Issue #152
#[module = "AddStruct"]
pub struct AddStruct {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn struct_echo(add_struct: AddStruct) -> AddStruct {
    add_struct
}

#[derive(NifUnitEnum)]
pub enum UnitEnum {
    FooBar,
    Baz,
}

#[rustler::nif]
pub fn unit_enum_echo(unit_enum: UnitEnum) -> UnitEnum {
    unit_enum
}

#[derive(NifUntaggedEnum)]
pub enum UntaggedEnum {
    Foo(u32),
    Bar(String),
    Baz(AddStruct),
    Bool(bool),
}

#[rustler::nif]
pub fn untagged_enum_echo(untagged_enum: UntaggedEnum) -> UntaggedEnum {
    untagged_enum
}

#[derive(NifUntaggedEnum)]
pub enum UntaggedEnumWithTruthy {
    Baz(AddStruct),
    Truthy(Truthy),
}

#[rustler::nif]
pub fn untagged_enum_with_truthy(untagged_enum: UntaggedEnumWithTruthy) -> UntaggedEnumWithTruthy {
    untagged_enum
}

#[derive(NifTuple)]
pub struct Newtype(i64);

#[rustler::nif]
pub fn newtype_echo(newtype: Newtype) -> Newtype {
    newtype
}

#[derive(NifTuple)]
pub struct TupleStruct(i64, i64, i64);

#[rustler::nif]
pub fn tuplestruct_echo(tuplestruct: TupleStruct) -> TupleStruct {
    tuplestruct
}

#[derive(NifRecord)]
#[tag = "newtype"]
pub struct NewtypeRecord(i64);

#[rustler::nif]
pub fn newtype_record_echo(newtype: NewtypeRecord) -> NewtypeRecord {
    newtype
}

#[derive(NifRecord)]
#[tag = "tuplestruct"]
pub struct TupleStructRecord(i64, i64, i64);

#[rustler::nif]
pub fn tuplestruct_record_echo(tuplestruct: TupleStructRecord) -> TupleStructRecord {
    tuplestruct
}

pub mod reserved_keywords {
    use rustler::{Encoder, NifMap, NifRecord, NifStruct, NifTuple, NifUntaggedEnum};

    #[derive(NifMap, Debug)]
    pub struct Map {
        r#override: i32,
    }

    #[derive(NifStruct, Debug)]
    #[module = "Struct"]
    pub struct Struct {
        r#override: i32,
    }

    #[derive(NifTuple, Debug)]
    pub struct Tuple {
        r#override: i32,
    }

    #[derive(NifRecord, Debug)]
    #[tag = "record"]
    pub struct Record {
        r#override: i32,
    }

    #[derive(NifUntaggedEnum)]
    pub enum ReservedKeywords {
        Struct(Struct),
        Map(Map),
        Tuple(Tuple),
        Record(Record),
    }

    #[rustler::nif]
    pub fn reserved_keywords_type_echo(reserved: ReservedKeywords) -> ReservedKeywords {
        reserved
    }
}
<commit_msg>Remove superfluous use from test_codegen<commit_after>use rustler::types::truthy::Truthy;
use rustler::{NifMap, NifRecord, NifStruct, NifTuple, NifUnitEnum, NifUntaggedEnum};

#[derive(NifTuple)]
pub struct AddTuple {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn tuple_echo(tuple: AddTuple) -> AddTuple {
    tuple
}

#[derive(NifRecord)]
#[rustler(encode, decode)] // Added to check encode/decode attribute, #180
#[must_use] // Added to check attribute order (see similar issue #152)
#[tag = "record"]
pub struct AddRecord {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn record_echo(record: AddRecord) -> AddRecord {
    record
}

#[derive(NifMap)]
pub struct AddMap {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn map_echo(map: AddMap) -> AddMap {
    map
}

#[derive(Debug, NifStruct)]
#[must_use] // Added to test Issue #152
#[module = "AddStruct"]
pub struct AddStruct {
    lhs: i32,
    rhs: i32,
}

#[rustler::nif]
pub fn struct_echo(add_struct: AddStruct) -> AddStruct {
    add_struct
}

#[derive(NifUnitEnum)]
pub enum UnitEnum {
    FooBar,
    Baz,
}

#[rustler::nif]
pub fn unit_enum_echo(unit_enum: UnitEnum) -> UnitEnum {
    unit_enum
}

#[derive(NifUntaggedEnum)]
pub enum UntaggedEnum {
    Foo(u32),
    Bar(String),
    Baz(AddStruct),
    Bool(bool),
}

#[rustler::nif]
pub fn untagged_enum_echo(untagged_enum: UntaggedEnum) -> UntaggedEnum {
    untagged_enum
}

#[derive(NifUntaggedEnum)]
pub enum UntaggedEnumWithTruthy {
    Baz(AddStruct),
    Truthy(Truthy),
}

#[rustler::nif]
pub fn untagged_enum_with_truthy(untagged_enum: UntaggedEnumWithTruthy) -> UntaggedEnumWithTruthy {
    untagged_enum
}

#[derive(NifTuple)]
pub struct Newtype(i64);

#[rustler::nif]
pub fn newtype_echo(newtype: Newtype) -> Newtype {
    newtype
}

#[derive(NifTuple)]
pub struct TupleStruct(i64, i64, i64);

#[rustler::nif]
pub fn tuplestruct_echo(tuplestruct: TupleStruct) -> TupleStruct {
    tuplestruct
}

#[derive(NifRecord)]
#[tag = "newtype"]
pub struct NewtypeRecord(i64);

#[rustler::nif]
pub fn newtype_record_echo(newtype: NewtypeRecord) -> NewtypeRecord {
    newtype
}

#[derive(NifRecord)]
#[tag = "tuplestruct"]
pub struct TupleStructRecord(i64, i64, i64);

#[rustler::nif]
pub fn tuplestruct_record_echo(tuplestruct: TupleStructRecord) -> TupleStructRecord {
    tuplestruct
}

pub mod reserved_keywords {
    use rustler::{NifMap, NifRecord, NifStruct, NifTuple, NifUntaggedEnum};

    #[derive(NifMap, Debug)]
    pub struct Map {
        r#override: i32,
    }

    #[derive(NifStruct, Debug)]
    #[module = "Struct"]
    pub struct Struct {
        r#override: i32,
    }

    #[derive(NifTuple, Debug)]
    pub struct Tuple {
        r#override: i32,
    }

    #[derive(NifRecord, Debug)]
    #[tag = "record"]
    pub struct Record {
        r#override: i32,
    }

    #[derive(NifUntaggedEnum)]
    pub enum ReservedKeywords {
        Struct(Struct),
        Map(Map),
        Tuple(Tuple),
        Record(Record),
    }

    #[rustler::nif]
    pub fn reserved_keywords_type_echo(reserved: ReservedKeywords) -> ReservedKeywords {
        reserved
    }
}
<|endoftext|>


<commit_before>#![allow(non_snake_case)]

//! A few linear algebra operations on two-dimensional arrays.

use libnum::{Num, zero, one, Zero, One};
use std::num::Float;
#[cfg(not(nocomplex))]
use libnum::Complex;
use std::ops::{Add, Sub, Mul, Div};

use super::{Array, Ix};

/// Column vector.
pub type Col = Array;
/// Rectangular matrix.
pub type Mat = Array;

/// Trait union for a ring with 1.
pub trait Ring : Clone + Zero + Add<Output=Self> + Sub<Output=Self>
    + One + Mul<Output=Self> { }
impl<A: Clone + Zero + Add<Output=A> + Sub<Output=A> + One + Mul<Output=A>> Ring for A { }

/// Trait union for a field.
pub trait Field : Ring + Div<Output=Self> { }
impl<A: Ring + Div<Output=A>> Field for A { }

/// A real or complex number.
pub trait ComplexField : Copy + Field
{
    #[inline]
    fn conjugate(self) -> Self { self }
    fn sqrt_real(self) -> Self;
    #[inline]
    fn is_complex() -> bool { false }
}

impl ComplexField for f32
{
    #[inline]
    fn sqrt_real(self) -> f32 { self.sqrt() }
}

impl ComplexField for f64
{
    #[inline]
    fn sqrt_real(self) -> f64 { self.sqrt() }
}

#[cfg(not(nocomplex))]
impl ComplexField for Complex
{
    #[inline]
    fn conjugate(self) -> Complex { self.conj() }
    fn sqrt_real(self) -> Complex { Complex::new(self.re.sqrt(), zero()) }
    #[inline]
    fn is_complex() -> bool { true }
}

/// Return the identity matrix of dimension *n*.
pub fn eye(n: Ix) -> Mat
{
    let mut eye = Array::zeros((n, n));
    for a_ii in eye.diag_iter_mut() {
        *a_ii = one::();
    }
    eye
}

/*
/// Return the inverse matrix of square matrix `a`.
pub fn inverse(a: &Mat) -> Mat
{
    fail!()
}
*/

/// Solve *a x = b* with linear least squares approximation.
///
/// It is used to find the best fit for an overdetermined system,
/// i.e. the number of rows in *a* is larger than the number of
/// unknowns *x*.
///
/// Return best fit for *x*.
pub fn least_squares(a: &Mat, b: &Col) -> Col
{
    // Using transpose: a.T a x = a.T b;
    // a.T a being square gives naive solution
    // x_lstsq = inv(a.T a) a.T b
    //
    // Solve using cholesky decomposition
    // aT a x = aT b
    //
    // Factor aT a into L L.T
    //
    // L L.T x = aT b
    //
    // => L z = aT b
    //  fw subst for z
    // => L.T x = z
    //  bw subst for x estimate
    //
    let mut aT = a.clone();
    aT.swap_axes(0, 1);
    if ::is_complex() {
        // conjugate transpose
        for elt in aT.iter_mut() {
            *elt = elt.conjugate();
        }
    }

    let aT_a = aT.mat_mul(a);
    let mut L = cholesky(aT_a);
    let rhs = aT.mat_mul_col(b);

    // Solve L z = aT b
    let z = subst_fw(&L, &rhs);

    // Solve L.T x = z
    if ::is_complex() {
        // conjugate transpose
        // only elements below the diagonal have imag part
        let (m, _) = L.dim();
        for i in (1..m) {
            for j in (0..i) {
                let elt = &mut L[(i, j)];
                *elt = elt.conjugate();
            }
        }
    }
    L.swap_axes(0, 1);

    // => x_lstsq
    subst_bw(&L, &z)
}

/// Factor *a = L LT*.
///
/// *a* should be a square matrix, hermitian and positive definite.
///
/// https://en.wikipedia.org/wiki/Cholesky_decomposition
///
/// “The Cholesky decomposition is mainly used for the numerical solution of
/// linear equations Ax = b.
///
/// If A is symmetric and positive definite, then we can solve Ax = b by first
/// computing the Cholesky decomposition A = LL*, then solving Ly = b for y by
/// forward substitution, and finally solving L*x = y for x by back
/// substitution.”
///
/// Return L.
pub fn cholesky(a: Mat) -> Mat
{
    let z = zero::();
    let (m, n) = a.dim();
    assert!(m == n);
    // Perform the operation in-place on `a`
    let mut L = a;
    for i in (0..m) {
        // Entries 0 .. i before the diagonal
        for j in (0..i) {
            // A = (
            // L²_1,1
            // L_2,1 L_1,1  L²_2,1 + L²_2,2
            // L_3,1 L_1,1  L_3,1 L_2,1 + L_3,2 L_2,2  L²_3,1 + L²_3,2 + L²_3,3
            // .. )
            let mut lik_ljk_sum = z.clone();
            {
                // L_ik for k = 0 .. j
                // L_jk for k = 0 .. j
                let Lik = L.row_iter(i);
                let Ljk = L.row_iter(j);
                for (&lik, &ljk) in Lik.zip(Ljk).take(j as usize) {
                    lik_ljk_sum = lik_ljk_sum + lik * ljk.conjugate();
                }
            }

            // L_ij = [ A_ij - Sum(k = 1 .. j) L_ik L_jk ] / L_jj
            L[(i, j)] = (L[(i, j)] - lik_ljk_sum) / L[(j, j)];
        }

        // Diagonal where i == j
        // L_jj = Sqrt[ A_jj - Sum(k = 1 .. j) L_jk L_jk ]
        let j = i;
        let mut ljk_sum = z.clone();
        // L_jk for k = 0 .. j
        for &ljk in L.row_iter(j).take(j as usize) {
            ljk_sum = ljk_sum + ljk * ljk.conjugate();
        }
        L[(j, j)] = (L[(j, j)] - ljk_sum).sqrt_real();

        // After the diagonal
        // L_ij = 0 for j > i
        for j in (i + 1..n) {
            L[(i, j)] = z.clone();
        }
    }
    L
}

fn vec_elem(elt: A, n: usize) -> Vec
{
    let mut v = Vec::with_capacity(n);
    for _ in (0..n) {
        v.push(elt);
    }
    v
}

/// Solve *L x = b* where *L* is a lower triangular matrix.
pub fn subst_fw(l: &Mat, b: &Col) -> Col
{
    let (m, n) = l.dim();
    assert!(m == n);
    assert!(m == b.dim());
    let mut x = vec_elem(zero::(), m as usize);
    for (i, bi) in b.indexed_iter() {
        // b_lx_sum = b[i] - Sum(for j = 0 .. i) L_ij x_j
        let mut b_lx_sum = bi.clone();
        for (lij, xj) in l.row_iter(i).zip(x.iter()).take(i as usize) {
            b_lx_sum = b_lx_sum - (*lij) * (*xj)
        }
        x.as_mut_slice()[i as usize] = b_lx_sum / l[(i, i)];
    }
    Array::from_vec(x)
}

/// Solve *U x = b* where *U* is an upper triangular matrix.
pub fn subst_bw(u: &Mat, b: &Col) -> Col
{
    let (m, n) = u.dim();
    assert!(m == n);
    assert!(m == b.dim());
    let mut x = vec_elem(zero::(), m as usize);
    for i in (0..m).rev() {
        // b_ux_sum = b[i] - Sum(for j = i .. m) U_ij x_j
        let mut b_ux_sum = b[i].clone();
        for (uij, xj) in u.row_iter(i).rev().zip(x.iter().rev()).take((m - i - 1) as usize) {
            b_ux_sum = b_ux_sum - (*uij) * (*xj);
        }
        x.as_mut_slice()[i as usize] = b_ux_sum / u[(i, i)];
    }
    Array::from_vec(x)
}
<commit_msg>Rm redundant slice call<commit_after>#![allow(non_snake_case)]

//! A few linear algebra operations on two-dimensional arrays.

use libnum::{Num, zero, one, Zero, One};
use std::num::Float;
#[cfg(not(nocomplex))]
use libnum::Complex;
use std::ops::{Add, Sub, Mul, Div};

use super::{Array, Ix};

/// Column vector.
pub type Col = Array;
/// Rectangular matrix.
pub type Mat = Array;

/// Trait union for a ring with 1.
pub trait Ring : Clone + Zero + Add<Output=Self> + Sub<Output=Self>
    + One + Mul<Output=Self> { }
impl<A: Clone + Zero + Add<Output=A> + Sub<Output=A> + One + Mul<Output=A>> Ring for A { }

/// Trait union for a field.
pub trait Field : Ring + Div<Output=Self> { }
impl<A: Ring + Div<Output=A>> Field for A { }

/// A real or complex number.
pub trait ComplexField : Copy + Field
{
    #[inline]
    fn conjugate(self) -> Self { self }
    fn sqrt_real(self) -> Self;
    #[inline]
    fn is_complex() -> bool { false }
}

impl ComplexField for f32
{
    #[inline]
    fn sqrt_real(self) -> f32 { self.sqrt() }
}

impl ComplexField for f64
{
    #[inline]
    fn sqrt_real(self) -> f64 { self.sqrt() }
}

#[cfg(not(nocomplex))]
impl ComplexField for Complex
{
    #[inline]
    fn conjugate(self) -> Complex { self.conj() }
    fn sqrt_real(self) -> Complex { Complex::new(self.re.sqrt(), zero()) }
    #[inline]
    fn is_complex() -> bool { true }
}

/// Return the identity matrix of dimension *n*.
pub fn eye(n: Ix) -> Mat
{
    let mut eye = Array::zeros((n, n));
    for a_ii in eye.diag_iter_mut() {
        *a_ii = one::();
    }
    eye
}

/*
/// Return the inverse matrix of square matrix `a`.
pub fn inverse(a: &Mat) -> Mat
{
    fail!()
}
*/

/// Solve *a x = b* with linear least squares approximation.
///
/// It is used to find the best fit for an overdetermined system,
/// i.e. the number of rows in *a* is larger than the number of
/// unknowns *x*.
///
/// Return best fit for *x*.
pub fn least_squares(a: &Mat, b: &Col) -> Col
{
    // Using transpose: a.T a x = a.T b;
    // a.T a being square gives naive solution
    // x_lstsq = inv(a.T a) a.T b
    //
    // Solve using cholesky decomposition
    // aT a x = aT b
    //
    // Factor aT a into L L.T
    //
    // L L.T x = aT b
    //
    // => L z = aT b
    //  fw subst for z
    // => L.T x = z
    //  bw subst for x estimate
    //
    let mut aT = a.clone();
    aT.swap_axes(0, 1);
    if ::is_complex() {
        // conjugate transpose
        for elt in aT.iter_mut() {
            *elt = elt.conjugate();
        }
    }

    let aT_a = aT.mat_mul(a);
    let mut L = cholesky(aT_a);
    let rhs = aT.mat_mul_col(b);

    // Solve L z = aT b
    let z = subst_fw(&L, &rhs);

    // Solve L.T x = z
    if ::is_complex() {
        // conjugate transpose
        // only elements below the diagonal have imag part
        let (m, _) = L.dim();
        for i in (1..m) {
            for j in (0..i) {
                let elt = &mut L[(i, j)];
                *elt = elt.conjugate();
            }
        }
    }
    L.swap_axes(0, 1);

    // => x_lstsq
    subst_bw(&L, &z)
}

/// Factor *a = L LT*.
///
/// *a* should be a square matrix, hermitian and positive definite.
///
/// https://en.wikipedia.org/wiki/Cholesky_decomposition
///
/// “The Cholesky decomposition is mainly used for the numerical solution of
/// linear equations Ax = b.
///
/// If A is symmetric and positive definite, then we can solve Ax = b by first
/// computing the Cholesky decomposition A = LL*, then solving Ly = b for y by
/// forward substitution, and finally solving L*x = y for x by back
/// substitution.”
///
/// Return L.
pub fn cholesky(a: Mat) -> Mat
{
    let z = zero::();
    let (m, n) = a.dim();
    assert!(m == n);
    // Perform the operation in-place on `a`
    let mut L = a;
    for i in (0..m) {
        // Entries 0 .. i before the diagonal
        for j in (0..i) {
            // A = (
            // L²_1,1
            // L_2,1 L_1,1  L²_2,1 + L²_2,2
            // L_3,1 L_1,1  L_3,1 L_2,1 + L_3,2 L_2,2  L²_3,1 + L²_3,2 + L²_3,3
            // .. )
            let mut lik_ljk_sum = z.clone();
            {
                // L_ik for k = 0 .. j
                // L_jk for k = 0 .. j
                let Lik = L.row_iter(i);
                let Ljk = L.row_iter(j);
                for (&lik, &ljk) in Lik.zip(Ljk).take(j as usize) {
                    lik_ljk_sum = lik_ljk_sum + lik * ljk.conjugate();
                }
            }

            // L_ij = [ A_ij - Sum(k = 1 .. j) L_ik L_jk ] / L_jj
            L[(i, j)] = (L[(i, j)] - lik_ljk_sum) / L[(j, j)];
        }

        // Diagonal where i == j
        // L_jj = Sqrt[ A_jj - Sum(k = 1 .. j) L_jk L_jk ]
        let j = i;
        let mut ljk_sum = z.clone();
        // L_jk for k = 0 .. j
        for &ljk in L.row_iter(j).take(j as usize) {
            ljk_sum = ljk_sum + ljk * ljk.conjugate();
        }
        L[(j, j)] = (L[(j, j)] - ljk_sum).sqrt_real();

        // After the diagonal
        // L_ij = 0 for j > i
        for j in (i + 1..n) {
            L[(i, j)] = z.clone();
        }
    }
    L
}

fn vec_elem(elt: A, n: usize) -> Vec
{
    let mut v = Vec::with_capacity(n);
    for _ in (0..n) {
        v.push(elt);
    }
    v
}

/// Solve *L x = b* where *L* is a lower triangular matrix.
pub fn subst_fw(l: &Mat, b: &Col) -> Col
{
    let (m, n) = l.dim();
    assert!(m == n);
    assert!(m == b.dim());
    let mut x = vec_elem(zero::(), m as usize);
    for (i, bi) in b.indexed_iter() {
        // b_lx_sum = b[i] - Sum(for j = 0 .. i) L_ij x_j
        let mut b_lx_sum = bi.clone();
        for (lij, xj) in l.row_iter(i).zip(x.iter()).take(i as usize) {
            b_lx_sum = b_lx_sum - (*lij) * (*xj)
        }
        x[i as usize] = b_lx_sum / l[(i, i)];
    }
    Array::from_vec(x)
}

/// Solve *U x = b* where *U* is an upper triangular matrix.
pub fn subst_bw(u: &Mat, b: &Col) -> Col
{
    let (m, n) = u.dim();
    assert!(m == n);
    assert!(m == b.dim());
    let mut x = vec_elem(zero::(), m as usize);
    for i in (0..m).rev() {
        // b_ux_sum = b[i] - Sum(for j = i .. m) U_ij x_j
        let mut b_ux_sum = b[i].clone();
        for (uij, xj) in u.row_iter(i).rev().zip(x.iter().rev()).take((m - i - 1) as usize) {
            b_ux_sum = b_ux_sum - (*uij) * (*xj);
        }
        x[i as usize] = b_ux_sum / u[(i, i)];
    }
    Array::from_vec(x)
}
<|endoftext|>


<commit_before>//! Warning: extreme macros

/// This macro creates structs that implement `ToTable` and `FromTable`.
///
/// # Examples:
///
/// ```rust
/// lua_convertible! {
///     #[derive(Debug)]
///     // #[attribtue]
///     struct Point {
///         x: i32,
///         y: i32
///     }
/// }
/// ```
#[macro_export]
macro_rules! lua_convertible {
    (  $(#[$attr:meta])*
       struct $name:ident { $($fname:ident : $ftype:ty),+  }  ) => {

        $(#[$attr])*
        pub struct $name {
            $($fname: $ftype),+
        }

        impl $crate::convert::ToTable for $name {
            fn to_table(self) -> hlua::any::AnyLuaValue {
                hlua::any::AnyLuaValue::LuaArray(vec![
                    $(  (hlua::any::AnyLuaValue::LuaString(stringify!($fname).to_string()),
                         self.$fname.to_table())  ),+
                ])
            }
        }

        impl $crate::convert::FromTable for $name {
            #[allow(unused_variables)]
            fn from_table(decoder: $crate::convert::LuaDecoder) ->
                $crate::convert::ConvertResult<$name> {
                $(  let (decoder, $fname) =
                    try!(decoder.read_field(stringify!($fname).to_string())); )+

                Ok($name {
                    $( $fname: $fname ),+
                })
            }
        }
    }
}

/// Create a keypress using fewer keystrokes. Provides a custom panic method.
#[macro_export]
macro_rules! keypress {
    ($modifier:expr, $key:expr) => {
        KeyPress::from_key_names(vec![$modifier],
                                 vec![$key])
            .expect(concat!("Unable to create keypress from macro with ",
                            $modifier, " and ", $key))
    };
}

#[cfg(test)]
mod tests {
    use super::super::convert::{ToTable, FromTable, LuaDecoder};
    use hlua;
    lua_convertible! {
        #[derive(Debug, Clone, PartialEq)]
        struct Point {
            x: f32,
            y: f32
        }
    }

    #[test]
    fn test_lua_convertible() {
        let point = Point { x: 0f32, y: 0f32 };
        let lua_point = point.clone().to_table();
        let maybe_point = Point::from_table(LuaDecoder::new(lua_point));
        assert!(maybe_point.is_ok());
        let parsed_point = maybe_point.unwrap();
        assert_eq!(parsed_point, point);
    }
}
<commit_msg>Add test for keypress macro<commit_after>//! Warning: extreme macros

/// This macro creates structs that implement `ToTable` and `FromTable`.
///
/// # Examples:
///
/// ```rust
/// lua_convertible! {
///     #[derive(Debug)]
///     // #[attribtue]
///     struct Point {
///         x: i32,
///         y: i32
///     }
/// }
/// ```
#[macro_export]
macro_rules! lua_convertible {
    (  $(#[$attr:meta])*
       struct $name:ident { $($fname:ident : $ftype:ty),+  }  ) => {

        $(#[$attr])*
        pub struct $name {
            $($fname: $ftype),+
        }

        impl $crate::convert::ToTable for $name {
            fn to_table(self) -> hlua::any::AnyLuaValue {
                hlua::any::AnyLuaValue::LuaArray(vec![
                    $(  (hlua::any::AnyLuaValue::LuaString(stringify!($fname).to_string()),
                         self.$fname.to_table())  ),+
                ])
            }
        }

        impl $crate::convert::FromTable for $name {
            #[allow(unused_variables)]
            fn from_table(decoder: $crate::convert::LuaDecoder) ->
                $crate::convert::ConvertResult<$name> {
                $(  let (decoder, $fname) =
                    try!(decoder.read_field(stringify!($fname).to_string())); )+

                Ok($name {
                    $( $fname: $fname ),+
                })
            }
        }
    }
}

/// Create a keypress using fewer keystrokes. Provides a custom panic method.
#[macro_export]
macro_rules! keypress {
    ($modifier:expr, $key:expr) => {
        $crate::keys::KeyPress::from_key_names(vec![$modifier],
                                 vec![$key])
            .expect(concat!("Unable to create keypress from macro with ",
                            $modifier, " and ", $key))
    };
}

#[cfg(test)]
mod tests {
    use super::super::convert::{ToTable, FromTable, LuaDecoder};
    use hlua;
    lua_convertible! {
        #[derive(Debug, Clone, PartialEq)]
        struct Point {
            x: f32,
            y: f32
        }
    }

    #[test]
    fn lua_convertible() {
        let point = Point { x: 0f32, y: 0f32 };
        let lua_point = point.clone().to_table();
        let maybe_point = Point::from_table(LuaDecoder::new(lua_point));
        assert!(maybe_point.is_ok());
        let parsed_point = maybe_point.unwrap();
        assert_eq!(parsed_point, point);
    }

    #[test]
    fn keypress() {
        use super::super::keys::KeyPress;
        use std::hash::{SipHasher, Hash};

        let press = KeyPress::from_key_names(vec!["Ctrl"], vec!["p"])
            .expect("Unable to construct regular keypress");
        let press_macro = keypress!("Ctrl", "p");
        let mut hasher = SipHasher::new();
        assert!(press.hash(&mut hasher) == press_macro.hash(&mut hasher),
                "Hashes do not match");
        assert_eq!(press, press_macro);
    }
}
<|endoftext|>


<commit_before>use std::ops::{Add, Sub, Div};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};
use std::cmp::Ordering::Equal;
use std::ops::Drop;
use std::cmp;
use std::path::Path;
use std::io::{BufWriter, Write, BufReader, Read, BufRead};
use std::fs::File;
use std::mem;

#[derive(Clone)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl PartialEq for WordVec {
    fn eq(&self, other: &WordVec) -> bool {
        self.vec == other.vec
    }
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    fn dot_prod(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| x * y)
                .fold(0.0, |x, y| x + y)
    }

    fn magnitude(&self) -> f32 {
        self.vec.iter()
                .map(|x| x * x)
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter().zip(other.vec.iter())
                .map(|(x, y)| (x - y)*(x - y))
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self) {
        let count = self.count as f32;
        for f in &mut self.vec {
            *f /= count;
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl Div<i32> for WordVec {
    type Output = WordVec;

    fn div(mut self, i: i32) -> WordVec {
        for f in &mut self.vec {
            *f /= i as f32;
        }

        self
    }
}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..(newvec.vec.len()) {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

#[derive(Clone, PartialEq, Debug)]
pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_radius: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
    sentence: Vec<Option<usize>>,
}

pub struct WordAcceptor<'a> {
    builder: &'a mut LanguageModelBuilder,
}

impl LanguageModelBuilder {
    pub fn new(window_radius: usize, words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_radius: window_radius,
            words: words,
            word_vecs: word_vecs,
            sentence: vec![],
        }
    }

    pub fn build(mut self) -> LanguageModel {
        for vec in self.word_vecs.iter_mut() {
            vec.normalize();
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_sentence<'a>(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            builder: self,
        }
    }

    fn word_seen(&mut self, word: usize) {
        self.word_vecs[word].count += 1;
    }

    pub fn save(&self, path: &Path) {
        let mut file = BufWriter::new(File::create(path).unwrap());

        write_raw(self.word_vecs.len(), &mut file);
        file.write(&[b'\n']).unwrap();

        for vec in &self.word_vecs {
            file.write(vec.word.as_bytes()).unwrap();
            file.write(&[b':']).unwrap();
            let count = vec.count;
            write_raw(count, &mut file);
            for f in &vec.vec {
                write_raw(*f, &mut file)
            }
            file.write(&[b'\n']).unwrap();
        }

    }

    pub fn load(path: &Path) -> LanguageModelBuilder {
        let mut file = BufReader::new(File::open(path).unwrap());
        let mut word_vecs = Vec::new();

        let size: u64 = unsafe { read_raw::<u64, _>(&mut file) };
        read_byte(b'\n', &mut file);

        for _ in 0..size {
            let mut word: Vec<u8> = Vec::new();
            file.read_until(b':', &mut word).unwrap();
            assert_eq!(Some(b':'), word.pop());
            let word = String::from_utf8(word).unwrap();
            let count: u64 = unsafe { read_raw::<u64, _>(&mut file) };

            let mut vec: Vec<f32> = repeat(0f32).take(size as usize).collect();
            for f in &mut vec {
                unsafe {
                    *f = read_raw::<f32, _>(&mut file);
                }
            }

            read_byte(b'\n', &mut file);

            word_vecs.push(WordVec {
                word: word,
                count: count,
                vec: vec,
            });
        }

        let words: HashMap<_, _> = word_vecs.iter()
                                            .map(|v| v.word.clone())
                                            .enumerate()
                                            .map(|(i, w)| (w, i))
                                            .collect();

        LanguageModelBuilder {
            window_radius: 0,
            words: words,
            word_vecs: word_vecs,
            sentence: Vec::new(),
        }
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: &str) {
        let idx_opt = self.builder.words.get(word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.builder.word_seen(next_idx);
            self.builder.sentence.push(Some(next_idx));
        }
        else {
            self.builder.sentence.push(None);
        }
    }
}

impl<'a> Drop for WordAcceptor<'a> {
    fn drop(&mut self) {
        let LanguageModelBuilder {
            ref mut sentence,
            ref mut word_vecs,
            window_radius,
            ..
        } = *self.builder;

        for (i, &word_idx) in sentence.iter().enumerate() {
            let start = cmp::max(0, i as isize - window_radius as isize) as usize;
            let end = cmp::min(sentence.len(), i + window_radius + 1);
            for &j in &sentence[start..end] {
                match (word_idx, j) {
                    (Some(w), Some(j)) if w != j =>
                        word_vecs[w].inc(j),
                    _ => {},
                }
            }
        }

        sentence.clear();
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        self.words.get(word).map(|i| &self.word_vecs[*i])
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word {
                Some((dist, w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(|| {
            println!("bad vector: {:?}, {:?}", a, b);
            Equal
        }));
        vec_refs.into_iter().map(|x| x.1).collect()
    }

}

fn read_byte<R: Read>(b: u8, read: &mut R) {
    let buf = &mut [0];
    while read.read(buf).unwrap() == 0 {}
    assert_eq!(&[b], buf);
}

unsafe fn read_raw<T: Copy, R: Read>(reader: &mut BufReader<R>) -> T {
    let mut buffer = [0u8; 64];
    let t_size = mem::size_of::<T>();
    assert!(t_size <= buffer.len());

    let mut remainder = t_size;
    while remainder > 0 {
        let bytes_read = reader.read(&mut buffer[(t_size-remainder)..t_size]).unwrap();
        remainder -= bytes_read;
    }

    let bptr: *mut T = mem::transmute((&buffer).as_ptr());
    *bptr
}

fn write_raw<T: Copy, F: Write>(t: T, writer: &mut BufWriter<F>) {
    let mut buffer = [0u8; 64];
    let t_size = mem::size_of::<T>();
    assert!(t_size <= buffer.len());
    unsafe {
        let bptr: *mut T = mem::transmute((&buffer).as_ptr());
        *bptr = t;
    }

    writer.write(&buffer[0..t_size]);
}

#[cfg(test)]
mod test {
    use super::{WordVec, LanguageModel, LanguageModelBuilder};
    use std::path::Path;

    fn get_builder() -> LanguageModelBuilder {
        let words = "foo bar baz blort".words().map(|w| w.to_string()).collect::<Vec<_>>();
        let mut builder = LanguageModelBuilder::new(1, words);

        let input = "x foo bar baz x x x x x x x blort".words();

        {
            let mut acc = builder.new_sentence();
            for word in input {
                acc.add_word(word);
            }
        }

        builder
    }

    #[test]
    fn test_accept_sentences() {
        let model = get_builder().build();
        let foo = model.get("foo").unwrap();
        let baz = model.get("baz").unwrap();
        let bar = model.get("bar").unwrap();
        let blort = model.get("blort").unwrap();

        assert!(foo.distance(baz) < foo.distance(blort));
        assert!(foo.distance(bar) == bar.distance(baz));
    }

    #[test]
    fn test_serialization() {
        let builder = get_builder();
        let path = Path::new("/tmp/model.data");
        builder.save(&path);

        let loaded_model = LanguageModelBuilder::load(path).build();
        assert_eq!(builder.build(), loaded_model);
    }
}
<commit_msg>Use the weird count normalization they use in the glove paper. Doesn't seem like an improvement.<commit_after>use std::ops::{Add, Sub, Div};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};
use std::cmp::Ordering::Equal;
use std::ops::Drop;
use std::cmp;
use std::path::Path;
use std::io::{BufWriter, Write, BufReader, Read, BufRead};
use std::fs::File;
use std::mem;

#[derive(Clone)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl PartialEq for WordVec {
    fn eq(&self, other: &WordVec) -> bool {
        self.vec == other.vec
    }
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    fn dot_prod(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| x * y)
                .fold(0.0, |x, y| x + y)
    }

    fn magnitude(&self) -> f32 {
        self.vec.iter()
                .map(|x| x * x)
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter().zip(other.vec.iter())
                .map(|(x, y)| (x - y)*(x - y))
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self, cooccurences: f32) {
        for f in &mut self.vec {
            *f /= cooccurences;
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl Div<i32> for WordVec {
    type Output = WordVec;

    fn div(mut self, i: i32) -> WordVec {
        for f in &mut self.vec {
            *f /= i as f32;
        }

        self
    }
}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..(newvec.vec.len()) {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

#[derive(Clone, PartialEq, Debug)]
pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_radius: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
    sentence: Vec<Option<usize>>,
}

pub struct WordAcceptor<'a> {
    builder: &'a mut LanguageModelBuilder,
}

impl LanguageModelBuilder {
    pub fn new(window_radius: usize, words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_radius: window_radius,
            words: words,
            word_vecs: word_vecs,
            sentence: vec![],
        }
    }

    pub fn build(mut self) -> LanguageModel {
        let mut cooccurences: Vec<f32> = repeat(0.0).take(self.word_vecs.len()).collect();
        for vec in &self.word_vecs {
            for (count, x) in cooccurences.iter_mut().zip(vec.vec.iter()) {
                *count += *x;
            }
        }

        for (vec, c) in self.word_vecs.iter_mut().zip(cooccurences.iter()) {
            vec.normalize(*c);
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_sentence<'a>(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            builder: self,
        }
    }

    fn word_seen(&mut self, word: usize) {
        self.word_vecs[word].count += 1;
    }

    pub fn save(&self, path: &Path) {
        let mut file = BufWriter::new(File::create(path).unwrap());

        write_raw(self.word_vecs.len(), &mut file);
        file.write(&[b'\n']).unwrap();

        for vec in &self.word_vecs {
            file.write(vec.word.as_bytes()).unwrap();
            file.write(&[b':']).unwrap();
            let count = vec.count;
            write_raw(count, &mut file);
            for f in &vec.vec {
                write_raw(*f, &mut file)
            }
            file.write(&[b'\n']).unwrap();
        }

    }

    pub fn load(path: &Path) -> LanguageModelBuilder {
        let mut file = BufReader::new(File::open(path).unwrap());
        let mut word_vecs = Vec::new();

        let size: u64 = unsafe { read_raw::<u64, _>(&mut file) };
        read_byte(b'\n', &mut file);

        for _ in 0..size {
            let mut word: Vec<u8> = Vec::new();
            file.read_until(b':', &mut word).unwrap();
            assert_eq!(Some(b':'), word.pop());
            let word = String::from_utf8(word).unwrap();
            let count: u64 = unsafe { read_raw::<u64, _>(&mut file) };

            let mut vec: Vec<f32> = repeat(0f32).take(size as usize).collect();
            for f in &mut vec {
                unsafe {
                    *f = read_raw::<f32, _>(&mut file);
                }
            }

            read_byte(b'\n', &mut file);

            word_vecs.push(WordVec {
                word: word,
                count: count,
                vec: vec,
            });
        }

        let words: HashMap<_, _> = word_vecs.iter()
                                            .map(|v| v.word.clone())
                                            .enumerate()
                                            .map(|(i, w)| (w, i))
                                            .collect();

        LanguageModelBuilder {
            window_radius: 0,
            words: words,
            word_vecs: word_vecs,
            sentence: Vec::new(),
        }
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: &str) {
        let idx_opt = self.builder.words.get(word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.builder.word_seen(next_idx);
            self.builder.sentence.push(Some(next_idx));
        }
        else {
            self.builder.sentence.push(None);
        }
    }
}

impl<'a> Drop for WordAcceptor<'a> {
    fn drop(&mut self) {
        let LanguageModelBuilder {
            ref mut sentence,
            ref mut word_vecs,
            window_radius,
            ..
        } = *self.builder;

        for (i, &word_idx) in sentence.iter().enumerate() {
            let start = cmp::max(0, i as isize - window_radius as isize) as usize;
            let end = cmp::min(sentence.len(), i + window_radius + 1);
            for &j in &sentence[start..end] {
                match (word_idx, j) {
                    (Some(w), Some(j)) if w != j =>
                        word_vecs[w].inc(j),
                    _ => {},
                }
            }
        }

        sentence.clear();
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        self.words.get(word).map(|i| &self.word_vecs[*i])
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word {
                Some((dist, w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(|| {
            println!("bad vector: {:?}, {:?}", a, b);
            Equal
        }));
        vec_refs.into_iter().map(|x| x.1).collect()
    }

}

fn read_byte<R: Read>(b: u8, read: &mut R) {
    let buf = &mut [0];
    while read.read(buf).unwrap() == 0 {}
    assert_eq!(&[b], buf);
}

unsafe fn read_raw<T: Copy, R: Read>(reader: &mut BufReader<R>) -> T {
    let mut buffer = [0u8; 64];
    let t_size = mem::size_of::<T>();
    assert!(t_size <= buffer.len());

    let mut remainder = t_size;
    while remainder > 0 {
        let bytes_read = reader.read(&mut buffer[(t_size-remainder)..t_size]).unwrap();
        remainder -= bytes_read;
    }

    let bptr: *mut T = mem::transmute((&buffer).as_ptr());
    *bptr
}

fn write_raw<T: Copy, F: Write>(t: T, writer: &mut BufWriter<F>) {
    let mut buffer = [0u8; 64];
    let t_size = mem::size_of::<T>();
    assert!(t_size <= buffer.len());
    unsafe {
        let bptr: *mut T = mem::transmute((&buffer).as_ptr());
        *bptr = t;
    }

    writer.write(&buffer[0..t_size]);
}

#[cfg(test)]
mod test {
    use super::{WordVec, LanguageModel, LanguageModelBuilder};
    use std::path::Path;

    fn get_builder() -> LanguageModelBuilder {
        let words = "foo bar baz blort".words().map(|w| w.to_string()).collect::<Vec<_>>();
        let mut builder = LanguageModelBuilder::new(1, words);

        let input = "x foo bar baz x x x x x x x blort".words();

        {
            let mut acc = builder.new_sentence();
            for word in input {
                acc.add_word(word);
            }
        }

        builder
    }

    #[test]
    fn test_accept_sentences() {
        let model = get_builder().build();
        let foo = model.get("foo").unwrap();
        let baz = model.get("baz").unwrap();
        let bar = model.get("bar").unwrap();
        let blort = model.get("blort").unwrap();

        assert!(foo.distance(baz) < foo.distance(blort));
        assert!(foo.distance(bar) == bar.distance(baz));
    }

    #[test]
    fn test_serialization() {
        let builder = get_builder();
        let path = Path::new("/tmp/model.data");
        builder.save(&path);

        let loaded_model = LanguageModelBuilder::load(path).build();
        assert_eq!(builder.build(), loaded_model);
    }
}
<|endoftext|>