<commit_before>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![allow(unused_imports, dead_code, unused_variables)]

use crate::compiler::args::*;
use crate::compiler::c::{CCompilerImpl, CCompilerKind, Language, ParsedArguments};
use crate::compiler::gcc::ArgData::*;
use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
use crate::dist;
use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use futures::future::{self, Future};
use futures_cpupool::CpuPool;
use log::Level::Trace;
use std::ffi::OsString;
use std::fs::File;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;

use crate::errors::*;

/// A unit struct on which to implement `CCompilerImpl`.
#[derive(Clone, Debug)]
pub struct NVCC;

impl CCompilerImpl for NVCC {
    fn kind(&self) -> CCompilerKind {
        CCompilerKind::NVCC
    }
    fn parse_arguments(
        &self,
        arguments: &[OsString],
        cwd: &Path,
    ) -> CompilerArguments<ParsedArguments> {
        gcc::parse_arguments(arguments, cwd, (&gcc::ARGS[..], &ARGS[..]))
    }

    fn preprocess<T>(
        &self,
        creator: &T,
        executable: &Path,
        parsed_args: &ParsedArguments,
        cwd: &Path,
        env_vars: &[(OsString, OsString)],
        may_dist: bool,
        rewrite_includes_only: bool,
    ) -> SFuture<process::Output>
    where
        T: CommandCreatorSync,
    {
        trace!("preprocess");
        let mut cmd = creator.clone().new_command_sync(executable);

        //NVCC only supports `-E` when it comes after preprocessor
        //and common flags
        cmd.args(&parsed_args.preprocessor_args);
        cmd.args(&parsed_args.common_args);

        //We need to add "-rdc=true" if we are compiling with `-dc`
        //So that the preprocessor has the correct implicit defines
        if parsed_args.compilation_flag == "-dc" {
            cmd.arg("-rdc=true");
        }

        let language = match parsed_args.language {
            Language::C => "c",
            Language::Cxx => "c++",
            Language::ObjectiveC => "objective-c",
            Language::ObjectiveCxx => "objective-c++",
            Language::Cuda => "cu",
        };

        cmd.arg("-x").arg(language)
           .arg(&parsed_args.input)
           .arg("-E")
           .env_clear()
           .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v)))
           .current_dir(cwd);
        if log_enabled!(Trace) {
            trace!("preprocess: {:?}", cmd);
        }
        run_input_output(cmd, None)
    }

    fn generate_compile_commands(
        &self,
        path_transformer: &mut dist::PathTransformer,
        executable: &Path,
        parsed_args: &ParsedArguments,
        cwd: &Path,
        env_vars: &[(OsString, OsString)],
        rewrite_includes_only: bool,
    ) -> Result<(CompileCommand, Option<dist::CompileCommand>, Cacheable)> {
        gcc::generate_compile_commands(
            path_transformer,
            executable,
            parsed_args,
            cwd,
            env_vars,
            self.kind(),
            rewrite_includes_only,
        )
    }
}

counted_array!(pub static ARGS: [ArgInfo<gcc::ArgData>; _] = [
    // take_arg!("--compiler-bindir", PathBuf, Separated, ExtraHashFile),
    // take_arg!("-ccbin", PathBuf, Separated, ExtraHashFile),

    take_arg!("--archive-options options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--compiler-options", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("--generate-code", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--gpu-architecture", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--gpu-code", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--include-path", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),
    take_arg!("--linker-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--maxrregcount", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--nvlink-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--ptxas-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--relocatable-device-code", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("--system-include", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),

    take_arg!("-Xarchive", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xcompiler", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("-Xlinker", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xnvlink", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xptxas", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-arch", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-code", OsString, CanBeSeparated('='), PassThrough),
    flag!("-dc", DoCompilation),
    take_arg!("-gencode", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-isystem", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),
    take_arg!("-maxrregcount", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-rdc", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("-x", OsString, CanBeSeparated('='), Language),

]);

#[cfg(test)]
mod test {
    use super::*;
    use crate::compiler::gcc;
    use crate::compiler::*;
    use crate::mock_command::*;
    use crate::test::utils::*;
    use futures::Future;
    use futures_cpupool::CpuPool;
    use std::collections::HashMap;
    use std::path::PathBuf;

    fn parse_arguments_(arguments: Vec<String>) -> CompilerArguments<ParsedArguments> {
        let arguments = arguments.iter().map(OsString::from).collect::<Vec<_>>();
        NVCC.parse_arguments(&arguments, ".".as_ref())
    }

    macro_rules! parses {
        ( $( $s:expr ),* ) => {
            match parse_arguments_(vec![ $( $s.to_string(), )* ]) {
                CompilerArguments::Ok(a) => a,
                o => panic!("Got unexpected parse result: {:?}", o),
            }
        }
    }

    #[test]
    fn test_parse_arguments_simple_c() {
        let a = parses!("-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::C, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_simple_cu() {
        let a = parses!("-c", "foo.cu", "-o", "foo.o");
        assert_eq!(Some("foo.cu"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_simple_c_as_cu() {
        let a = parses!("-x","cu", "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_dc_compile_flag() {
        let a = parses!("-x","cu", "-dc", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_eq!(Some("-dc"), a.compilation_flag.to_str());
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_values() {
        let a = parses!("-c", "foo.cpp", "-fabc",
                        "-I", "include-file", "-o", "foo.o",
                        "--include-path", "include-file",
                        "-isystem=/system/include/file");
        assert_eq!(Some("foo.cpp"), a.input.to_str());
        assert_eq!(Language::Cxx, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-Iinclude-file", "--include-path", "include-file", "-isystem", "/system/include/file"],
            a.preprocessor_args
        );
        assert!(a.dependency_args.is_empty());
        assert_eq!(ovec!["-fabc"], a.common_args);
    }

    #[test]
    fn test_parse_md_mt_flags_cu() {
        let a = parses!("-x", "cu", "-c", "foo.c", "-fabc",
                           "-MD", "-MT", "foo.o", "-MF", "foo.o.d", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_eq!(Some("-c"), a.compilation_flag.to_str());
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-MD", "-MF", "foo.o.d", "-MT", "foo.o"],
            a.preprocessor_args
        );
        assert_eq!(ovec!["-fabc"], a.common_args);
    }

    #[test]
    fn test_parse_generate_code_flags() {
        let a = parses!("-x","cu", "--generate-code=arch=compute_61,code=sm_61",
                        "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert_eq!(
            ovec!["--generate-code", "arch=compute_61,code=sm_61"],
            a.common_args
        );
    }

    #[test]
    fn test_parse_pass_to_host_flags() {
        let a = parses!("-x=cu",
                        "--generate-code=arch=compute_60,code=[sm_60,sm_61]",
                        "-Xnvlink=--suppress-stack-size-warning",
                        "-Xcompiler", "-fPIC,-fno-common",
                        "-Xcompiler=-fvisibility=hidden",
                        "-Xcompiler=-Wall,-Wno-unknown-pragmas,-Wno-unused-local-typedefs",
                        "-Xcudafe", "--display_error_number",
                        "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-Xcompiler", "-fPIC,-fno-common", "-Xcompiler", "-fvisibility=hidden",
                  "-Xcompiler", "-Wall,-Wno-unknown-pragmas,-Wno-unused-local-typedefs"],
            a.preprocessor_args
        );
        assert_eq!(
            ovec!["--generate-code", "arch=compute_60,code=[sm_60,sm_61]",
                  "-Xnvlink", "--suppress-stack-size-warning", "-Xcudafe",
                  "--display_error_number"],
            a.common_args
        );
    }

}
<commit_msg>Make hashes path agnostic for NVCC<commit_after>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![allow(unused_imports, dead_code, unused_variables)]

use crate::compiler::args::*;
use crate::compiler::c::{CCompilerImpl, CCompilerKind, Language, ParsedArguments};
use crate::compiler::gcc::ArgData::*;
use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
use crate::dist;
use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use futures::future::{self, Future};
use futures_cpupool::CpuPool;
use log::Level::Trace;
use std::ffi::OsString;
use std::fs::File;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;

use crate::errors::*;

/// A unit struct on which to implement `CCompilerImpl`.
#[derive(Clone, Debug)]
pub struct NVCC;

impl CCompilerImpl for NVCC {
    fn kind(&self) -> CCompilerKind {
        CCompilerKind::NVCC
    }
    fn parse_arguments(
        &self,
        arguments: &[OsString],
        cwd: &Path,
    ) -> CompilerArguments<ParsedArguments> {
        gcc::parse_arguments(arguments, cwd, (&gcc::ARGS[..], &ARGS[..]))
    }

    fn preprocess<T>(
        &self,
        creator: &T,
        executable: &Path,
        parsed_args: &ParsedArguments,
        cwd: &Path,
        env_vars: &[(OsString, OsString)],
        may_dist: bool,
        rewrite_includes_only: bool,
    ) -> SFuture<process::Output>
    where
        T: CommandCreatorSync,
    {
        trace!("preprocess");
        let mut cmd = creator.clone().new_command_sync(executable);

        //NVCC only supports `-E` when it comes after preprocessor
        //and common flags
        cmd.args(&parsed_args.preprocessor_args);
        cmd.args(&parsed_args.common_args);

        //We need to add "-rdc=true" if we are compiling with `-dc`
        //So that the preprocessor has the correct implicit defines
        if parsed_args.compilation_flag == "-dc" {
            cmd.arg("-rdc=true");
        }

        let language = match parsed_args.language {
            Language::C => "c",
            Language::Cxx => "c++",
            Language::ObjectiveC => "objective-c",
            Language::ObjectiveCxx => "objective-c++",
            Language::Cuda => "cu",
        };

        cmd.arg("-x").arg(language)
           .arg(&parsed_args.input)
           .arg("-E")
           .arg("-Xcompiler=-P")
           .env_clear()
           .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v)))
           .current_dir(cwd);
        if log_enabled!(Trace) {
            trace!("preprocess: {:?}", cmd);
        }
        run_input_output(cmd, None)
    }

    fn generate_compile_commands(
        &self,
        path_transformer: &mut dist::PathTransformer,
        executable: &Path,
        parsed_args: &ParsedArguments,
        cwd: &Path,
        env_vars: &[(OsString, OsString)],
        rewrite_includes_only: bool,
    ) -> Result<(CompileCommand, Option<dist::CompileCommand>, Cacheable)> {
        gcc::generate_compile_commands(
            path_transformer,
            executable,
            parsed_args,
            cwd,
            env_vars,
            self.kind(),
            rewrite_includes_only,
        )
    }
}

counted_array!(pub static ARGS: [ArgInfo<gcc::ArgData>; _] = [
    // take_arg!("--compiler-bindir", PathBuf, Separated, ExtraHashFile),
    // take_arg!("-ccbin", PathBuf, Separated, ExtraHashFile),

    take_arg!("--archive-options options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--compiler-options", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("--generate-code", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--gpu-architecture", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--gpu-code", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--include-path", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),
    take_arg!("--linker-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--maxrregcount", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--nvlink-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--ptxas-options", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("--relocatable-device-code", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("--system-include", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),

    take_arg!("-Xarchive", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xcompiler", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("-Xlinker", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xnvlink", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-Xptxas", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-arch", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-code", OsString, CanBeSeparated('='), PassThrough),
    flag!("-dc", DoCompilation),
    take_arg!("-gencode", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-isystem", PathBuf, CanBeSeparated('='), PreprocessorArgumentPath),
    take_arg!("-maxrregcount", OsString, CanBeSeparated('='), PassThrough),
    take_arg!("-rdc", OsString, CanBeSeparated('='), PreprocessorArgument),
    take_arg!("-x", OsString, CanBeSeparated('='), Language),

]);

#[cfg(test)]
mod test {
    use super::*;
    use crate::compiler::gcc;
    use crate::compiler::*;
    use crate::mock_command::*;
    use crate::test::utils::*;
    use futures::Future;
    use futures_cpupool::CpuPool;
    use std::collections::HashMap;
    use std::path::PathBuf;

    fn parse_arguments_(arguments: Vec<String>) -> CompilerArguments<ParsedArguments> {
        let arguments = arguments.iter().map(OsString::from).collect::<Vec<_>>();
        NVCC.parse_arguments(&arguments, ".".as_ref())
    }

    macro_rules! parses {
        ( $( $s:expr ),* ) => {
            match parse_arguments_(vec![ $( $s.to_string(), )* ]) {
                CompilerArguments::Ok(a) => a,
                o => panic!("Got unexpected parse result: {:?}", o),
            }
        }
    }

    #[test]
    fn test_parse_arguments_simple_c() {
        let a = parses!("-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::C, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_simple_cu() {
        let a = parses!("-c", "foo.cu", "-o", "foo.o");
        assert_eq!(Some("foo.cu"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_simple_c_as_cu() {
        let a = parses!("-x","cu", "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_dc_compile_flag() {
        let a = parses!("-x","cu", "-dc", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_eq!(Some("-dc"), a.compilation_flag.to_str());
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert!(a.common_args.is_empty());
    }

    #[test]
    fn test_parse_arguments_values() {
        let a = parses!("-c", "foo.cpp", "-fabc",
                        "-I", "include-file", "-o", "foo.o",
                        "--include-path", "include-file",
                        "-isystem=/system/include/file");
        assert_eq!(Some("foo.cpp"), a.input.to_str());
        assert_eq!(Language::Cxx, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-Iinclude-file", "--include-path", "include-file", "-isystem", "/system/include/file"],
            a.preprocessor_args
        );
        assert!(a.dependency_args.is_empty());
        assert_eq!(ovec!["-fabc"], a.common_args);
    }

    #[test]
    fn test_parse_md_mt_flags_cu() {
        let a = parses!("-x", "cu", "-c", "foo.c", "-fabc",
                           "-MD", "-MT", "foo.o", "-MF", "foo.o.d", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_eq!(Some("-c"), a.compilation_flag.to_str());
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-MD", "-MF", "foo.o.d", "-MT", "foo.o"],
            a.preprocessor_args
        );
        assert_eq!(ovec!["-fabc"], a.common_args);
    }

    #[test]
    fn test_parse_generate_code_flags() {
        let a = parses!("-x","cu", "--generate-code=arch=compute_61,code=sm_61",
                        "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert!(a.preprocessor_args.is_empty());
        assert_eq!(
            ovec!["--generate-code", "arch=compute_61,code=sm_61"],
            a.common_args
        );
    }

    #[test]
    fn test_parse_pass_to_host_flags() {
        let a = parses!("-x=cu",
                        "--generate-code=arch=compute_60,code=[sm_60,sm_61]",
                        "-Xnvlink=--suppress-stack-size-warning",
                        "-Xcompiler", "-fPIC,-fno-common",
                        "-Xcompiler=-fvisibility=hidden",
                        "-Xcompiler=-Wall,-Wno-unknown-pragmas,-Wno-unused-local-typedefs",
                        "-Xcudafe", "--display_error_number",
                        "-c", "foo.c", "-o", "foo.o");
        assert_eq!(Some("foo.c"), a.input.to_str());
        assert_eq!(Language::Cuda, a.language);
        assert_map_contains!(a.outputs, ("obj", PathBuf::from("foo.o")));
        //TODO: fix assert_map_contains to assert no extra keys!
        assert_eq!(1, a.outputs.len());
        assert_eq!(
            ovec!["-Xcompiler", "-fPIC,-fno-common", "-Xcompiler", "-fvisibility=hidden",
                  "-Xcompiler", "-Wall,-Wno-unknown-pragmas,-Wno-unused-local-typedefs"],
            a.preprocessor_args
        );
        assert_eq!(
            ovec!["--generate-code", "arch=compute_60,code=[sm_60,sm_61]",
                  "-Xnvlink", "--suppress-stack-size-warning", "-Xcudafe",
                  "--display_error_number"],
            a.common_args
        );
    }

}
<|endoftext|>


<commit_before><commit_msg>pythagorean-triplet stub<commit_after>pub fn find() -> Option<u32> {
    unimplemented!();
}
<|endoftext|>


<commit_before>#[derive(Debug)]
struct ListNode {
    Val: i32,
    Next: Option<Box<ListNode>>,
}

impl ListNode {
    fn new(i: i32) -> Self {
        ListNode { Val: i, Next: None }
    }

    fn addL(&mut self, l: &Vec<i32>) {
        let org: *mut ListNode = self as *mut ListNode;
        let mut p: *mut ListNode = org;
        for i in l {
            unsafe {
                println!("{:?}", *p);
                (*p).Next = Some(Box::new(ListNode::new(*i)));
                let temp_b = Box::from_raw(p);
                p = Box::into_raw(temp_b.Next.unwrap());
            };
        }
    }
    fn removeNthFromEnd(&self, n: i32) {}
}
fn main() {
    let mut a = ListNode::new(1);
    a.addL(&vec![2, 3, 4, 5]);
    println!("{:?}", a);
}
<commit_msg>a bit<commit_after>#[derive(Debug)]
struct ListNode {
    val: i32,
    next: Option<Box<ListNode>>,
}

impl ListNode {
    fn new(i: i32) -> Self {
        ListNode { val: i, next: None }
    }

    //:= DEBUG: self point changed
    //:= https://stackoverflow.com/questions/50957738/how-to-copy-a-raw-pointer-in-rust?noredirect=1#comment88913158_50957738
    fn add_l(&mut self, l: &Vec<i32>) {
        //let mut p: *mut ListNode = self as *mut ListNode;
        let org = self;
        let mut p: *mut ListNode = org as *mut ListNode;

        for i in l {
            unsafe {
                //println!("{:?}", *p);
                (*p).next = Some(Box::new(ListNode::new(*i)));
                let temp_b = Box::from_raw(p);
                p = Box::into_raw(temp_b.next.unwrap());
            };
        }
    }

    fn add(&mut self, i: &i32) {
        self.next = Some(Box::new(ListNode::new(*i)));
    }

    fn add_head(self, i: i32) -> Self {
        ListNode {
            val: i,
            next: Some(Box::new(self)),
        }
    }

    fn new_l(l: Vec<i32>) -> Self {
        let mut inner_l = l;
        let mut temp = Self::new(*&inner_l.pop().unwrap());
        for i in inner_l.into_iter().rev() {
            temp = temp.add_head(i)
        }
        temp
    }

    fn removeNthFromEnd(&self, n: i32) {}
}
fn main() {
    //let mut a = ListNode::new(1);
    //a.add_l(&vec![2, 3, 4, 5]);

    let a = ListNode::new_l(vec![1, 2, 3, 4, 5]);
    println!("{:?}", a);
}
<|endoftext|>


<commit_before>//! # OpenTelemetry Metrics API

use std::result;
use std::sync::PoisonError;
use thiserror::Error;

mod async_instrument;
mod config;
mod counter;
mod descriptor;
mod kind;
mod meter;
pub mod noop;
mod number;
mod observer;
pub mod registry;
pub mod sdk_api;
mod sync_instrument;
mod up_down_counter;
mod value_recorder;

pub use async_instrument::{AsyncRunner, BatchObserverCallback, Observation, ObserverResult};
pub use config::InstrumentConfig;
pub use counter::{BoundCounter, Counter, CounterBuilder};
pub use descriptor::Descriptor;
pub use kind::InstrumentKind;
pub use meter::{Meter, MeterProvider};
pub use number::{AtomicNumber, Number, NumberKind};
pub use observer::{
    BatchObserver, SumObserver, SumObserverBuilder, UpDownSumObserver, UpDownSumObserverBuilder,
    ValueObserver, ValueObserverBuilder,
};
pub use sync_instrument::Measurement;
pub use up_down_counter::{BoundUpDownCounter, UpDownCounter, UpDownCounterBuilder};
pub use value_recorder::{BoundValueRecorder, ValueRecorder, ValueRecorderBuilder};

/// A specialized `Result` type for metric operations.
pub type Result<T> = result::Result<T, MetricsError>;

/// Errors returned by the metrics API.
#[derive(Error, Debug, PartialEq)]
pub enum MetricsError {
    /// Other errors not covered by specific cases.
    #[error("Metrics error: {0}")]
    Other(String),
    /// Errors when requesting quantiles out of the 0-1 range.
    #[error("The requested quantile is out of range")]
    InvalidQuantile,
    /// Errors when recording nan values.
    #[error("NaN value is an invalid input")]
    NaNInput,
    /// Errors when recording negative values in monotonic sums.
    #[error("Negative value is out of range for this instrument")]
    NegativeInput,
    /// Errors when merging aggregators of incompatible types.
    #[error("Inconsistent aggregator types: {0}")]
    InconsistentAggregator(String),
    /// Errors when requesting data when no data has been collected
    #[error("No data collected by this aggregator")]
    NoDataCollected,
    /// Errors when registering to instruments with the same name and kind
    #[error("A metric was already registered by this name with another kind or number type: {0}")]
    MetricKindMismatch(String),
    /// Errors when processor logic is incorrect
    #[error("Inconsistent processor state")]
    InconsistentState,
    /// Errors when aggregator cannot subtract
    #[error("Aggregator does not subtract")]
    NoSubtraction,
}

impl<T> From<PoisonError<T>> for MetricsError {
    fn from(err: PoisonError<T>) -> Self {
        MetricsError::Other(err.to_string())
    }
}
<commit_msg>Add #[non_exhaustive] to metrics errors (#190)<commit_after>//! # OpenTelemetry Metrics API

use std::result;
use std::sync::PoisonError;
use thiserror::Error;

mod async_instrument;
mod config;
mod counter;
mod descriptor;
mod kind;
mod meter;
pub mod noop;
mod number;
mod observer;
pub mod registry;
pub mod sdk_api;
mod sync_instrument;
mod up_down_counter;
mod value_recorder;

pub use async_instrument::{AsyncRunner, BatchObserverCallback, Observation, ObserverResult};
pub use config::InstrumentConfig;
pub use counter::{BoundCounter, Counter, CounterBuilder};
pub use descriptor::Descriptor;
pub use kind::InstrumentKind;
pub use meter::{Meter, MeterProvider};
pub use number::{AtomicNumber, Number, NumberKind};
pub use observer::{
    BatchObserver, SumObserver, SumObserverBuilder, UpDownSumObserver, UpDownSumObserverBuilder,
    ValueObserver, ValueObserverBuilder,
};
pub use sync_instrument::Measurement;
pub use up_down_counter::{BoundUpDownCounter, UpDownCounter, UpDownCounterBuilder};
pub use value_recorder::{BoundValueRecorder, ValueRecorder, ValueRecorderBuilder};

/// A specialized `Result` type for metric operations.
pub type Result<T> = result::Result<T, MetricsError>;

/// Errors returned by the metrics API.
#[derive(Error, Debug, PartialEq)]
#[non_exhaustive]
pub enum MetricsError {
    /// Other errors not covered by specific cases.
    #[error("Metrics error: {0}")]
    Other(String),
    /// Errors when requesting quantiles out of the 0-1 range.
    #[error("The requested quantile is out of range")]
    InvalidQuantile,
    /// Errors when recording nan values.
    #[error("NaN value is an invalid input")]
    NaNInput,
    /// Errors when recording negative values in monotonic sums.
    #[error("Negative value is out of range for this instrument")]
    NegativeInput,
    /// Errors when merging aggregators of incompatible types.
    #[error("Inconsistent aggregator types: {0}")]
    InconsistentAggregator(String),
    /// Errors when requesting data when no data has been collected
    #[error("No data collected by this aggregator")]
    NoDataCollected,
    /// Errors when registering to instruments with the same name and kind
    #[error("A metric was already registered by this name with another kind or number type: {0}")]
    MetricKindMismatch(String),
    /// Errors when processor logic is incorrect
    #[error("Inconsistent processor state")]
    InconsistentState,
    /// Errors when aggregator cannot subtract
    #[error("Aggregator does not subtract")]
    NoSubtraction,
}

impl<T> From<PoisonError<T>> for MetricsError {
    fn from(err: PoisonError<T>) -> Self {
        MetricsError::Other(err.to_string())
    }
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn fetch_ledger_info(&mut self) -> State {
        self.client
            .get_ledger_information()
            .await
            .expect("ledger info must be present")
            .into_inner()
    }

    pub async fn set_highest_known_version(&mut self) {
        let info = self.client.get_ledger_information().await;
        let res = info.unwrap();
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version == self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                self.set_highest_known_version().await;
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        self.client
            .get_ledger_information()
            .await
            .expect("ledger info must be present")
            .into_inner()
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<commit_msg>[ECO] Fix 'attempt to subtract with overflow' (#3465)<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn fetch_ledger_info(&mut self) -> State {
        self.client
            .get_ledger_information()
            .await
            .expect("ledger info must be present")
            .into_inner()
    }

    pub async fn set_highest_known_version(&mut self) {
        let info = self.client.get_ledger_information().await;
        let res = info.unwrap();
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version >= self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                self.set_highest_known_version().await;
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        self.client
            .get_ledger_information()
            .await
            .expect("ledger info must be present")
            .into_inner()
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<|endoftext|>


<commit_before>use std::collections::BinaryHeap;
use std::fmt;
use std::path::Path;

use config::Config;
use platform::Platform;
use traffic::Traffic;
use workload::Workload;
use {Job, Result, Source};

pub struct System {
    platform: Platform,
    traffic: Traffic,
    workload: Workload,
    queue: BinaryHeap<Event>,
}

time! {
    #[derive(Clone, Debug)]
    pub struct Event {
        pub kind: EventKind,
    }
}

#[derive(Clone, Debug)]
pub enum EventKind {
    Arrival(Job),
    Start(Job),
    Finish(Job),
}

impl System {
    pub fn new<T: AsRef<Path>>(config: T, source: &Source) -> Result<System> {
        let config = try!(Config::new(config));

        let platform = {
            let config = some!(config.branch("platform"), "a platform configuration is required");
            try!(Platform::new(&config))
        };
        let traffic = {
            let config = some!(config.branch("traffic"), "a traffic configuration is required");
            try!(Traffic::new(&config, source))
        };
        let workload = {
            let config = some!(config.branch("workload"), "a workload configuration is required");
            try!(Workload::new(&config, source))
        };

        Ok(System {
            platform: platform,
            traffic: traffic,
            workload: workload,
            queue: BinaryHeap::new(),
        })
    }
}

impl Iterator for System {
    type Item = Event;

    fn next(&mut self) -> Option<Self::Item> {
        match (self.traffic.peek(), self.queue.peek()) {
            (Some(&arrival), Some(&Event { time, .. })) => if time < arrival {
                return self.queue.pop();
            },
            _ => {},
        }

        let job = match (self.traffic.next(), self.workload.next()) {
            (Some(arrival), Some(pattern)) => Job::new(arrival, pattern),
            _ => return None,
        };

        self.queue.push(time!(job.arrival, Event {
            kind: EventKind::Arrival(job.clone()),
        }));

        let (start, finish) = match self.platform.next(&job) {
            Some((start, finish)) => (start, finish),
            _ => return None,
        };

        self.queue.push(time!(start, Event {
            kind: EventKind::Start(job.clone()),
        }));
        self.queue.push(time!(finish, Event {
            kind: EventKind::Finish(job),
        }));

        self.queue.pop()
    }
}

impl fmt::Display for Event {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "{:9.2} s: {:<30}", self.time, &self.kind)
    }
}

impl fmt::Display for EventKind {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        macro_rules! job(
            ($job:expr, $note:expr) => (
                write!(formatter, "{} {}", $job, $note)
            )
        );

        match self {
            &EventKind::Arrival(ref job) => job!(job, "arrival"),
            &EventKind::Start(ref job) => job!(job, "start"),
            &EventKind::Finish(ref job) => job!(job, "finish"),
        }
    }
}
<commit_msg>A bit of refactoring<commit_after>use std::collections::BinaryHeap;
use std::fmt;
use std::path::Path;

use config::Config;
use platform::Platform;
use traffic::Traffic;
use workload::Workload;
use {Job, Result, Source};

pub struct System {
    platform: Platform,
    traffic: Traffic,
    workload: Workload,
    queue: BinaryHeap<Event>,
}

time! {
    #[derive(Clone, Debug)]
    pub struct Event {
        pub kind: EventKind,
    }
}

#[derive(Clone, Debug)]
pub enum EventKind {
    Arrival(Job),
    Start(Job),
    Finish(Job),
}

impl System {
    pub fn new<T: AsRef<Path>>(config: T, source: &Source) -> Result<System> {
        let config = try!(Config::new(config));

        let platform = {
            let config = some!(config.branch("platform"), "a platform configuration is required");
            try!(Platform::new(&config))
        };
        let traffic = {
            let config = some!(config.branch("traffic"), "a traffic configuration is required");
            try!(Traffic::new(&config, source))
        };
        let workload = {
            let config = some!(config.branch("workload"), "a workload configuration is required");
            try!(Workload::new(&config, source))
        };

        Ok(System {
            platform: platform,
            traffic: traffic,
            workload: workload,
            queue: BinaryHeap::new(),
        })
    }

    fn update(&mut self) -> Result<()> {
        match (self.traffic.peek(), self.queue.peek()) {
            (Some(_), None) => try!(self.enqueue_job()),
            (Some(&arrival), Some(&Event { time, .. })) => if arrival < time {
                try!(self.enqueue_job());
            },
            _ => {},
        }
        Ok(())
    }

    fn enqueue_job(&mut self) -> Result<()> {
        let job = match (self.traffic.next(), self.workload.next()) {
            (Some(arrival), Some(pattern)) => Job::new(arrival, pattern),
            _ => raise!("failed to generate a new job"),
        };

        self.queue.push(time!(job.arrival, Event {
            kind: EventKind::Arrival(job.clone()),
        }));

        let (start, finish) = match self.platform.next(&job) {
            Some((start, finish)) => (start, finish),
            _ => raise!("failed to schedule {}", job),
        };

        self.queue.push(time!(start, Event {
            kind: EventKind::Start(job.clone()),
        }));
        self.queue.push(time!(finish, Event {
            kind: EventKind::Finish(job),
        }));

        Ok(())
    }
}

impl Iterator for System {
    type Item = Event;

    fn next(&mut self) -> Option<Self::Item> {
        if let Err(error) = self.update() {
            error!(target: "system", "Failed to update the state ({}).", error);
        }
        self.queue.pop()
    }
}

impl fmt::Display for Event {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "{:9.2} s: {:<30}", self.time, &self.kind)
    }
}

impl fmt::Display for EventKind {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        macro_rules! job(
            ($job:expr, $note:expr) => (
                write!(formatter, "{} {}", $job, $note)
            )
        );

        match self {
            &EventKind::Arrival(ref job) => job!(job, "arrival"),
            &EventKind::Start(ref job) => job!(job, "start"),
            &EventKind::Finish(ref job) => job!(job, "finish"),
        }
    }
}
<|endoftext|>


<commit_before>//! A load-balancing task pool.
//!
//! This differs in implementation from std::sync::TaskPool in that each job is
//! up for grabs by any of the child tasks in the pool.
//!
//! This should be upstreamed at some point.

use std::sync::{Arc, Mutex};

pub struct TaskPool {
    state: Arc<Mutex<State>>,
}

struct State { done: bool, jobs: Vec<proc():Send> }

impl TaskPool {
    pub fn new(tasks: uint) -> TaskPool {
        assert!(tasks > 0);

        let state = Arc::new(Mutex::new(State {
            done: false,
            jobs: Vec::new(),
        }));

        for _ in range(0, tasks) {
            let myjobs = state.clone();
            spawn(proc() worker(&*myjobs));
        }

        return TaskPool { state: state };

        fn worker(mystate: &Mutex<State>) {
            let mut state = mystate.lock();
            while !state.done {
                match state.jobs.pop() {
                    Some(job) => {
                        drop(state);
                        job();
                        state = mystate.lock();
                    }
                    None => state.cond.wait(),
                }
            }
        }
    }

    pub fn execute(&self, job: proc():Send) {
        let mut state = self.state.lock();
        state.jobs.push(job);
        state.cond.signal();
    }
}

impl Drop for TaskPool {
    fn drop(&mut self) {
        let mut state = self.state.lock();
        state.done = true;
        state.cond.broadcast();
        drop(state);
    }
}
<commit_msg>Power the TaskPool with a sync channel<commit_after>//! A load-balancing task pool.
//!
//! This differs in implementation from std::sync::TaskPool in that each job is
//! up for grabs by any of the child tasks in the pool.
//!
//! This should be upstreamed at some point.

use std::sync::{Arc, Mutex};

pub struct TaskPool {
    tx: SyncSender<proc():Send>,
}

impl TaskPool {
    pub fn new(tasks: uint) -> TaskPool {
        assert!(tasks > 0);
        let (tx, rx) = sync_channel(tasks);

        let state = Arc::new(Mutex::new(rx));

        for _ in range(0, tasks) {
            let state = state.clone();
            spawn(proc() worker(&*state));
        }

        return TaskPool { tx: tx };

        fn worker(rx: &Mutex<Receiver<proc():Send>>) {
            loop {
                let job = rx.lock().recv_opt();
                match job {
                    Ok(job) => job(),
                    Err(..) => break,
                }
            }
        }
    }

    pub fn execute(&self, job: proc():Send) {
        self.tx.send(job);
    }
}
<|endoftext|>


<commit_before>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::*;
use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = init_lsm_level_trees(neb_client);
        debug!("Initialized LSM-tree");
        Arc::new(LSMTree { trees, max_sizes })
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }

    pub fn check_and_merge(&self) {
        for i in 0..self.trees.len() - 1 {
            debug!("Checking tree merge {}", i);
            let lower = &*self.trees[i];
            let upper = &*self.trees[i + 1];
            if lower.count() > self.max_sizes[i] {
                lower.merge_to(upper);
            }
        }
    }

    fn sentinel(this: &Arc<Self>) {
        let this = this.clone();
        thread::Builder::new().name("LSM-Tree Sentinel".to_string()).spawn(move || {
            loop {
                this.check_and_merge();
                thread::sleep(Duration::from_millis(500));
            }
        });
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        if let Some(prev_key) = self.current().map(|k| k.to_owned()) {
            let prob = self.level_cursors
                .iter_mut()
                .filter(|c| c.current().is_some())
                .min_by(|a, b| a.current().unwrap().cmp(b.current().unwrap()))
                .map(|c| (c.next(), c))
                .map(|(has_next, c)| if !has_next || c.current().unwrap() <= &prev_key {
                    None
                } else {
                    Some(has_next)
                })
                .unwrap_or(Some(false));
            if let Some(res) = prob {
                res
            } else {
                self.next()
            }
        } else {
            false
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    fn dump_trees(lsm_tree: &LSMTree, name: &str) {
        for i in 0..lsm_tree.trees.len() {
            lsm_tree.trees[i].dump(&format!("{}_lsm_{}_dump.json", name, i));
        }
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5700");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 3 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(btree::page_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        debug!("Testing LSM-tree");
        thread::spawn(move || loop {
            thread::sleep(Duration::from_secs(10));
            let tree_len = tree_clone.len();
            debug!(
                "LSM-Tree now have {}/{} elements, total {:.2}%",
                tree_len,
                num,
                tree_len as f32 / num as f32 * 100.0
            );
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        if tree.len() > LEVEL_M_MAX_ELEMENTS_COUNT {
            debug!("Level trees sizes are {:?}, wait for continuous merge", tree.level_sizes());
            thread::sleep(Duration::from_secs(30));
            debug!("Level trees sizes are {:?}", tree.level_sizes());
        }
        debug!("Start validations");

        dump_trees(&*tree, "before_insertion");
        tree.check_and_merge();
        dump_trees(&*tree, "after_insertion");

        (0..num).collect::<Vec<_>>().iter().for_each(|i| {
            let mut rng = rand::rngs::OsRng::new().unwrap();
            let die_range = Uniform::new_inclusive(1, 15);
            let mut roll_die = rng.sample_iter(&die_range);
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
            if roll_die.next().unwrap() == 15 {
                for j in i..num {
                    let id = Id::new(0, j);
                    let key_slice = u64_to_slice(j);
                    let mut key = SmallVec::from_slice(&key_slice);
                    key_with_id(&mut key, &id);
                    assert_eq!(cursor.current(), Some(&key), "{}/{}", i, j);
                    assert_eq!(cursor.next(), j != num - 1, "{}/{}", i, j);
                }
            }
        });
    }
}
<commit_msg>two stage lsm tree test<commit_after>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::*;
use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = init_lsm_level_trees(neb_client);
        debug!("Initialized LSM-tree");
        Arc::new(LSMTree { trees, max_sizes })
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }

    pub fn check_and_merge(&self) {
        for i in 0..self.trees.len() - 1 {
            debug!("Checking tree merge {}", i);
            let lower = &*self.trees[i];
            let upper = &*self.trees[i + 1];
            if lower.count() > self.max_sizes[i] {
                lower.merge_to(upper);
            }
        }
    }

    pub fn start_sentinel(this: &Arc<Self>) {
        let this = this.clone();
        thread::Builder::new().name("LSM-Tree Sentinel".to_string()).spawn(move || {
            loop {
                this.check_and_merge();
                thread::sleep(Duration::from_millis(500));
            }
        });
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        if let Some(prev_key) = self.current().map(|k| k.to_owned()) {
            self.level_cursors
                .iter_mut()
                .filter(|c| c.current().is_some())
                .min_by(|a, b| a.current().unwrap().cmp(b.current().unwrap()))
                .map(|c| c.next());
            let dedupe_current = if let Some(current) = self.current() {
                if current <= &prev_key {
                    None
                } else {
                    Some(true)
                }
            } else {
                Some(false)
            };
            if let Some(has_next) = dedupe_current {
                has_next
            } else {
                self.next()
            }
        } else {
            false
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    fn dump_trees(lsm_tree: &LSMTree, name: &str) {
        for i in 0..lsm_tree.trees.len() {
            lsm_tree.trees[i].dump(&format!("{}_lsm_{}_dump.json", name, i));
        }
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5700");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 3 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(btree::page_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        debug!("Testing LSM-tree");
        thread::spawn(move || loop {
            thread::sleep(Duration::from_secs(10));
            let tree_len = tree_clone.len();
            debug!(
                "LSM-Tree now have {}/{} elements, total {:.2}%",
                tree_len,
                num,
                tree_len as f32 / num as f32 * 100.0
            );
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        dump_trees(&*tree, "stage_1_before_insertion");
        tree.check_and_merge();
        dump_trees(&*tree, "stage_1_after_insertion");

        debug!("Start point search validations");
        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
        });

        LSMTree::start_sentinel(&tree);

        (num..num * 2).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        dump_trees(&*tree, "stage_2_after_insertion");
        thread::sleep(Duration::new(30, 0));
        dump_trees(&*tree, "stage_2_waited_insertion");

        (0..num * 2).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
        });
    }
}
<|endoftext|>


<commit_before>use std::cell::RefCell;

use ecs::*;
use scene::Scene;
use super::EntitySet;

pub type AlarmCallback = Fn(&Scene, Entity);

struct Alarm {
    start_time: f32,
    remaining_time: f32,
    callback: Box<Fn(&Scene, Entity)>,
    entity: Entity,
    id: AlarmID,
    repeating: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AlarmID(usize);

pub struct AlarmManager {
    id_counter: usize,
    alarms: Vec<Alarm>,
    marked_for_destroy: RefCell<EntitySet>,
}

impl AlarmManager {
    pub fn new() -> AlarmManager {
        AlarmManager {
            id_counter: 0,
            alarms: Vec::new(),
            marked_for_destroy: RefCell::new(EntitySet::default()),
        }
    }

    pub fn assign<T>(&mut self, entity: Entity, duration: f32, callback: T) -> AlarmID
        where T: 'static + Fn(&Scene, Entity) {
        self.id_counter += 1;
        let id = AlarmID(self.id_counter);
        let alarm = Alarm {
            start_time: duration,
            remaining_time: duration,
            callback: Box::new(callback),
            entity: entity,
            id: id,
            repeating: false,
        };

        self.insert_alarm(alarm);

        id
    }

    pub fn assign_repeating<T>(&mut self, entity: Entity, duration: f32, callback: T) -> AlarmID
        where T: 'static + Fn(&Scene, Entity) {
        self.id_counter += 1;
        let id = AlarmID(self.id_counter);
        let alarm = Alarm {
            start_time: duration,
            remaining_time: duration,
            callback: Box::new(callback),
            entity: entity,
            id: id,
            repeating: true,
        };

        self.insert_alarm(alarm);

        id
    }

    pub fn cancel(&mut self, id: AlarmID) {
        self.alarms.retain(|alarm| alarm.id != id);
    }

    fn insert_alarm(&mut self, mut new_alarm: Alarm) {
        let mut insertion_index = None;
        new_alarm.remaining_time = new_alarm.start_time;
        for (index, alarm) in self.alarms.iter().enumerate() {
            if new_alarm.remaining_time < alarm.remaining_time {
                // There's less time than the current alarm, which means we should insert
                // the new alarm before the current one.
                insertion_index = Some(index);
            } else {
                // There's more time remaining than the current alarm, so update the remaining
                // time by subtracting the current alarm's time.
                new_alarm.remaining_time -= alarm.remaining_time;
            }
        }

        if let Some(index) = insertion_index {
            self.alarms.insert(index, new_alarm);
        } else {
            // Alarm doesn't come before any other alarms, push it on the back.
            self.alarms.push(new_alarm);
        }
    }
}

impl ComponentManager for AlarmManager {
    fn destroy_all(&self, entity: Entity) {
        self.marked_for_destroy.borrow_mut().insert(entity);
    }

    fn destroy_marked(&mut self) {
        let mut marked_for_destroy = self.marked_for_destroy.borrow_mut();
        for entity in marked_for_destroy.drain() {
            // Iterate through all alarms and remove any associated with the entity.
            loop {
                let (index, remaining_time) = match self.alarms.iter().enumerate().find(|&(_, alarm)| alarm.entity == entity) {
                    Some((index, alarm)) => {
                        (index, alarm.remaining_time)
                    }
                    None => break,
                };

                if index < self.alarms.len() - 1 {
                    // There's another alarm after the one wer're removing, so update it.
                    self.alarms[index + 1].remaining_time += remaining_time;
                }

                self.alarms.remove(index);
            }
        }
    }
}

impl Clone for AlarmManager {
    fn clone(&self) -> AlarmManager {
        println!("WARNING: Cloning the alarm manager is not supported yet, hotloading will cancel all pending alarms");

        AlarmManager {
            id_counter: self.id_counter,
            alarms: Vec::new(),
            marked_for_destroy: self.marked_for_destroy.clone(),
        }
    }
}

pub struct AlarmSystem;

impl System for AlarmSystem {
    fn update(&mut self, scene: &Scene, delta: f32) {
        let mut callbacks_to_trigger: Vec<Alarm> = Vec::new();

        // The first time we borrow the alarm manager we collect up all the alarms that need
        // to be triggered, but we need to end our borrow of the alarm manager before triggering
        // them because the callback might try to borrow the alarm manager too.
        {
            let mut alarm_manager = scene.get_manager_mut::<AlarmManager>();

            let mut remaining_delta = delta;
            while alarm_manager.alarms.len() > 0 {
                {
                    let alarm = &mut alarm_manager.alarms[0];
                    if remaining_delta > alarm.remaining_time {
                        // Alarm is done, invoke the callback and then remove it from the manager.
                        remaining_delta -= alarm.remaining_time;
                    } else {
                        // Not enough delta time to finish the alarm,
                        // update its remaining time and break.
                        alarm.remaining_time -= remaining_delta;
                        break;
                    }
                }

                // If the alarm shouldn't be removed the loop breaks before this point, so we're good
                // to remove the first alarm.
                let alarm = alarm_manager.alarms.remove(0);
                callbacks_to_trigger.push(alarm);
            }
        }

        // Do callbacks.
        for alarm in &callbacks_to_trigger {
            let entity = alarm.entity;
            (alarm.callback)(scene, entity);
        }

        // Once we've done all the callbacks then we can put the repeating alarms back into the
        // alarm manager.
        {
            let mut alarm_manager = scene.get_manager_mut::<AlarmManager>();
            for alarm in callbacks_to_trigger.drain(0..) {
                if alarm.repeating {
                    alarm_manager.insert_alarm(alarm);
                }
            }
        }
    }
}
<commit_msg>refactor(alarm): rename AlarmID to AlarmId<commit_after>use std::cell::RefCell;

use ecs::*;
use scene::Scene;
use super::EntitySet;

pub type AlarmCallback = Fn(&Scene, Entity);

struct Alarm {
    start_time: f32,
    remaining_time: f32,
    callback: Box<Fn(&Scene, Entity)>,
    entity: Entity,
    id: AlarmId,
    repeating: bool,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AlarmId(usize);

pub struct AlarmManager {
    id_counter: usize,
    alarms: Vec<Alarm>,
    marked_for_destroy: RefCell<EntitySet>,
    pending_insertions: RefCell<Vec<Alarm>>,
    pending_cancellations: RefCell<Vec<AlarmId>>,
}

impl AlarmManager {
    pub fn new() -> AlarmManager {
        AlarmManager {
            id_counter: 0,
            alarms: Vec::new(),
            marked_for_destroy: RefCell::new(EntitySet::default()),
        }
    }

    pub fn assign<T>(&mut self, entity: Entity, duration: f32, callback: T) -> AlarmId
        where T: 'static + Fn(&Scene, Entity) {
        self.id_counter += 1;
        let id = AlarmId(self.id_counter);
        let alarm = Alarm {
            start_time: duration,
            remaining_time: duration,
            callback: Box::new(callback),
            entity: entity,
            id: id,
            repeating: false,
        };

        self.insert_alarm(alarm);

        id
    }

    pub fn assign_repeating<T>(&mut self, entity: Entity, duration: f32, callback: T) -> AlarmId
        where T: 'static + Fn(&Scene, Entity) {
        self.id_counter += 1;
        let id = AlarmId(self.id_counter);
        let alarm = Alarm {
            start_time: duration,
            remaining_time: duration,
            callback: Box::new(callback),
            entity: entity,
            id: id,
            repeating: true,
        };

        self.insert_alarm(alarm);

        id
    }

    pub fn cancel(&mut self, id: AlarmId) {
        self.alarms.retain(|alarm| alarm.id != id);
    }

    fn insert_alarm(&mut self, mut new_alarm: Alarm) {
        let mut insertion_index = None;
        new_alarm.remaining_time = new_alarm.start_time;
        for (index, alarm) in self.alarms.iter().enumerate() {
            if new_alarm.remaining_time < alarm.remaining_time {
                // There's less time than the current alarm, which means we should insert
                // the new alarm before the current one.
                insertion_index = Some(index);
            } else {
                // There's more time remaining than the current alarm, so update the remaining
                // time by subtracting the current alarm's time.
                new_alarm.remaining_time -= alarm.remaining_time;
            }
        }

        if let Some(index) = insertion_index {
            self.alarms.insert(index, new_alarm);
        } else {
            // Alarm doesn't come before any other alarms, push it on the back.
            self.alarms.push(new_alarm);
        }
    }
}

impl ComponentManager for AlarmManager {
    fn destroy_all(&self, entity: Entity) {
        self.marked_for_destroy.borrow_mut().insert(entity);
    }

    fn destroy_marked(&mut self) {
        let mut marked_for_destroy = self.marked_for_destroy.borrow_mut();
        for entity in marked_for_destroy.drain() {
            // Iterate through all alarms and remove any associated with the entity.
            loop {
                let (index, remaining_time) = match self.alarms.iter().enumerate().find(|&(_, alarm)| alarm.entity == entity) {
                    Some((index, alarm)) => {
                        (index, alarm.remaining_time)
                    }
                    None => break,
                };

                if index < self.alarms.len() - 1 {
                    // There's another alarm after the one wer're removing, so update it.
                    self.alarms[index + 1].remaining_time += remaining_time;
                }

                self.alarms.remove(index);
            }
        }
    }
}

impl Clone for AlarmManager {
    fn clone(&self) -> AlarmManager {
        println!("WARNING: Cloning the alarm manager is not supported yet, hotloading will cancel all pending alarms");

        AlarmManager {
            id_counter: self.id_counter,
            alarms: Vec::new(),
            marked_for_destroy: self.marked_for_destroy.clone(),
        }
    }
}

pub struct AlarmSystem;

impl System for AlarmSystem {
    fn update(&mut self, scene: &Scene, delta: f32) {
        let mut callbacks_to_trigger: Vec<Alarm> = Vec::new();

        // The first time we borrow the alarm manager we collect up all the alarms that need
        // to be triggered, but we need to end our borrow of the alarm manager before triggering
        // them because the callback might try to borrow the alarm manager too.
        {
            let mut alarm_manager = scene.get_manager_mut::<AlarmManager>();

            let mut remaining_delta = delta;
            while alarm_manager.alarms.len() > 0 {
                {
                    let alarm = &mut alarm_manager.alarms[0];
                    if remaining_delta > alarm.remaining_time {
                        // Alarm is done, invoke the callback and then remove it from the manager.
                        remaining_delta -= alarm.remaining_time;
                    } else {
                        // Not enough delta time to finish the alarm,
                        // update its remaining time and break.
                        alarm.remaining_time -= remaining_delta;
                        break;
                    }
                }

                // If the alarm shouldn't be removed the loop breaks before this point, so we're good
                // to remove the first alarm.
                let alarm = alarm_manager.alarms.remove(0);
                callbacks_to_trigger.push(alarm);
            }
        }

        // Do callbacks.
        for alarm in &callbacks_to_trigger {
            let entity = alarm.entity;
            (alarm.callback)(scene, entity);
        }

        // Once we've done all the callbacks then we can put the repeating alarms back into the
        // alarm manager.
        {
            let mut alarm_manager = scene.get_manager_mut::<AlarmManager>();
            for alarm in callbacks_to_trigger.drain(0..) {
                if alarm.repeating {
                    alarm_manager.insert_alarm(alarm);
                }
            }
        }
    }
}
<|endoftext|>