<commit_before>//! Execute benchmarks in a sysroot.

use std::env;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use std::str;
use std::f64;
use std::fs::{self, File};
use std::collections::{HashMap, HashSet};
use std::cmp;

use tempdir::TempDir;

use collector::{Benchmark as CollectedBenchmark, BenchmarkState, Patch, Run, Stat};

use failure::{err_msg, Error, ResultExt};
use rust_sysroot::sysroot::Sysroot;
use serde_json;

use Mode;

fn command_output(cmd: &mut Command) -> Result<process::Output, Error> {
    trace!("running: {:?}", cmd);
    let output = cmd.output()?;
    if !output.status.success() {
        bail!(
            "expected success, got {}\n\nstderr={}\n\n stdout={}",
            output.status,
            String::from_utf8_lossy(&output.stderr),
            String::from_utf8_lossy(&output.stdout)
        );
    }
    Ok(output)
}

fn touch_all(path: &Path) -> Result<(), Error> {
    let mut cmd = Command::new("bash");
    cmd.current_dir(path)
        .args(&["-c", "find . -name '*.rs' | xargs touch"]);
    command_output(&mut cmd)?;
    Ok(())
}

fn default_runs() -> usize {
    3
}

fn default_true() -> bool {
    true
}

#[derive(Debug, Clone, Deserialize)]
struct BenchmarkConfig {
    cargo_opts: Option<String>,
    cargo_rustc_opts: Option<String>,
    cargo_toml: Option<String>,
    #[serde(default)]
    disabled: bool,
    #[serde(default = "default_runs")]
    runs: usize,
    #[serde(default = "default_true")]
    run_optimized: bool,
    #[serde(default = "default_true")]
    run_debug: bool,
    #[serde(default = "default_true")]
    nll: bool,
}

impl Default for BenchmarkConfig {
    fn default() -> BenchmarkConfig {
        BenchmarkConfig {
            cargo_opts: None,
            cargo_rustc_opts: None,
            cargo_toml: None,
            disabled: false,
            runs: default_runs(),
            run_optimized: true,
            run_debug: true,
            nll: true,
        }
    }
}

pub struct Benchmark {
    pub name: String,
    pub path: PathBuf,
    patches: Vec<Patch>,
    config: BenchmarkConfig,
}

struct CargoProcess<'sysroot> {
    sysroot: &'sysroot Sysroot,
    incremental: Incremental,
    nll: bool,
    options: Options,
    perf: bool,
    manifest_path: String,
    cargo_args: Vec<String>,
    rustc_args: Vec<String>,
}

#[derive(Copy, Clone, Debug)]
struct Incremental(bool);

impl<'sysroot> CargoProcess<'sysroot> {
    fn base_command(&self, cwd: &Path, subcommand: &str) -> Command {
        let mut cmd = self.sysroot.command("cargo");
        cmd
            .env("SHELL", env::var_os("SHELL").unwrap_or_default())
            .env("RUSTC", &*FAKE_RUSTC)
            .env("RUSTC_REAL", &self.sysroot.rustc)
            .env(
                "CARGO_INCREMENTAL",
                &format!("{}", self.incremental.0 as usize),
            )
            .env("USE_PERF", &format!("{}", self.perf as usize))
            .current_dir(cwd)
            .arg(subcommand)
            .arg("--manifest-path")
            .arg(&self.manifest_path);
        cmd
    }

    fn nll(mut self, nll: bool) -> Self {
        self.nll = nll;
        self
    }

    fn perf(mut self, perf: bool) -> Self {
        self.perf = perf;
        self
    }

    fn get_pkgid(&self, cwd: &Path) -> String {
        let mut pkgid_cmd = self.base_command(cwd, "pkgid");
        let out = command_output(&mut pkgid_cmd)
            .unwrap_or_else(|e| {
                panic!("failed to obtain pkgid in {:?}: {:?}", cwd, e);
            })
            .stdout;
        let package_id = str::from_utf8(&out).unwrap();
        package_id.trim().to_string()
    }

    fn run(self, cwd: &Path, mode: CargoMode) -> Result<Vec<Stat>, Error> {
        let subcommand = match mode {
            CargoMode::Build => "rustc",
            CargoMode::Clean => "clean",
        };
        let mut cmd = self.base_command(cwd, subcommand);
        cmd.arg("-p").arg(self.get_pkgid(cwd));
        if mode == CargoMode::Build && self.options.check {
            cmd.arg("--profile").arg("check");
        }
        if self.options.release {
            cmd.arg("--release");
        }
        if mode == CargoMode::Build {
            cmd.args(&self.cargo_args);
            cmd.arg("--");
            if self.nll {
                cmd.arg("-Znll");
            }
            cmd.arg("-Ztime-passes");
            cmd.args(&self.rustc_args);
        }

        debug!("running cargo: {:?}", cmd);

        match mode {
            CargoMode::Build => loop {
                touch_all(&cwd)?;
                let output = command_output(&mut cmd)?;
                match process_output(output) {
                    Ok(stats) => return Ok(stats),
                    Err(DeserializeStatError::NoOutput(output)) => {
                        warn!(
                            "failed to deserialize stats, retrying; output: {:?}",
                            output
                        );
                    }
                    Err(e @ DeserializeStatError::ParseError { .. }) => {
                        panic!("process_output failed: {:?}", e);
                    }
                }
            },
            CargoMode::Clean => {
                let _output = command_output(&mut cmd)?;
                return Ok(Vec::new());
            }
        }
    }
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
enum CargoMode {
    Build,
    Clean,
}

#[derive(Debug, PartialEq, Clone, Copy)]
struct Options {
    release: bool,
    check: bool,
}

lazy_static! {
    static ref FAKE_RUSTC: PathBuf = {
        let mut fake_rustc = env::current_exe().unwrap();
        fake_rustc.pop();
        fake_rustc.push("rustc-fake");
        fake_rustc
    };
}

impl Benchmark {
    pub fn new(name: String, path: PathBuf) -> Result<Self, Error> {
        let mut patches = vec![];
        for entry in fs::read_dir(&path)? {
            let entry = entry?;
            let path = entry.path();
            if let Some(ext) = path.extension() {
                if ext == "patch" {
                    patches.push(path.clone());
                }
            }
        }

        patches.sort();

        let patches = patches.into_iter().map(|p| Patch::new(p)).collect();

        let config_path = path.join("perf-config.json");
        let config: BenchmarkConfig = if config_path.exists() {
            serde_json::from_reader(File::open(&config_path)
                .with_context(|_| format!("failed to open {:?}", config_path))?)
                .with_context(|_| {
                format!("failed to parse {:?}", config_path)
            })?
        } else {
            BenchmarkConfig::default()
        };

        Ok(Benchmark {
            name,
            path,
            patches,
            config,
        })
    }

    fn make_temp_dir(&self, base: &Path) -> Result<TempDir, Error> {
        let tmp_dir = TempDir::new(&format!("rustc-benchmark-{}", self.name))?;
        let mut cmd = Command::new("cp");
        cmd.arg("-r")
            .arg("-T")
            .arg("--")
            .arg(base)
            .arg(tmp_dir.path());
        command_output(&mut cmd).with_context(|_| format!("copying {} to tmp dir", self.name))?;
        Ok(tmp_dir)
    }

    fn mk_cargo_process<'a>(
        &self,
        sysroot: &'a Sysroot,
        incremental: Incremental,
        options: Options,
    ) -> CargoProcess<'a> {
        CargoProcess {
            sysroot: sysroot,
            incremental: incremental,
            options: options,
            perf: true,
            nll: false,
            manifest_path: self.config
                .cargo_toml
                .clone()
                .unwrap_or_else(|| String::from("Cargo.toml")),
            cargo_args: self.config
                .cargo_opts
                .clone()
                .unwrap_or_default()
                .split_whitespace()
                .map(String::from)
                .collect(),
            rustc_args: self.config
                .cargo_rustc_opts
                .clone()
                .unwrap_or_default()
                .split_whitespace()
                .map(String::from)
                .collect(),
        }
    }

    /// Run a specific benchmark on a specific commit
    pub fn run(
        &self,
        sysroot: &Sysroot,
        iterations: usize,
        mode: Mode,
    ) -> Result<CollectedBenchmark, Error> {
        let iterations = cmp::min(iterations, self.config.runs);
        if self.config.disabled {
            eprintln!("skipping {}: disabled", self.name);
            bail!("disabled benchmark");
        }

        let has_perf = Command::new("perf").output().is_ok();
        assert!(has_perf);

        let mut ret = CollectedBenchmark {
            name: self.name.clone(),
            runs: Vec::new(),
        };

        let mut opts = Vec::with_capacity(3);
        match mode {
            Mode::Normal => {
                if self.config.run_debug {
                    opts.push(Options {
                        check: true,
                        release: false,
                    });
                    opts.push(Options {
                        check: false,
                        release: false,
                    });
                }
                if self.config.run_optimized {
                    opts.push(Options {
                        check: false,
                        release: true,
                    });
                }
            }
            Mode::Test => {
                opts.push(Options {
                    check: true,
                    release: false,
                });
            }
        }

        for opt in opts {
            let mut clean_stats = Vec::new();
            let mut nll_stats = Vec::new();
            let mut incr_stats = Vec::new();
            let mut incr_clean_stats = Vec::new();
            let mut incr_patched_stats: Vec<(Patch, Vec<Vec<Stat>>)> = Vec::new();

            info!(
                "Benchmarking {} with release: {}, check: {}, iterations: {}",
                self.name, opt.release, opt.check, iterations
            );

            let base_build = self.make_temp_dir(&self.path)?;
            let clean = self.mk_cargo_process(sysroot, Incremental(false), opt)
                .run(base_build.path(), CargoMode::Build)?;
            clean_stats.push(clean);
            self.mk_cargo_process(sysroot, Incremental(false), opt)
                .perf(false)
                .run(base_build.path(), CargoMode::Clean)?;

            for i in 0..iterations {
                debug!("Benchmark iteration {}/{}", i + 1, iterations);
                let tmp_dir = self.make_temp_dir(base_build.path())?;

                let clean = self.mk_cargo_process(sysroot, Incremental(false), opt)
                    .run(tmp_dir.path(), CargoMode::Build)?;
                if self.config.nll {
                    let nll = self.mk_cargo_process(sysroot, Incremental(false), opt)
                        .nll(true)
                        .run(base_build.path(), CargoMode::Build)?;
                    nll_stats.push(nll);
                }
                let incr = self.mk_cargo_process(sysroot, Incremental(true), opt)
                    .run(tmp_dir.path(), CargoMode::Build)?;
                let incr_clean = self.mk_cargo_process(sysroot, Incremental(true), opt)
                    .run(tmp_dir.path(), CargoMode::Build)?;

                clean_stats.push(clean);
                incr_stats.push(incr);
                incr_clean_stats.push(incr_clean);

                for patch in &self.patches {
                    debug!("applying patch {}", patch.name);
                    patch.apply(tmp_dir.path()).map_err(|s| err_msg(s))?;
                    let out = self.mk_cargo_process(sysroot, Incremental(true), opt)
                        .run(tmp_dir.path(), CargoMode::Build)?;
                    if let Some(mut entry) = incr_patched_stats.iter_mut().find(|s| &s.0 == patch) {
                        entry.1.push(out);
                        continue;
                    }

                    incr_patched_stats.push((patch.clone(), vec![out]));
                }
            }

            ret.runs
                .push(process_stats(opt, BenchmarkState::Clean, clean_stats));
            if self.config.nll {
                ret.runs
                    .push(process_stats(opt, BenchmarkState::Nll, nll_stats));
            }
            ret.runs.push(process_stats(
                opt,
                BenchmarkState::IncrementalStart,
                incr_stats,
            ));
            ret.runs.push(process_stats(
                opt,
                BenchmarkState::IncrementalClean,
                incr_clean_stats,
            ));

            for (patch, results) in incr_patched_stats {
                ret.runs.push(process_stats(
                    opt,
                    BenchmarkState::IncrementalPatched(patch),
                    results,
                ));
            }
        }

        Ok(ret)
    }
}

#[derive(Fail, PartialEq, Eq, Debug)]
enum DeserializeStatError {
    #[fail(display = "could not deserialize empty output to stats, output: {:?}", _0)]
    NoOutput(process::Output),
    #[fail(display = "could not parse `{}` as a float", _0)]
    ParseError(String, #[fail(cause)] ::std::num::ParseFloatError),
}

fn process_output(output: process::Output) -> Result<Vec<Stat>, DeserializeStatError> {
    let stdout = String::from_utf8(output.stdout.clone()).expect("utf8 output");
    let mut stats = Vec::new();

    for line in stdout.lines() {
        // github.com/torvalds/linux/blob/bc78d646e708/tools/perf/Documentation/perf-stat.txt#L281
        macro_rules! get {
            ($e: expr) => {
                match $e {
                    Some(s) => s,
                    None => {
                        warn!("unhandled line: {}", line);
                        continue;
                    }
                }
            };
        }
        let mut parts = line.split(';').map(|s| s.trim());
        let cnt = get!(parts.next());
        let _unit = get!(parts.next());
        let name = get!(parts.next());
        let _time = get!(parts.next());
        let pct = get!(parts.next());
        if cnt == "<not supported>" {
            continue;
        }
        if !pct.starts_with("100.") {
            panic!(
                "measurement of `{}` only active for {}% of the time",
                name, pct
            );
        }
        stats.push(Stat {
            name: name.to_string(),
            cnt: cnt.parse()
                .map_err(|e| DeserializeStatError::ParseError(cnt.to_string(), e))?,
        });
    }

    if stats.is_empty() {
        return Err(DeserializeStatError::NoOutput(output));
    }

    Ok(stats)
}

fn process_stats(options: Options, state: BenchmarkState, runs: Vec<Vec<Stat>>) -> Run {
    let mut stats: HashMap<String, Vec<f64>> = HashMap::new();
    for run in runs.clone() {
        for stat in run {
            stats
                .entry(stat.name.clone())
                .or_insert_with(|| Vec::new())
                .push(stat.cnt);
        }
    }
    // all stats should be present in all runs
    let map = stats.values().map(|v| v.len()).collect::<HashSet<_>>();
    if map.len() != 1 {
        eprintln!("options: {:?}", options);
        eprintln!("state: {:?}", state);
        eprintln!("lengths: {:?}", map);
        eprintln!("runs: {:?}", runs);
        eprintln!("stats: {:?}", stats);
        panic!("expected all stats to be present in all runs");
    }
    let stats = stats
        .into_iter()
        .map(|(stat, counts)| Stat {
            name: stat,
            cnt: counts
                .into_iter()
                .fold(f64::INFINITY, |acc, v| f64::min(acc, v)),
        })
        .collect();

    Run {
        stats,
        check: options.check,
        release: options.release,
        state: state,
    }
}
<commit_msg>Split CargoProcess::run in two.<commit_after>//! Execute benchmarks in a sysroot.

use std::env;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use std::str;
use std::f64;
use std::fs::{self, File};
use std::collections::{HashMap, HashSet};
use std::cmp;

use tempdir::TempDir;

use collector::{Benchmark as CollectedBenchmark, BenchmarkState, Patch, Run, Stat};

use failure::{err_msg, Error, ResultExt};
use rust_sysroot::sysroot::Sysroot;
use serde_json;

use Mode;

fn command_output(cmd: &mut Command) -> Result<process::Output, Error> {
    trace!("running: {:?}", cmd);
    let output = cmd.output()?;
    if !output.status.success() {
        bail!(
            "expected success, got {}\n\nstderr={}\n\n stdout={}",
            output.status,
            String::from_utf8_lossy(&output.stderr),
            String::from_utf8_lossy(&output.stdout)
        );
    }
    Ok(output)
}

fn touch_all(path: &Path) -> Result<(), Error> {
    let mut cmd = Command::new("bash");
    cmd.current_dir(path)
        .args(&["-c", "find . -name '*.rs' | xargs touch"]);
    command_output(&mut cmd)?;
    Ok(())
}

fn default_runs() -> usize {
    3
}

fn default_true() -> bool {
    true
}

#[derive(Debug, Clone, Deserialize)]
struct BenchmarkConfig {
    cargo_opts: Option<String>,
    cargo_rustc_opts: Option<String>,
    cargo_toml: Option<String>,
    #[serde(default)]
    disabled: bool,
    #[serde(default = "default_runs")]
    runs: usize,
    #[serde(default = "default_true")]
    run_optimized: bool,
    #[serde(default = "default_true")]
    run_debug: bool,
    #[serde(default = "default_true")]
    nll: bool,
}

impl Default for BenchmarkConfig {
    fn default() -> BenchmarkConfig {
        BenchmarkConfig {
            cargo_opts: None,
            cargo_rustc_opts: None,
            cargo_toml: None,
            disabled: false,
            runs: default_runs(),
            run_optimized: true,
            run_debug: true,
            nll: true,
        }
    }
}

pub struct Benchmark {
    pub name: String,
    pub path: PathBuf,
    patches: Vec<Patch>,
    config: BenchmarkConfig,
}

struct CargoProcess<'sysroot> {
    sysroot: &'sysroot Sysroot,
    incremental: Incremental,
    nll: bool,
    options: Options,
    perf: bool,
    manifest_path: String,
    cargo_args: Vec<String>,
    rustc_args: Vec<String>,
}

#[derive(Copy, Clone, Debug)]
struct Incremental(bool);

impl<'sysroot> CargoProcess<'sysroot> {
    fn base_command(&self, cwd: &Path, subcommand: &str) -> Command {
        let mut cmd = self.sysroot.command("cargo");
        cmd
            .env("SHELL", env::var_os("SHELL").unwrap_or_default())
            .env("RUSTC", &*FAKE_RUSTC)
            .env("RUSTC_REAL", &self.sysroot.rustc)
            .env(
                "CARGO_INCREMENTAL",
                &format!("{}", self.incremental.0 as usize),
            )
            .env("USE_PERF", &format!("{}", self.perf as usize))
            .current_dir(cwd)
            .arg(subcommand)
            .arg("--manifest-path")
            .arg(&self.manifest_path);
        cmd
    }

    fn nll(mut self, nll: bool) -> Self {
        self.nll = nll;
        self
    }

    fn perf(mut self, perf: bool) -> Self {
        self.perf = perf;
        self
    }

    fn get_pkgid(&self, cwd: &Path) -> String {
        let mut pkgid_cmd = self.base_command(cwd, "pkgid");
        let out = command_output(&mut pkgid_cmd)
            .unwrap_or_else(|e| {
                panic!("failed to obtain pkgid in {:?}: {:?}", cwd, e);
            })
            .stdout;
        let package_id = str::from_utf8(&out).unwrap();
        package_id.trim().to_string()
    }

    fn run_base_command(&self, cwd: &Path, subcommand: &str) -> Command {
        let mut cmd = self.base_command(cwd, subcommand);
        cmd.arg("-p").arg(self.get_pkgid(cwd));
        if self.options.release {
            cmd.arg("--release");
        }
        cmd
    }

    fn run_rustc(self, cwd: &Path) -> Result<Vec<Stat>, Error> {
        let mut cmd = self.run_base_command(cwd, "rustc");
        if self.options.check {
            cmd.arg("--profile").arg("check");
        }
        cmd.args(&self.cargo_args);
        cmd.arg("--");
        if self.nll {
            cmd.arg("-Znll");
        }
        cmd.arg("-Ztime-passes");
        cmd.args(&self.rustc_args);

        debug!("run_rustc: {:?}", cmd);

        loop {
            touch_all(&cwd)?;
            let output = command_output(&mut cmd)?;
            match process_output(output) {
                Ok(stats) => return Ok(stats),
                Err(DeserializeStatError::NoOutput(output)) => {
                    warn!(
                        "failed to deserialize stats, retrying; output: {:?}",
                        output
                    );
                }
                Err(e @ DeserializeStatError::ParseError { .. }) => {
                    panic!("process_output failed: {:?}", e);
                }
            }
        }
    }

    fn run_clean(self, cwd: &Path) -> Result<(), Error> {
        let mut cmd = self.run_base_command(cwd, "clean");

        debug!("run_clean: {:?}", cmd);

        let _output = command_output(&mut cmd)?;
        return Ok(());
    }
}

#[derive(Debug, PartialEq, Clone, Copy)]
struct Options {
    release: bool,
    check: bool,
}

lazy_static! {
    static ref FAKE_RUSTC: PathBuf = {
        let mut fake_rustc = env::current_exe().unwrap();
        fake_rustc.pop();
        fake_rustc.push("rustc-fake");
        fake_rustc
    };
}

impl Benchmark {
    pub fn new(name: String, path: PathBuf) -> Result<Self, Error> {
        let mut patches = vec![];
        for entry in fs::read_dir(&path)? {
            let entry = entry?;
            let path = entry.path();
            if let Some(ext) = path.extension() {
                if ext == "patch" {
                    patches.push(path.clone());
                }
            }
        }

        patches.sort();

        let patches = patches.into_iter().map(|p| Patch::new(p)).collect();

        let config_path = path.join("perf-config.json");
        let config: BenchmarkConfig = if config_path.exists() {
            serde_json::from_reader(File::open(&config_path)
                .with_context(|_| format!("failed to open {:?}", config_path))?)
                .with_context(|_| {
                format!("failed to parse {:?}", config_path)
            })?
        } else {
            BenchmarkConfig::default()
        };

        Ok(Benchmark {
            name,
            path,
            patches,
            config,
        })
    }

    fn make_temp_dir(&self, base: &Path) -> Result<TempDir, Error> {
        let tmp_dir = TempDir::new(&format!("rustc-benchmark-{}", self.name))?;
        let mut cmd = Command::new("cp");
        cmd.arg("-r")
            .arg("-T")
            .arg("--")
            .arg(base)
            .arg(tmp_dir.path());
        command_output(&mut cmd).with_context(|_| format!("copying {} to tmp dir", self.name))?;
        Ok(tmp_dir)
    }

    fn mk_cargo_process<'a>(
        &self,
        sysroot: &'a Sysroot,
        incremental: Incremental,
        options: Options,
    ) -> CargoProcess<'a> {
        CargoProcess {
            sysroot: sysroot,
            incremental: incremental,
            options: options,
            perf: true,
            nll: false,
            manifest_path: self.config
                .cargo_toml
                .clone()
                .unwrap_or_else(|| String::from("Cargo.toml")),
            cargo_args: self.config
                .cargo_opts
                .clone()
                .unwrap_or_default()
                .split_whitespace()
                .map(String::from)
                .collect(),
            rustc_args: self.config
                .cargo_rustc_opts
                .clone()
                .unwrap_or_default()
                .split_whitespace()
                .map(String::from)
                .collect(),
        }
    }

    /// Run a specific benchmark on a specific commit
    pub fn run(
        &self,
        sysroot: &Sysroot,
        iterations: usize,
        mode: Mode,
    ) -> Result<CollectedBenchmark, Error> {
        let iterations = cmp::min(iterations, self.config.runs);
        if self.config.disabled {
            eprintln!("skipping {}: disabled", self.name);
            bail!("disabled benchmark");
        }

        let has_perf = Command::new("perf").output().is_ok();
        assert!(has_perf);

        let mut ret = CollectedBenchmark {
            name: self.name.clone(),
            runs: Vec::new(),
        };

        let mut opts = Vec::with_capacity(3);
        match mode {
            Mode::Normal => {
                if self.config.run_debug {
                    opts.push(Options {
                        check: true,
                        release: false,
                    });
                    opts.push(Options {
                        check: false,
                        release: false,
                    });
                }
                if self.config.run_optimized {
                    opts.push(Options {
                        check: false,
                        release: true,
                    });
                }
            }
            Mode::Test => {
                opts.push(Options {
                    check: true,
                    release: false,
                });
            }
        }

        for opt in opts {
            let mut clean_stats = Vec::new();
            let mut nll_stats = Vec::new();
            let mut incr_stats = Vec::new();
            let mut incr_clean_stats = Vec::new();
            let mut incr_patched_stats: Vec<(Patch, Vec<Vec<Stat>>)> = Vec::new();

            info!(
                "Benchmarking {} with release: {}, check: {}, iterations: {}",
                self.name, opt.release, opt.check, iterations
            );

            let base_build = self.make_temp_dir(&self.path)?;
            let clean = self.mk_cargo_process(sysroot, Incremental(false), opt)
                .run_rustc(base_build.path())?;
            clean_stats.push(clean);
            self.mk_cargo_process(sysroot, Incremental(false), opt)
                .perf(false)
                .run_clean(base_build.path())?;

            for i in 0..iterations {
                debug!("Benchmark iteration {}/{}", i + 1, iterations);
                let tmp_dir = self.make_temp_dir(base_build.path())?;

                let clean = self.mk_cargo_process(sysroot, Incremental(false), opt)
                    .run_rustc(tmp_dir.path())?;
                if self.config.nll {
                    let nll = self.mk_cargo_process(sysroot, Incremental(false), opt)
                        .nll(true)
                        .run_rustc(base_build.path())?;
                    nll_stats.push(nll);
                }
                let incr = self.mk_cargo_process(sysroot, Incremental(true), opt)
                    .run_rustc(tmp_dir.path())?;
                let incr_clean = self.mk_cargo_process(sysroot, Incremental(true), opt)
                    .run_rustc(tmp_dir.path())?;

                clean_stats.push(clean);
                incr_stats.push(incr);
                incr_clean_stats.push(incr_clean);

                for patch in &self.patches {
                    debug!("applying patch {}", patch.name);
                    patch.apply(tmp_dir.path()).map_err(|s| err_msg(s))?;
                    let out = self.mk_cargo_process(sysroot, Incremental(true), opt)
                        .run_rustc(tmp_dir.path())?;
                    if let Some(mut entry) = incr_patched_stats.iter_mut().find(|s| &s.0 == patch) {
                        entry.1.push(out);
                        continue;
                    }

                    incr_patched_stats.push((patch.clone(), vec![out]));
                }
            }

            ret.runs
                .push(process_stats(opt, BenchmarkState::Clean, clean_stats));
            if self.config.nll {
                ret.runs
                    .push(process_stats(opt, BenchmarkState::Nll, nll_stats));
            }
            ret.runs.push(process_stats(
                opt,
                BenchmarkState::IncrementalStart,
                incr_stats,
            ));
            ret.runs.push(process_stats(
                opt,
                BenchmarkState::IncrementalClean,
                incr_clean_stats,
            ));

            for (patch, results) in incr_patched_stats {
                ret.runs.push(process_stats(
                    opt,
                    BenchmarkState::IncrementalPatched(patch),
                    results,
                ));
            }
        }

        Ok(ret)
    }
}

#[derive(Fail, PartialEq, Eq, Debug)]
enum DeserializeStatError {
    #[fail(display = "could not deserialize empty output to stats, output: {:?}", _0)]
    NoOutput(process::Output),
    #[fail(display = "could not parse `{}` as a float", _0)]
    ParseError(String, #[fail(cause)] ::std::num::ParseFloatError),
}

fn process_output(output: process::Output) -> Result<Vec<Stat>, DeserializeStatError> {
    let stdout = String::from_utf8(output.stdout.clone()).expect("utf8 output");
    let mut stats = Vec::new();

    for line in stdout.lines() {
        // github.com/torvalds/linux/blob/bc78d646e708/tools/perf/Documentation/perf-stat.txt#L281
        macro_rules! get {
            ($e: expr) => {
                match $e {
                    Some(s) => s,
                    None => {
                        warn!("unhandled line: {}", line);
                        continue;
                    }
                }
            };
        }
        let mut parts = line.split(';').map(|s| s.trim());
        let cnt = get!(parts.next());
        let _unit = get!(parts.next());
        let name = get!(parts.next());
        let _time = get!(parts.next());
        let pct = get!(parts.next());
        if cnt == "<not supported>" {
            continue;
        }
        if !pct.starts_with("100.") {
            panic!(
                "measurement of `{}` only active for {}% of the time",
                name, pct
            );
        }
        stats.push(Stat {
            name: name.to_string(),
            cnt: cnt.parse()
                .map_err(|e| DeserializeStatError::ParseError(cnt.to_string(), e))?,
        });
    }

    if stats.is_empty() {
        return Err(DeserializeStatError::NoOutput(output));
    }

    Ok(stats)
}

fn process_stats(options: Options, state: BenchmarkState, runs: Vec<Vec<Stat>>) -> Run {
    let mut stats: HashMap<String, Vec<f64>> = HashMap::new();
    for run in runs.clone() {
        for stat in run {
            stats
                .entry(stat.name.clone())
                .or_insert_with(|| Vec::new())
                .push(stat.cnt);
        }
    }
    // all stats should be present in all runs
    let map = stats.values().map(|v| v.len()).collect::<HashSet<_>>();
    if map.len() != 1 {
        eprintln!("options: {:?}", options);
        eprintln!("state: {:?}", state);
        eprintln!("lengths: {:?}", map);
        eprintln!("runs: {:?}", runs);
        eprintln!("stats: {:?}", stats);
        panic!("expected all stats to be present in all runs");
    }
    let stats = stats
        .into_iter()
        .map(|(stat, counts)| Stat {
            name: stat,
            cnt: counts
                .into_iter()
                .fold(f64::INFINITY, |acc, v| f64::min(acc, v)),
        })
        .collect();

    Run {
        stats,
        check: options.check,
        release: options.release,
        state: state,
    }
}
<|endoftext|>


<commit_before>use utils::crypto::chacha20poly1305_ietf::ChaCha20Poly1305IETF;
use errors::common::CommonError;
use errors::wallet::WalletQueryError;

use super::wallet::Keys;
use super::language::{Operator,TargetValue,TagName};

// Performs encryption of WQL query
// WQL query is provided as top-level Operator
// Recursively transforms operators using encrypt_operator function
pub(super) fn encrypt_query(operator: Operator, keys: &Keys) -> Result<Operator, WalletQueryError> {
    let encrypted_op = operator.transform_result(&|op: Operator| -> Result<Operator, CommonError> {encrypt_operator(op, keys)})?;
    Ok(encrypted_op)
}

fn encrypt_operator(op: Operator, keys: &Keys) -> Result<Operator, CommonError> {
    match op {
        Operator::Eq(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Eq(encrypted_name, encrypted_value))
        },
        Operator::Neq(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Neq(encrypted_name, encrypted_value))
        },
       Operator::Gt(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Gt(encrypted_name, encrypted_value))
        },
        Operator::Gte(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Gte(encrypted_name, encrypted_value))
        },
        Operator::Lt(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Lt(encrypted_name, encrypted_value))
        },
        Operator::Lte(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Lte(encrypted_name, encrypted_value))
        },
        Operator::Like(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Like(encrypted_name, encrypted_value))
        },
        Operator::Regex(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Regex(encrypted_name, encrypted_value))
        },
        Operator::In(name, values) => {
            let name = match name {
                TagName::EncryptedTagName(ref name) => {
                    let encrypted_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
                    TagName::EncryptedTagName(encrypted_name)
                },
                TagName::PlainTagName(ref name) => {
                    let encrypted_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
                    TagName::PlainTagName(encrypted_name)
                }
            };
            let mut encrypted_values: Vec<TargetValue> = Vec::new();

            for value in values {
                encrypted_values.push(encrypt_name_value(&name, value, keys)?.1);
            }
            Ok(Operator::In(name, encrypted_values))
        },
        _ => Ok(op)
    }
}

// Encrypts a single tag name, tag value pair.
// If the tag name is EncryptedTagName enum variant, encrypts both the tag name and the tag value
// If the tag name is PlainTagName enum variant, encrypts only the tag name
fn encrypt_name_value(name: &TagName, value: TargetValue, keys: &Keys) -> Result<(TagName, TargetValue), CommonError> {
    match (name, value) {
        (&TagName::EncryptedTagName(ref name), TargetValue::Unencrypted(ref s)) => {
            let encrypted_tag_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
            let encrypted_tag_value = ChaCha20Poly1305IETF::encrypt_as_searchable(s.as_bytes(), &keys.tag_value_key, &keys.tags_hmac_key);
            Ok((TagName::EncryptedTagName(encrypted_tag_name), TargetValue::Encrypted(encrypted_tag_value)))
        },
        (&TagName::PlainTagName(ref name), TargetValue::Unencrypted(ref s)) => {
            let encrypted_tag_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
            Ok((TagName::PlainTagName(encrypted_tag_name), TargetValue::Unencrypted(s.clone())))
        },
        _ => Err(CommonError::InvalidState("Reached invalid combination of tag name and value while encrypting query".to_string()))
    }
}<commit_msg>Replace CommonError with WalletQueryError<commit_after>use utils::crypto::chacha20poly1305_ietf::ChaCha20Poly1305IETF;
use errors::wallet::WalletQueryError;

use super::wallet::Keys;
use super::language::{Operator,TargetValue,TagName};

// Performs encryption of WQL query
// WQL query is provided as top-level Operator
// Recursively transforms operators using encrypt_operator function
pub(super) fn encrypt_query(operator: Operator, keys: &Keys) -> Result<Operator, WalletQueryError> {
    operator.transform_result(&|op: Operator| -> Result<Operator, WalletQueryError> {encrypt_operator(op, keys)})
}

fn encrypt_operator(op: Operator, keys: &Keys) -> Result<Operator, WalletQueryError> {
    match op {
        Operator::Eq(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Eq(encrypted_name, encrypted_value))
        },
        Operator::Neq(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Neq(encrypted_name, encrypted_value))
        },
       Operator::Gt(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Gt(encrypted_name, encrypted_value))
        },
        Operator::Gte(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Gte(encrypted_name, encrypted_value))
        },
        Operator::Lt(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Lt(encrypted_name, encrypted_value))
        },
        Operator::Lte(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Lte(encrypted_name, encrypted_value))
        },
        Operator::Like(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Like(encrypted_name, encrypted_value))
        },
        Operator::Regex(name, value) => {
            let (encrypted_name, encrypted_value) = encrypt_name_value(&name, value, keys)?;
            Ok(Operator::Regex(encrypted_name, encrypted_value))
        },
        Operator::In(name, values) => {
            let name = match name {
                TagName::EncryptedTagName(ref name) => {
                    let encrypted_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
                    TagName::EncryptedTagName(encrypted_name)
                },
                TagName::PlainTagName(ref name) => {
                    let encrypted_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
                    TagName::PlainTagName(encrypted_name)
                }
            };
            let mut encrypted_values: Vec<TargetValue> = Vec::new();

            for value in values {
                encrypted_values.push(encrypt_name_value(&name, value, keys)?.1);
            }
            Ok(Operator::In(name, encrypted_values))
        },
        _ => Ok(op)
    }
}

// Encrypts a single tag name, tag value pair.
// If the tag name is EncryptedTagName enum variant, encrypts both the tag name and the tag value
// If the tag name is PlainTagName enum variant, encrypts only the tag name
fn encrypt_name_value(name: &TagName, value: TargetValue, keys: &Keys) -> Result<(TagName, TargetValue), WalletQueryError> {
    match (name, value) {
        (&TagName::EncryptedTagName(ref name), TargetValue::Unencrypted(ref s)) => {
            let encrypted_tag_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
            let encrypted_tag_value = ChaCha20Poly1305IETF::encrypt_as_searchable(s.as_bytes(), &keys.tag_value_key, &keys.tags_hmac_key);
            Ok((TagName::EncryptedTagName(encrypted_tag_name), TargetValue::Encrypted(encrypted_tag_value)))
        },
        (&TagName::PlainTagName(ref name), TargetValue::Unencrypted(ref s)) => {
            let encrypted_tag_name = ChaCha20Poly1305IETF::encrypt_as_searchable(&name[..], &keys.tag_name_key, &keys.tags_hmac_key);
            Ok((TagName::PlainTagName(encrypted_tag_name), TargetValue::Unencrypted(s.clone())))
        },
        _ => Err(WalletQueryError::StructureErr("Reached invalid combination of tag name and value while encrypting query".to_string()))
    }
}<|endoftext|>


<commit_before>extern crate bindgen;
extern crate pkg_config;

use bindgen::Builder;
use pkg_config::Config;
use std::env;
use std::path::PathBuf;

fn main() {
    #[cfg(windows)]
    {
        use std::path::Path;

        // get the path to GDAL_HOME
        let home_path = env::var("GDAL_HOME").expect("Environment variable $GDAL_HOME not found!");

        // detect the path to gdal_i.lib (works for MSVC and GNU)
        let lib_suffix = "_i";
        let lib_search_path = Path::new(&home_path).join("lib");
        let lib_path = lib_search_path.join(&format!("{}{}.lib", lib_name, lib_suffix));

        if lib_search_path.exists() && lib_path.exists() {
            println!("cargo:rustc-link-search={}", lib_search_path.to_string_lossy());
            println!("cargo:rustc-link-lib={}={}", link_type, format!("{}{}",lib_name, lib_suffix));
        } else {
            #[cfg(target_env="msvc")]
            {
                panic!("windows-msvc requires gdal_i.lib to be found in $GDAL_HOME\\lib.");
            }

            #[cfg(target_env="gnu")]
            {
                // detect if a gdal{version}.dll is available
                let versions = [201, 200, 111, 110];
                let bin_path = Path::new(&home_path).join("bin");
                if let Some(version) = versions.iter().find(|v| bin_path.join(&format!("{}{}.dll", lib_name, v)).exists()){
                    println!("cargo:rustc-link-search={}", bin_path.to_string_lossy());
                    println!("cargo:rustc-link-lib={}={}", link_type, format!("{}{}",lib_name, version));
                }
                else {
                    panic!("windows-gnu requires either gdal_i.lib in $GDAL_HOME\\lib OR gdal{version}.dll in $GDAL_HOME\\bin.");
                }
            }
        }
    }

    let mut builder = Builder::default();

    if let Ok(gdal) = Config::new().probe("gdal") {
        for path in &gdal.include_paths {
            builder = builder.clang_arg("-I");
            builder = builder.clang_arg(path.to_str().unwrap());
        }
    }

    let bindings = builder
        .header("wrapper.h")
        .prepend_enum_name(false)
        .constified_enum_module(".*")
        .ctypes_prefix("libc")
        .whitelist_function("CPL.*")
        .whitelist_function("GDAL.*")
        .whitelist_function("OGR.*")
        .whitelist_function("OSR.*")
        .whitelist_function("OCT.*")
        .whitelist_function("VSI.*")
        .generate()
        .expect("Unable to generate bindings");

    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
    bindings
        .write_to_file(out_path.join("bindings.rs"))
        .expect("Couldn't write bindings!");
}
<commit_msg>Disable prepend_enum_name<commit_after>extern crate bindgen;
extern crate pkg_config;

use bindgen::Builder;
use pkg_config::Config;
use std::env;
use std::path::PathBuf;

fn main() {
    #[cfg(windows)]
    {
        use std::path::Path;

        // get the path to GDAL_HOME
        let home_path = env::var("GDAL_HOME").expect("Environment variable $GDAL_HOME not found!");

        // detect the path to gdal_i.lib (works for MSVC and GNU)
        let lib_suffix = "_i";
        let lib_search_path = Path::new(&home_path).join("lib");
        let lib_path = lib_search_path.join(&format!("{}{}.lib", lib_name, lib_suffix));

        if lib_search_path.exists() && lib_path.exists() {
            println!("cargo:rustc-link-search={}", lib_search_path.to_string_lossy());
            println!("cargo:rustc-link-lib={}={}", link_type, format!("{}{}",lib_name, lib_suffix));
        } else {
            #[cfg(target_env="msvc")]
            {
                panic!("windows-msvc requires gdal_i.lib to be found in $GDAL_HOME\\lib.");
            }

            #[cfg(target_env="gnu")]
            {
                // detect if a gdal{version}.dll is available
                let versions = [201, 200, 111, 110];
                let bin_path = Path::new(&home_path).join("bin");
                if let Some(version) = versions.iter().find(|v| bin_path.join(&format!("{}{}.dll", lib_name, v)).exists()){
                    println!("cargo:rustc-link-search={}", bin_path.to_string_lossy());
                    println!("cargo:rustc-link-lib={}={}", link_type, format!("{}{}",lib_name, version));
                }
                else {
                    panic!("windows-gnu requires either gdal_i.lib in $GDAL_HOME\\lib OR gdal{version}.dll in $GDAL_HOME\\bin.");
                }
            }
        }
    }

    let mut builder = Builder::default();

    if let Ok(gdal) = Config::new().probe("gdal") {
        for path in &gdal.include_paths {
            builder = builder.clang_arg("-I");
            builder = builder.clang_arg(path.to_str().unwrap());
        }
    }

    let bindings = builder
        .header("wrapper.h")
        .constified_enum_module(".*")
        .ctypes_prefix("libc")
        .whitelist_function("CPL.*")
        .whitelist_function("GDAL.*")
        .whitelist_function("OGR.*")
        .whitelist_function("OSR.*")
        .whitelist_function("OCT.*")
        .whitelist_function("VSI.*")
        .generate()
        .expect("Unable to generate bindings");

    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
    bindings
        .write_to_file(out_path.join("bindings.rs"))
        .expect("Couldn't write bindings!");
}
<|endoftext|>


<commit_before>use std::mem::size_of;

// some data
static B: [u8; 10] = [99, 97, 114, 114, 121, 116, 111, 119, 101, 108];
static C: [u8; 11] = [116, 104, 97, 110, 107, 115, 102, 105, 115, 104, 0];

// print_info
fn print_info_ptr<T>(desc: &str, x: T)
where T : std::fmt::Pointer + std::fmt::Debug {
    println!("{}:", desc);
    println!(" location:  {:p}", &x);
    println!(" size:      {:?} bytes", size_of::<T>());
    println!(" points to: {:p}", x);
}

fn print_info<T: std::fmt::Debug>(desc: &str, x: T) {
    println!("{}:", desc);
    println!(" location:  {:p}", &x);
    println!(" size:      {:?} bytes", size_of::<T>());
    println!(" value:     {:?}", x);
}

fn main() {
    let a = 42;
    let b = &B;
    let c = Box::new(C);

    print_info("a (an unsigned integer)", a);
    println!();

    print_info_ptr("b (a reference to B)", b);
    println!();

    print_info_ptr("c (a 'box' for C):", c);
    println!();

    print_info("B (an array of 10 bytes)", B);
    println!();

    print_info("C (an array of 11 bytes)", C);
}
<commit_msg>add demo of raw pointer<commit_after>use std::mem::size_of;

// some data
static B: [u8; 10] = [99, 97, 114, 114, 121, 116, 111, 119, 101, 108];
static C: [u8; 11] = [116, 104, 97, 110, 107, 115, 102, 105, 115, 104, 0];

// print_info
fn print_info_ptr<T>(desc: &str, x: T)
where
    T: std::fmt::Pointer + std::fmt::Debug,
{
    println!("{}:", desc);
    println!(" location:  {:p}", &x);
    println!(" size:      {:?} bytes", size_of::<T>());
    println!(" points to: {:p}", x);
}

fn print_info<T: std::fmt::Debug>(desc: &str, x: T) {
    println!("{}:", desc);
    println!(" location:  {:p}", &x);
    println!(" size:      {:?} bytes", size_of::<T>());
    println!(" value:     {:?}", x);
}

fn main() {
    // introduction
    let a = 42;
    let b = &B;
    let c = Box::new(C);

    print_info("a (an unsigned integer)", a);
    println!();

    print_info_ptr("b (a reference to B)", b);
    println!();

    print_info_ptr("c (a 'box' for C):", c);
    println!();

    print_info("B (an array of 10 bytes)", B);
    println!();

    print_info("C (an array of 11 bytes)", C);

    // raw pointers
    println!("");
    println!("raw pointers:");
    println!("-------------");

    let a: i64 = 42;
    let a_ptr = &a as *const i64;
    let a_addr: usize = unsafe { std::mem::transmute(a_ptr) };
    println!("a: {} ({:p}...0x{:x})", a, a_ptr, a_addr + 7);
}
<|endoftext|>


<commit_before>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

//! A generic TiKV storage engine
//!
//! This is a work-in-progress attempt to abstract all the features needed by
//! TiKV to persist its data.
//!
//! This crate must not have any transitive dependencies on RocksDB. The RocksDB
//! implementation is in the `engine_rocks` crate.
//!
//! This documentation contains a description of the porting process, current
//! design decisions and design guidelines, and refactoring tips.
//!
//!
//! # The porting process
//!
//! These are some guidelines that seem to make the porting managable. As the
//! process continues new strategies are discovered and written here. This is a
//! big refactoring and will take many monthse.
//!
//! Refactoring is a craft, not a science, and figuring out how to overcome any
//! particular situation takes experience and intuation, but these principles
//! can help.
//!
//! A guiding principle is to do one thing at a time. In particular, don't
//! redesign while encapsulating.
//!
//! The port is happening in stages:
//!
//!   1) Migrating the `engine` abstractions
//!   2) Eliminating direct-use of `rocksdb` re-exports
//!   3) "Pulling up" the generic abstractions though TiKV
//!   4) Isolating test cases from RocksDB
//!
//! These stages are described in more detail:
//!
//! ## 1) Migrating the `engine` abstractions
//!
//! The engine crate was an earlier attempt to abstract the storage engine. Much
//! of its structure is duplicated near-identically in engine_traits, the
//! difference being that engine_traits has no RocksDB dependencies. Having no
//! RocksDB dependencies makes it trivial to guarantee that the abstractions are
//! truly abstract.
//!
//! `engine` also reexports raw bindings from `rust-rocksdb` for every purpose
//! for which there is not yet an abstract trait.
//!
//! During this stage, we will eliminate the wrappers from `engine` to reduce
//! code duplication. We do this by identifying a small subsystem within
//! `engine`, duplicating it within `engine_traits` and `engine_rocks`, deleting
//! the code from `engine`, and fixing all the callers to work with the
//! abstracted implementation.
//!
//! At the end of this stage the `engine` dependency will contain no code except
//! for `rust-rocksdb` reexports. TiKV will still depend on the concrete
//! RocksDB implementations from `engine_rocks`, as well as the raw API's from
//! reexported from the `rust-rocksdb` crate.
//!
//! ## 2) Eliminating the `engine` dep from TiKV with new abstractions
//!
//! TiKV uses reexported `rust-rocksdb` APIs via the `engine` crate. During this
//! stage we need to identify each of these APIs, duplicate them generically in
//! the `engine_traits` and `engine_rocks` crate, and convert all callers to use
//! the `engine_rocks` crate instead.
//!
//! At the end of this phase the `engine` crate will be deleted.
//!
//! ## 3) "Pulling up" the generic abstractions through TiKv
//!
//! With all of TiKV using the `engine_traits` traits in conjunction with the
//! concrete `engine_rocks` types, we can push generic type parameters up
//! through the application. Then we will remove the concrete `engine_rocks`
//! dependency from TiKV so that it is impossible to re-introduce
//! engine-specific code again.
//!
//! We will probably introduce some other crate to mediate between multiple
//! engine implementations, such that at the end of this phase TiKV will
//! not have a dependency on `engine_rocks`.
//!
//! It will though still have a dev-dependency on `engine_rocks` for the
//! test cases.
//!
//! ## 4) Isolating test cases from RocksDB
//!
//! Eventually we need our test suite to run over multiple engines.
//! The exact strategy here is yet to be determined, but it may begin by
//! breaking the `engine_rocks` dependency with a new `engine_test`, that
//! begins by simply wrapping `engine_rocks`.
//!
//!
//! # Design notes
//!
//! - `KvEngine` is the main engine trait. It requires many other traits, which
//!   have many other associated types that implement yet more traits.
//!
//! - Features should be grouped into their own modules with their own
//!   traits. A common pattern is to have an associated type that implements
//!   a trait, and an "extension" trait that associates that type with `KvEngine`,
//!   which is part of `KvEngine's trait requirements.
//!
//! - For now, for simplicity, all extension traits are required by `KvEngine`.
//!   In the future it may be feasible to separate them for engines with
//!   different feature sets.
//!
//! - Associated types generally have the same name as the trait they
//!   are required to implement. Engine extensions generally have the same
//!   name suffixed with `Ext`. Concrete implementations usually have the
//!   same name prefixed with the database name, i.e. `Rocks`.
//!
//!   Example:
//!
//!   ```ignore
//!   // in engine_traits
//!
//!   trait IOLimiterExt {
//!       type IOLimiter: IOLimiter;
//!   }
//!
//!   trait IOLimiter { }
//!   ```
//!
//!   ```ignore
//!   // in engine_rust
//!
//!   impl IOLimiterExt for RocksEngine {
//!       type IOLimiter = RocksIOLimiter;
//!   }
//!
//!   impl IOLimiter for RocksIOLimiter { }
//!   ```
//!
//! - All engines use the same error type, defined in this crate. Thus
//!   engine-specific type information is boxed and hidden.
//!
//! - `KvEngine` is a factory type for some of its associated types, but not
//!   others. For now, use factory methods when RocksDB would require factory
//!   method (that is, when the DB is required to create the associated type -
//!   if the associated type can be created without context from the database,
//!   use a standard new method). If future engines require factory methods, the
//!   traits can be converted then.
//!
//! - Types that require a handle to the engine (or some other "parent" type)
//!   do so with either Rc or Arc. An example is EngineIterator. The reason
//!   for this is that associated types cannot contain lifetimes. That requires
//!   "generic associated types". See
//!
//!   - https://github.com/rust-lang/rfcs/pull/1598
//!   - https://github.com/rust-lang/rust/issues/44265
//!
//! - Traits can't have mutually-recursive associated types. That is, if
//!   `KvEngine` has a `Snapshot` associated type, `Snapshot` can't then have a
//!   `KvEngine` associated type - the compiler will not be able to resolve both
//!   `KvEngine`s to the same type. In these cases, e.g. `Snapshot` needs to be
//!   parameterized over its engine type and `impl Snapshot<RocksEngine> for
//!   RocksSnapshot`.
//!
//!
//! # Refactoring tips
//!
//! - Port modules with the fewest RocksDB dependencies at a time, modifying
//!   those modules's callers to convert to and from the engine traits as
//!   needed. Move in and out of the engine_traits world with the
//!   `RocksDB::from_ref` and `RocksDB::as_inner` methods.
//!
//! - Down follow the type system too far "down the rabbit hole". When you see
//!   that another subsystem is blocking you from refactoring the system you
//!   are trying to refactor, stop, stash your changes, and focus on the other
//!   system instead.
//!
//! - You will through away branches that lead to dead ends. Learn from the
//!   experience and try again from a different angle.
//!
//! - For now, use the same APIs as the RocksDB bindings, as methods
//!   on the various engine traits, and with this crate's error type.
//!
//! - When new types are needed from the RocksDB API, add a new module, define a
//!   new trait (possibly with the same name as the RocksDB type), then define a
//!   `TraitExt` trait to "mixin" to the `KvEngine` trait.
//!
//! - Port methods directly from the existing `engine` crate by re-implementing
//!   it in engine_traits and engine_rocks, replacing all the callers with calls
//!   into the traits, then delete the versions in the `engine` crate.
//!
//! - Use the .c() method from engine_rocks::compat::Compat to get a
//!   KvEngine reference from Arc<DB> in the fewest characters. It also
//!   works on Snapshot, and can be adapted to other types.
//!
//! - Use tikv::into_other::IntoOther to adapt between error types of dependencies
//!   that are not themselves interdependent. E.g. raft::Error can be created
//!   from engine_traits::Error even though neither `raft` tor `engine_traits`
//!   know about each other.
//!
//! - "Plain old data" types in `engine` can be moved directly into
//!   `engine_traits` and reexported from `engine` to ease the transition.
//!   Likewise `engine_rocks` can temporarily call code from inside `engine`.

#![recursion_limit = "200"]

#[macro_use]
extern crate quick_error;
#[allow(unused_extern_crates)]
extern crate tikv_alloc;

// These modules contain traits that need to be implemented by engines, either
// they are required by KvEngine or are an associated type of KvEngine. It is
// recommended that engines follow the same module layout.
//
// Many of these define "extension" traits, that end in `Ext`.

mod cf_handle;
pub use crate::cf_handle::*;
mod cf_options;
pub use crate::cf_options::*;
mod db_options;
pub use crate::db_options::*;
mod db_vector;
pub use crate::db_vector::*;
mod engine;
pub use crate::engine::*;
mod import;
pub use import::*;
mod snapshot;
pub use crate::snapshot::*;
mod sst;
pub use crate::sst::*;
mod table_properties;
pub use crate::table_properties::*;
mod write_batch;
pub use crate::write_batch::*;

// These modules contain more general traits, some of which may be implemented
// by multiple types.

mod iterable;
pub use crate::iterable::*;
mod mutable;
pub use crate::mutable::*;
mod peekable;
pub use crate::peekable::*;

// These modules contain support code that does not need to be implemented by
// engines.

mod cfdefs;
pub use crate::cfdefs::*;
mod engines;
pub use engines::*;
mod errors;
pub use crate::errors::*;
mod options;
pub use crate::options::*;
pub mod range;
pub use crate::range::*;
pub mod util;

pub const DATA_KEY_PREFIX_LEN: usize = 1;
<commit_msg>engine_traits: fix typo (#6590)<commit_after>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

//! A generic TiKV storage engine
//!
//! This is a work-in-progress attempt to abstract all the features needed by
//! TiKV to persist its data.
//!
//! This crate must not have any transitive dependencies on RocksDB. The RocksDB
//! implementation is in the `engine_rocks` crate.
//!
//! This documentation contains a description of the porting process, current
//! design decisions and design guidelines, and refactoring tips.
//!
//!
//! # The porting process
//!
//! These are some guidelines that seem to make the porting managable. As the
//! process continues new strategies are discovered and written here. This is a
//! big refactoring and will take many monthse.
//!
//! Refactoring is a craft, not a science, and figuring out how to overcome any
//! particular situation takes experience and intuation, but these principles
//! can help.
//!
//! A guiding principle is to do one thing at a time. In particular, don't
//! redesign while encapsulating.
//!
//! The port is happening in stages:
//!
//!   1) Migrating the `engine` abstractions
//!   2) Eliminating direct-use of `rocksdb` re-exports
//!   3) "Pulling up" the generic abstractions though TiKV
//!   4) Isolating test cases from RocksDB
//!
//! These stages are described in more detail:
//!
//! ## 1) Migrating the `engine` abstractions
//!
//! The engine crate was an earlier attempt to abstract the storage engine. Much
//! of its structure is duplicated near-identically in engine_traits, the
//! difference being that engine_traits has no RocksDB dependencies. Having no
//! RocksDB dependencies makes it trivial to guarantee that the abstractions are
//! truly abstract.
//!
//! `engine` also reexports raw bindings from `rust-rocksdb` for every purpose
//! for which there is not yet an abstract trait.
//!
//! During this stage, we will eliminate the wrappers from `engine` to reduce
//! code duplication. We do this by identifying a small subsystem within
//! `engine`, duplicating it within `engine_traits` and `engine_rocks`, deleting
//! the code from `engine`, and fixing all the callers to work with the
//! abstracted implementation.
//!
//! At the end of this stage the `engine` dependency will contain no code except
//! for `rust-rocksdb` reexports. TiKV will still depend on the concrete
//! RocksDB implementations from `engine_rocks`, as well as the raw API's from
//! reexported from the `rust-rocksdb` crate.
//!
//! ## 2) Eliminating the `engine` dep from TiKV with new abstractions
//!
//! TiKV uses reexported `rust-rocksdb` APIs via the `engine` crate. During this
//! stage we need to identify each of these APIs, duplicate them generically in
//! the `engine_traits` and `engine_rocks` crate, and convert all callers to use
//! the `engine_rocks` crate instead.
//!
//! At the end of this phase the `engine` crate will be deleted.
//!
//! ## 3) "Pulling up" the generic abstractions through TiKv
//!
//! With all of TiKV using the `engine_traits` traits in conjunction with the
//! concrete `engine_rocks` types, we can push generic type parameters up
//! through the application. Then we will remove the concrete `engine_rocks`
//! dependency from TiKV so that it is impossible to re-introduce
//! engine-specific code again.
//!
//! We will probably introduce some other crate to mediate between multiple
//! engine implementations, such that at the end of this phase TiKV will
//! not have a dependency on `engine_rocks`.
//!
//! It will though still have a dev-dependency on `engine_rocks` for the
//! test cases.
//!
//! ## 4) Isolating test cases from RocksDB
//!
//! Eventually we need our test suite to run over multiple engines.
//! The exact strategy here is yet to be determined, but it may begin by
//! breaking the `engine_rocks` dependency with a new `engine_test`, that
//! begins by simply wrapping `engine_rocks`.
//!
//!
//! # Design notes
//!
//! - `KvEngine` is the main engine trait. It requires many other traits, which
//!   have many other associated types that implement yet more traits.
//!
//! - Features should be grouped into their own modules with their own
//!   traits. A common pattern is to have an associated type that implements
//!   a trait, and an "extension" trait that associates that type with `KvEngine`,
//!   which is part of `KvEngine's trait requirements.
//!
//! - For now, for simplicity, all extension traits are required by `KvEngine`.
//!   In the future it may be feasible to separate them for engines with
//!   different feature sets.
//!
//! - Associated types generally have the same name as the trait they
//!   are required to implement. Engine extensions generally have the same
//!   name suffixed with `Ext`. Concrete implementations usually have the
//!   same name prefixed with the database name, i.e. `Rocks`.
//!
//!   Example:
//!
//!   ```ignore
//!   // in engine_traits
//!
//!   trait IOLimiterExt {
//!       type IOLimiter: IOLimiter;
//!   }
//!
//!   trait IOLimiter { }
//!   ```
//!
//!   ```ignore
//!   // in engine_rocks
//!
//!   impl IOLimiterExt for RocksEngine {
//!       type IOLimiter = RocksIOLimiter;
//!   }
//!
//!   impl IOLimiter for RocksIOLimiter { }
//!   ```
//!
//! - All engines use the same error type, defined in this crate. Thus
//!   engine-specific type information is boxed and hidden.
//!
//! - `KvEngine` is a factory type for some of its associated types, but not
//!   others. For now, use factory methods when RocksDB would require factory
//!   method (that is, when the DB is required to create the associated type -
//!   if the associated type can be created without context from the database,
//!   use a standard new method). If future engines require factory methods, the
//!   traits can be converted then.
//!
//! - Types that require a handle to the engine (or some other "parent" type)
//!   do so with either Rc or Arc. An example is EngineIterator. The reason
//!   for this is that associated types cannot contain lifetimes. That requires
//!   "generic associated types". See
//!
//!   - https://github.com/rust-lang/rfcs/pull/1598
//!   - https://github.com/rust-lang/rust/issues/44265
//!
//! - Traits can't have mutually-recursive associated types. That is, if
//!   `KvEngine` has a `Snapshot` associated type, `Snapshot` can't then have a
//!   `KvEngine` associated type - the compiler will not be able to resolve both
//!   `KvEngine`s to the same type. In these cases, e.g. `Snapshot` needs to be
//!   parameterized over its engine type and `impl Snapshot<RocksEngine> for
//!   RocksSnapshot`.
//!
//!
//! # Refactoring tips
//!
//! - Port modules with the fewest RocksDB dependencies at a time, modifying
//!   those modules's callers to convert to and from the engine traits as
//!   needed. Move in and out of the engine_traits world with the
//!   `RocksDB::from_ref` and `RocksDB::as_inner` methods.
//!
//! - Down follow the type system too far "down the rabbit hole". When you see
//!   that another subsystem is blocking you from refactoring the system you
//!   are trying to refactor, stop, stash your changes, and focus on the other
//!   system instead.
//!
//! - You will through away branches that lead to dead ends. Learn from the
//!   experience and try again from a different angle.
//!
//! - For now, use the same APIs as the RocksDB bindings, as methods
//!   on the various engine traits, and with this crate's error type.
//!
//! - When new types are needed from the RocksDB API, add a new module, define a
//!   new trait (possibly with the same name as the RocksDB type), then define a
//!   `TraitExt` trait to "mixin" to the `KvEngine` trait.
//!
//! - Port methods directly from the existing `engine` crate by re-implementing
//!   it in engine_traits and engine_rocks, replacing all the callers with calls
//!   into the traits, then delete the versions in the `engine` crate.
//!
//! - Use the .c() method from engine_rocks::compat::Compat to get a
//!   KvEngine reference from Arc<DB> in the fewest characters. It also
//!   works on Snapshot, and can be adapted to other types.
//!
//! - Use tikv::into_other::IntoOther to adapt between error types of dependencies
//!   that are not themselves interdependent. E.g. raft::Error can be created
//!   from engine_traits::Error even though neither `raft` tor `engine_traits`
//!   know about each other.
//!
//! - "Plain old data" types in `engine` can be moved directly into
//!   `engine_traits` and reexported from `engine` to ease the transition.
//!   Likewise `engine_rocks` can temporarily call code from inside `engine`.

#![recursion_limit = "200"]

#[macro_use]
extern crate quick_error;
#[allow(unused_extern_crates)]
extern crate tikv_alloc;

// These modules contain traits that need to be implemented by engines, either
// they are required by KvEngine or are an associated type of KvEngine. It is
// recommended that engines follow the same module layout.
//
// Many of these define "extension" traits, that end in `Ext`.

mod cf_handle;
pub use crate::cf_handle::*;
mod cf_options;
pub use crate::cf_options::*;
mod db_options;
pub use crate::db_options::*;
mod db_vector;
pub use crate::db_vector::*;
mod engine;
pub use crate::engine::*;
mod import;
pub use import::*;
mod snapshot;
pub use crate::snapshot::*;
mod sst;
pub use crate::sst::*;
mod table_properties;
pub use crate::table_properties::*;
mod write_batch;
pub use crate::write_batch::*;

// These modules contain more general traits, some of which may be implemented
// by multiple types.

mod iterable;
pub use crate::iterable::*;
mod mutable;
pub use crate::mutable::*;
mod peekable;
pub use crate::peekable::*;

// These modules contain support code that does not need to be implemented by
// engines.

mod cfdefs;
pub use crate::cfdefs::*;
mod engines;
pub use engines::*;
mod errors;
pub use crate::errors::*;
mod options;
pub use crate::options::*;
pub mod range;
pub use crate::range::*;
pub mod util;

pub const DATA_KEY_PREFIX_LEN: usize = 1;
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Eliza Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015-2017 Eliza Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Parsing and loading Executable and Linkable Format (ELF) 32- and 64-bit
//! binaries.
//!
//! For more information on the ELF format, refer to:
//!
//!  + [Wikipedia](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format)
//!  + The [OS Dev Wiki](http://wiki.osdev.org/ELF)
//!  + The [ELF Format Specification](elfspec)
//!
//! [elfspec]: http://www.skyfree.org/linux/references/ELF_Format.pdf
#![feature(core_intrinsics)]
#![feature(try_from)]
#![no_std]

#[macro_use] extern crate bitflags;
#[macro_use] extern crate macro_attr;

extern crate memory;

use core::{ convert, ops, mem, slice };
use core::convert::TryFrom;

macro_rules! impl_getters {
    ($(#[$attr:meta])* pub fn $name:ident(&self) -> $ty:ty; $($rest:tt)*) => {
        $(#[$attr])* #[inline] pub fn $name(&self) -> $ty { self.$name as $ty }
        impl_getters!{ $( $rest )* }
    };
    ($(#[$attr:meta])* fn $name:ident(&self) -> $ty:ty; $($rest:tt)*) => {
        $(#[$attr])* #[inline] fn $name(&self) -> $ty { self.$name as $ty }
        impl_getters!{ $( $rest )* }
    };
    ( $(#[$attr:meta])* pub fn $name: ident (&self)-> $ty:ty; ) => {
        $(#[$attr])* #[inline] pub fn $name(&self) -> $ty { self.$name as $ty }
    };
    ( $(#[$attr:meta])* fn $name: ident (&self)-> $ty:ty; ) => {
        $(#[$attr])* #[inline] fn $name(&self) -> $ty { self.$name as $ty }
    };
    () => {};
}

pub mod section;
pub mod file;
pub mod program;

/// An ELF section header.
pub type Section<'a> = section::Header<'a>;
/// An ELF header file.
pub type FileHeader<W> = file::HeaderRepr<W>;

/// TODO: should ELF have its own error type?
pub type ElfResult<T> = Result<T, &'static str>;

pub trait ElfWord: Sized + Copy + Clone
                         + ops::Add<Self> + ops::Sub<Self>
                         + ops::Mul<Self> + ops::Div<Self>
                         + ops::Shl<Self> + ops::Shr<Self> { }
impl ElfWord for u64 { }
impl ElfWord for u32 { }

/// Hack to make the type-system let me do what I want
trait ValidatesWord<Word: ElfWord> {
    fn check(&self) -> ElfResult<()>;
}

/// A handle on a parsed ELF binary
///  TODO: do we want this to own a HashMap of section names to section headers,
///        to speed up section lookup?
//          - eliza, 03/08/2017
#[derive(Debug)]
pub struct Image<'a, Word, ProgHeader, Header = FileHeader<Word>>
where Word: ElfWord + 'a
    , ProgHeader: program::Header<Word = Word> + 'a
    , Header: file::Header<Word = Word> + 'a
    {
    /// the binary's [file header](file/trait.Header.html)
    pub header: &'a Header
  , /// references to each [section header](section/struct.Header.html)
    pub sections: &'a [section::Header<'a>]
  , /// references to each [program header](program/trait.Header.html)
    pub program_headers: &'a [ProgHeader]
  , /// the raw binary contents of the ELF binary.
    /// note that this includes the _entire_ binary contents of the file,
    /// so the file header and each section header is included in this slice.
    binary: &'a [u8]
}

impl<'a, Word, ProgHeader, Header> Image<'a, Word, ProgHeader, Header>
where Word: ElfWord + 'a
    , ProgHeader: program::Header<Word = Word> + 'a
    , Header: file::Header<Word = Word> + 'a
    {
    /// Returns the section header [string table].
    ///
    /// [string table]: section/struct.StrTable.html
    pub fn sh_str_table(&'a self) -> section::StrTable<'a> {
        // TODO: do we want to validate that the string table index is
        //       reasonable (e.g. it's not longer than the binary)?
        //          - eliza, 03/08/2017
        // TODO: do we want to cache a ref to the string table?
        //          - eliza, 03/08/2017
        section::StrTable::from(&self.binary[self.header.sh_str_idx()..])
    }

}

impl<'a, Word, PH, H> TryFrom<&'a [u8]> for Image<'a, Word, PH, H>
where Word: ElfWord + 'a
    , PH: program::Header<Word = Word> + 'a
    , H: file::Header<Word = Word> + 'a
    , &'a H: convert::TryFrom<&'a [u8], Err = &'static str>
    {

    type Err = &'static str;

    fn try_from(bytes: &'a [u8]) -> ElfResult<Self> {
        let header: &'a H = <&'a H>::try_from(bytes)?;

        // TODO: this won't actually work; need to rewrite section headers to
        //       work the same as program headers.
        //          - eliza, 03/14/2017
        let sections = unsafe { extract_from_slice::<Section<'a>>(
            &bytes[header.sh_range()]
          , 0
          , header.sh_count()
        )? };
        let prog_headers = unsafe { extract_from_slice::<PH>(
            &bytes[header.ph_range()]
          , 0
          , header.ph_count()
        )? };
        Ok(Image { header: header
              , sections: sections
              , program_headers: prog_headers
              , binary: bytes
        })
    }
}

/// Extract `n` instances of type `T` from a byte slice.
///
/// This is essentially just a _slightly_ safer wrapper around
/// [`slice::from_raw_parts`]. Unlike `from_raw_parts`, this function takes
/// a valid byte slice, rather than a pointer. Therefore, some of the safety
/// issues with `from_raw_parts` are avoided:
///
/// + the lifetime (`'slice`) of the returned slice should be the same as the
///   lifetime of the input slice (`data`), rather than inferred arbitrarily.
/// + this function will panic rather than reading past the end of the slice.
///
/// # Arguments
///
/// + `data`: the byte slice to extract a slice of `&[T]`s from
/// + `offset`: a start offset into `data`
/// + `n`: the number of instances of `T` which should be contained
///        in `data[offset..]`
///
/// # Safety
///
/// While this function is safer than [`slice::from_raw_parts`],
/// it is still unsafe for the following reasons:
///
/// + The contents of `data` may not be able to be interpreted as instances of
///   type `T`.
///
/// # Caveats
///
/// + If `n` == 0, this will give you an `&[]`. Just a warning.
//    thanks to Max for making  me figure this out.
/// + `offset` must be aligned on a `T`-sized boundary.
///
/// # Panics
///
/// + If the index `offset` is longer than `T`
///
/// TODO: rewrite this as a `TryFrom` implementation (see issue #85)
//          - eliza, 03/09/2017
///       wait, possibly we should NOT do that. actually we should
///       almost certainly not do that. since this function is unsafe,
///       but `TryFrom` is not, and because this would be WAY generic.
//          - eliza, 03/09/2017
/// TODO: is this general enough to move into util?
//          - eliza, 03/09/2017
/// TODO: refactor this to take a `RangeArgument`?
//          - eliza, 03/13/2017
///       or, we could just remove the offset and expect the caller to
///       offset the slice?
//          - eliza, 03/14/2017
///
/// [`slice::from_raw_parts`]: https://doc.rust-lang.org/stable/std/slice/fn.from_raw_parts.html
unsafe fn extract_from_slice<'slice, T: Sized>( data: &'slice [u8]
                                              , offset: usize
                                              , n: usize)
                                              -> ElfResult<&'slice [T]> {
    use core::intrinsics::type_name;
    if offset % mem::align_of::<T>() != 0 {
        // TODO: these error messages don't contain as much information as they
        //       used to, since the return type is `&'static str` that can't be
        //       dynamically formatted as the panic was. refactor this?
        //       (e.g. should ELF get its own error type?)
        //        - eliza, 03/15/2017
        Err("extract_from_slice: Offset not aligned on type T sized boundary!")
        // assert!(
        //        , "Offset {} not aligned on a {}-sized boundary (must be \
        //           divisible by {})."
        //        , offset, type_name::<T>(), mem::align_of::<T>()
        //        );
    } else if data.len() - offset < mem::size_of::<T>() * n {
        Err("extract_from_slice: Slice too short to contain n instances of T!")
        // assert!(
        //        , "Slice too short to contain {} objects of type {}"
        //        , n, type_name::<T>()
        //        );
    } else {
        Ok(slice::from_raw_parts(data[offset..].as_ptr() as *const T, n))
    }
}
<commit_msg>feat(elf): add default type parameters for Image, default word type<commit_after>//
//  SOS: the Stupid Operating System
//  by Eliza Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015-2017 Eliza Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Parsing and loading Executable and Linkable Format (ELF) 32- and 64-bit
//! binaries.
//!
//! For more information on the ELF format, refer to:
//!
//!  + [Wikipedia](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format)
//!  + The [OS Dev Wiki](http://wiki.osdev.org/ELF)
//!  + The [ELF Format Specification](elfspec)
//!
//! [elfspec]: http://www.skyfree.org/linux/references/ELF_Format.pdf
#![feature(core_intrinsics)]
#![feature(try_from)]
#![no_std]

#[macro_use] extern crate bitflags;
#[macro_use] extern crate macro_attr;

extern crate memory;

use core::{ convert, ops, mem, slice };
use core::convert::TryFrom;

macro_rules! impl_getters {
    ($(#[$attr:meta])* pub fn $name:ident(&self) -> $ty:ty; $($rest:tt)*) => {
        $(#[$attr])* #[inline] pub fn $name(&self) -> $ty { self.$name as $ty }
        impl_getters!{ $( $rest )* }
    };
    ($(#[$attr:meta])* fn $name:ident(&self) -> $ty:ty; $($rest:tt)*) => {
        $(#[$attr])* #[inline] fn $name(&self) -> $ty { self.$name as $ty }
        impl_getters!{ $( $rest )* }
    };
    ( $(#[$attr:meta])* pub fn $name: ident (&self)-> $ty:ty; ) => {
        $(#[$attr])* #[inline] pub fn $name(&self) -> $ty { self.$name as $ty }
    };
    ( $(#[$attr:meta])* fn $name: ident (&self)-> $ty:ty; ) => {
        $(#[$attr])* #[inline] fn $name(&self) -> $ty { self.$name as $ty }
    };
    () => {};
}

pub mod section;
pub mod file;
pub mod program;

/// An ELF section header.
pub type Section<'a> = section::Header<'a>;
/// An ELF header file.
pub type FileHeader<W> = file::HeaderRepr<W>;

/// TODO: should ELF have its own error type?
pub type ElfResult<T> = Result<T, &'static str>;

pub trait ElfWord: Sized + Copy + Clone
                         + ops::Add<Self> + ops::Sub<Self>
                         + ops::Mul<Self> + ops::Div<Self>
                         + ops::Shl<Self> + ops::Shr<Self> { }
impl ElfWord for u64 { }
impl ElfWord for u32 { }

#[cfg(target_pointer_width = "32")]
type DefaultWord = u32;
#[cfg(target_pointer_width = "64")]
type DefaultWord = u64;

/// Hack to make the type-system let me do what I want
trait ValidatesWord<Word: ElfWord> {
    fn check(&self) -> ElfResult<()>;
}

/// A handle on a parsed ELF binary
///  TODO: do we want this to own a HashMap of section names to section headers,
///        to speed up section lookup?
//          - eliza, 03/08/2017
#[derive(Debug)]
pub struct Image< 'a
                , Word = DefaultWord
                , ProgHeader = program::Header<Word = Word>
                , Header = FileHeader<Word>
                >
where Word: ElfWord + 'a
    , ProgHeader: program::Header<Word = Word> + Sized +'a
    , Header: file::Header<Word = Word> + 'a
    {
    /// the binary's [file header](file/trait.Header.html)
    pub header: &'a Header
  , /// references to each [section header](section/struct.Header.html)
    pub sections: &'a [section::Header<'a>]
  , /// references to each [program header](program/trait.Header.html)
    pub program_headers: &'a [ProgHeader]
  , /// the raw binary contents of the ELF binary.
    /// note that this includes the _entire_ binary contents of the file,
    /// so the file header and each section header is included in this slice.
    binary: &'a [u8]
}

impl<'a, Word, ProgHeader, Header> Image<'a, Word, ProgHeader, Header>
where Word: ElfWord + 'a
    , ProgHeader: program::Header<Word = Word> + 'a
    , Header: file::Header<Word = Word> + 'a
    {
    /// Returns the section header [string table].
    ///
    /// [string table]: section/struct.StrTable.html
    pub fn sh_str_table(&'a self) -> section::StrTable<'a> {
        // TODO: do we want to validate that the string table index is
        //       reasonable (e.g. it's not longer than the binary)?
        //          - eliza, 03/08/2017
        // TODO: do we want to cache a ref to the string table?
        //          - eliza, 03/08/2017
        section::StrTable::from(&self.binary[self.header.sh_str_idx()..])
    }

}

impl<'a, Word, PH, H> TryFrom<&'a [u8]> for Image<'a, Word, PH, H>
where Word: ElfWord + 'a
    , PH: program::Header<Word = Word> + 'a
    , H: file::Header<Word = Word> + 'a
    , &'a H: convert::TryFrom<&'a [u8], Err = &'static str>
    {

    type Err = &'static str;

    fn try_from(bytes: &'a [u8]) -> ElfResult<Self> {
        let header: &'a H = <&'a H>::try_from(bytes)?;

        // TODO: this won't actually work; need to rewrite section headers to
        //       work the same as program headers.
        //          - eliza, 03/14/2017
        let sections = unsafe { extract_from_slice::<Section<'a>>(
            &bytes[header.sh_range()]
          , 0
          , header.sh_count()
        )? };
        let prog_headers = unsafe { extract_from_slice::<PH>(
            &bytes[header.ph_range()]
          , 0
          , header.ph_count()
        )? };
        Ok(Image { header: header
              , sections: sections
              , program_headers: prog_headers
              , binary: bytes
        })
    }
}

/// Extract `n` instances of type `T` from a byte slice.
///
/// This is essentially just a _slightly_ safer wrapper around
/// [`slice::from_raw_parts`]. Unlike `from_raw_parts`, this function takes
/// a valid byte slice, rather than a pointer. Therefore, some of the safety
/// issues with `from_raw_parts` are avoided:
///
/// + the lifetime (`'slice`) of the returned slice should be the same as the
///   lifetime of the input slice (`data`), rather than inferred arbitrarily.
/// + this function will panic rather than reading past the end of the slice.
///
/// # Arguments
///
/// + `data`: the byte slice to extract a slice of `&[T]`s from
/// + `offset`: a start offset into `data`
/// + `n`: the number of instances of `T` which should be contained
///        in `data[offset..]`
///
/// # Safety
///
/// While this function is safer than [`slice::from_raw_parts`],
/// it is still unsafe for the following reasons:
///
/// + The contents of `data` may not be able to be interpreted as instances of
///   type `T`.
///
/// # Caveats
///
/// + If `n` == 0, this will give you an `&[]`. Just a warning.
//    thanks to Max for making  me figure this out.
/// + `offset` must be aligned on a `T`-sized boundary.
///
/// # Panics
///
/// + If the index `offset` is longer than `T`
///
/// TODO: rewrite this as a `TryFrom` implementation (see issue #85)
//          - eliza, 03/09/2017
///       wait, possibly we should NOT do that. actually we should
///       almost certainly not do that. since this function is unsafe,
///       but `TryFrom` is not, and because this would be WAY generic.
//          - eliza, 03/09/2017
/// TODO: is this general enough to move into util?
//          - eliza, 03/09/2017
/// TODO: refactor this to take a `RangeArgument`?
//          - eliza, 03/13/2017
///       or, we could just remove the offset and expect the caller to
///       offset the slice?
//          - eliza, 03/14/2017
///
/// [`slice::from_raw_parts`]: https://doc.rust-lang.org/stable/std/slice/fn.from_raw_parts.html
unsafe fn extract_from_slice<'slice, T: Sized>( data: &'slice [u8]
                                              , offset: usize
                                              , n: usize)
                                              -> ElfResult<&'slice [T]> {
    use core::intrinsics::type_name;
    if offset % mem::align_of::<T>() != 0 {
        // TODO: these error messages don't contain as much information as they
        //       used to, since the return type is `&'static str` that can't be
        //       dynamically formatted as the panic was. refactor this?
        //       (e.g. should ELF get its own error type?)
        //        - eliza, 03/15/2017
        Err("extract_from_slice: Offset not aligned on type T sized boundary!")
        // assert!(
        //        , "Offset {} not aligned on a {}-sized boundary (must be \
        //           divisible by {})."
        //        , offset, type_name::<T>(), mem::align_of::<T>()
        //        );
    } else if data.len() - offset < mem::size_of::<T>() * n {
        Err("extract_from_slice: Slice too short to contain n instances of T!")
        // assert!(
        //        , "Slice too short to contain {} objects of type {}"
        //        , n, type_name::<T>()
        //        );
    } else {
        Ok(slice::from_raw_parts(data[offset..].as_ptr() as *const T, n))
    }
}
<|endoftext|>


<commit_before>#![cfg(all(
    not(target_os = "fuchsia"),
    not(target_os = "android"),
    not(target_os = "windows"),
    not(target_os = "macos")
))]

extern crate libc;
extern crate pagecache;
extern crate rand;
extern crate sled;
extern crate tests;

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use pagecache::{Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            slice_to_u32(&*v),
            lowest,
            "expected key {} to have value {}, instead it had value {}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v)
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    tests::setup_logger();
    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = sled::Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 100_000);
        value.append(&mut vec![0u8; additional_len]);

        tree.set(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(1 << 10)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
#[ignore]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
#[ignore]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<commit_msg>make crash test use much smaller io buffers to increase the chances of an interesting bug popping out<commit_after>#![cfg(all(
    not(target_os = "fuchsia"),
    not(target_os = "android"),
    not(target_os = "windows"),
    not(target_os = "macos")
))]

extern crate libc;
extern crate pagecache;
extern crate rand;
extern crate sled;
extern crate tests;

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use pagecache::{Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            slice_to_u32(&*v),
            lowest,
            "expected key {} to have value {}, instead it had value {}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v)
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    tests::setup_logger();
    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = sled::Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 1000);
        value.append(&mut vec![0u8; additional_len]);

        tree.set(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(1_000)
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(1_000)
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(1 << 10)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
#[ignore]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
#[ignore]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<|endoftext|>