<commit_before>extern crate directories;

#[cfg(target_family = "unix")]
use directories::{BaseDirs, ProjectDirs};

use std::fs::{create_dir_all, File};
use std::io::{ErrorKind, Read};
use std::path::PathBuf;
use std::process::Command;

#[cfg(target_family = "unix")]
use std::fs::read_dir;
#[cfg(target_family = "unix")]
use std::io::Write;

fn generate_resources() {
    let target_path = PathBuf::from("target").join("resources");
    create_dir_all(&target_path).unwrap();

    // Icons
    let input_path = PathBuf::from("assets").join("icons").join("hicolor");

    let mut compile_res = Command::new("glib-compile-resources");
    compile_res
        .arg("--generate")
        .arg(format!("--sourcedir={}", input_path.to_str().unwrap()))
        .arg(format!(
            "--target={}",
            target_path.join("icons.gresource").to_str().unwrap(),
        ))
        .arg(input_path.join("icons.gresource.xml").to_str().unwrap());

    match compile_res.status() {
        Ok(status) => if !status.success() {
            panic!(format!(
                "Failed to generate resources file for icons\n{:?}",
                compile_res,
            ));
        },
        Err(ref error) => match error.kind() {
            ErrorKind::NotFound => {
                panic!("Can't generate translations: command `compile_res` not available");
            }
            _ => panic!("Error invoking `compile_res`: {}", error),
        },
    }

    // UI
    let input_path = PathBuf::from("assets").join("ui");

    let mut compile_res = Command::new("glib-compile-resources");
    compile_res
        .arg("--generate")
        .arg(format!("--sourcedir={}", input_path.to_str().unwrap()))
        .arg(format!(
            "--target={}",
            target_path.join("ui.gresource").to_str().unwrap(),
        ))
        .arg(input_path.join("ui.gresource.xml").to_str().unwrap());

    match compile_res.status() {
        Ok(status) => if !status.success() {
            panic!(format!(
                "Failed to generate resources file for the UI\n{:?}",
                compile_res,
            ));
        },
        Err(ref error) => match error.kind() {
            ErrorKind::NotFound => {
                panic!("Can't generate translations: command `compile_res` not available");
            }
            _ => panic!("Error invoking `compile_res`: {}", error),
        },
    }
}

fn generate_translations() {
    if let Ok(mut linguas_file) = File::open(&PathBuf::from("po").join("LINGUAS")) {
        let mut linguas = String::new();
        linguas_file
            .read_to_string(&mut linguas)
            .expect("Couldn't read po/LINGUAS as string");

        for lingua in linguas.lines() {
            let mo_path = PathBuf::from("target")
                .join("locale")
                .join(lingua)
                .join("LC_MESSAGES");
            create_dir_all(&mo_path).unwrap();

            let mut msgfmt = Command::new("msgfmt");
            msgfmt
                .arg(format!(
                    "--output-file={}",
                    mo_path.join("media-toc.mo").to_str().unwrap()
                ))
                .arg("--directory=po")
                .arg(format!("{}.po", lingua));

            match msgfmt.status() {
                Ok(status) => if !status.success() {
                    panic!(format!(
                        "Failed to generate mo file for lingua {}\n{:?}",
                        lingua, msgfmt,
                    ));
                },
                Err(ref error) => match error.kind() {
                    ErrorKind::NotFound => {
                        eprintln!("Can't generate translations: command `msgfmt` not available");
                        return;
                    }
                    _ => panic!("Error invoking `msgfmt`: {}", error),
                },
            }
        }
    }
}

// FIXME: figure out macOS conventions for icons & translations
#[cfg(target_family = "unix")]
fn generate_install_script() {
    let base_dirs = BaseDirs::new();
    // Note: `base_dirs.executable_dir()` is `None` on macOS
    if let Some(exe_dir) = base_dirs.executable_dir() {
        let project_dirs = ProjectDirs::from("org", "fengalin", env!("CARGO_PKG_NAME"));
        let app_data_dir = project_dirs.data_dir();
        let data_dir = app_data_dir.parent().unwrap();

        match File::create(&PathBuf::from("target").join("install")) {
            Ok(mut install_file) => {
                install_file
                    .write_all(
                        format!("# User install script for {}\n", env!("CARGO_PKG_NAME"))
                            .as_bytes(),
                    )
                    .unwrap();

                install_file.write_all(b"\n# Install executable\n").unwrap();
                let exe_source_path = PathBuf::from("target")
                    .join("release")
                    .join(env!("CARGO_PKG_NAME"));
                install_file
                    .write_all(format!("mkdir -p {}\n", exe_dir.to_str().unwrap()).as_bytes())
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp {} {}\n",
                            exe_source_path.to_str().unwrap(),
                            exe_dir.join(env!("CARGO_PKG_NAME")).to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();

                install_file.write_all(b"\n# Install icons\n").unwrap();
                let icon_target_dir = data_dir.join("icons").join("hicolor");
                let mut entry_iter =
                    read_dir(PathBuf::from("assets").join("icons").join("hicolor")).unwrap();
                for entry in entry_iter {
                    let entry = entry.unwrap();
                    let entry_path = entry.path();
                    if entry_path.is_dir() {
                        let target_dir = icon_target_dir.join(&entry.file_name());
                        install_file
                            .write_all(
                                format!("mkdir -p {}\n", target_dir.to_str().unwrap()).as_bytes(),
                            )
                            .unwrap();

                        install_file
                            .write_all(
                                format!(
                                    "cp -r {:?}/* {:?}\n",
                                    entry_path.to_str().unwrap(),
                                    target_dir,
                                ).as_bytes(),
                            )
                            .unwrap();
                    }
                }

                install_file
                    .write_all(b"\n# Install translations\n")
                    .unwrap();
                install_file
                    .write_all(format!("mkdir -p {}\n", data_dir.to_str().unwrap()).as_bytes())
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp -r {} {}\n",
                            PathBuf::from("target").join("locale").to_str().unwrap(),
                            data_dir.to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();

                install_file
                    .write_all(b"\n# Install desktop file\n")
                    .unwrap();
                let desktop_target_dir = data_dir.join("applications");
                install_file
                    .write_all(
                        format!("mkdir -p {}\n", desktop_target_dir.to_str().unwrap()).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp {} {}\n",
                            PathBuf::from("assets")
                                .join(&format!("org.fengalin.{}.desktop", env!("CARGO_PKG_NAME")))
                                .to_str()
                                .unwrap(),
                            desktop_target_dir.to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
            }
            Err(err) => panic!("Couldn't create file `target/install`: {:?}", err),
        }
    }
}

// FIXME: figure out macOS conventions for icons & translations
#[cfg(target_family = "unix")]
fn generate_uninstall_script() {
    let base_dirs = BaseDirs::new();
    // Note: `base_dirs.executable_dir()` is `None` on macOS
    if let Some(exe_dir) = base_dirs.executable_dir() {
        let project_dirs = ProjectDirs::from("org", "fengalin", env!("CARGO_PKG_NAME"));
        let app_data_dir = project_dirs.data_dir();
        let data_dir = app_data_dir.parent().unwrap();

        match File::create(&PathBuf::from("target").join("uninstall")) {
            Ok(mut install_file) => {
                install_file
                    .write_all(
                        format!("# User uninstall script for {}\n", env!("CARGO_PKG_NAME"))
                            .as_bytes(),
                    )
                    .unwrap();

                install_file
                    .write_all(b"\n# Uninstall executable\n")
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "rm {}\n",
                            exe_dir.join(env!("CARGO_PKG_NAME")).to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(format!("rmdir -p {}\n", exe_dir.to_str().unwrap()).as_bytes())
                    .unwrap();

                install_file.write_all(b"\n# Uninstall icons\n").unwrap();
                let icon_target_dir = data_dir.join("icons").join("hicolor");
                let mut entry_iter =
                    read_dir(PathBuf::from("assets").join("icons").join("hicolor")).unwrap();
                for entry in entry_iter {
                    let entry = entry.unwrap();
                    let entry_path = entry.path();
                    if entry_path.is_dir() {
                        let icon_target_dir = icon_target_dir
                            .join(entry_path.file_name().unwrap())
                            .join("apps");
                        let mut icons_subdir_iter = read_dir(entry_path.join("apps")).unwrap();
                        for icon_subdir_entry in icons_subdir_iter {
                            let icon_subdir_entry = icon_subdir_entry.unwrap();
                            install_file
                                .write_all(
                                    format!(
                                        "rm {}\n",
                                        icon_target_dir
                                            .join(icon_subdir_entry.file_name())
                                            .to_str()
                                            .unwrap(),
                                    ).as_bytes(),
                                )
                                .unwrap();
                        }

                        install_file
                            .write_all(
                                format!("rmdir -p {}\n", icon_target_dir.to_str().unwrap())
                                    .as_bytes(),
                            )
                            .unwrap();
                    }
                }

                if let Ok(mut linguas_file) = File::open(&PathBuf::from("po").join("LINGUAS")) {
                    let mut linguas = String::new();
                    linguas_file
                        .read_to_string(&mut linguas)
                        .expect("Couldn't read po/LINGUAS as string");

                    install_file
                        .write_all(b"\n# Uninstall translations\n")
                        .unwrap();
                    let locale_base_dir = data_dir.join("locale");
                    for lingua in linguas.lines() {
                        let lingua_dir = locale_base_dir.join(lingua).join("LC_MESSAGES");
                        install_file
                            .write_all(
                                format!(
                                    "rm {}\n",
                                    lingua_dir
                                        .join(&format!("{}.mo", env!("CARGO_PKG_NAME")))
                                        .to_str()
                                        .unwrap(),
                                ).as_bytes(),
                            )
                            .unwrap();

                        install_file
                            .write_all(
                                format!("rmdir -p {}\n", lingua_dir.to_str().unwrap()).as_bytes(),
                            )
                            .unwrap();
                    }
                }

                install_file
                    .write_all(b"\n# Uninstall desktop file\n")
                    .unwrap();
                let desktop_target_dir = data_dir.join("applications");
                install_file
                    .write_all(
                        format!(
                            "rm {}\n",
                            desktop_target_dir
                                .join(&format!("org.fengalin.{}.desktop", env!("CARGO_PKG_NAME")))
                                .to_str()
                                .unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(
                        format!("rmdir -p {}\n", desktop_target_dir.to_str().unwrap()).as_bytes(),
                    )
                    .unwrap();
            }
            Err(err) => panic!("Couldn't create file `target/uninstall`: {:?}", err),
        }
    }
}

fn main() {
    generate_resources();
    generate_translations();

    #[cfg(target_family = "unix")]
    generate_install_script();

    #[cfg(target_family = "unix")]
    generate_uninstall_script();
}
<commit_msg>build.rs minor fixes<commit_after>extern crate directories;

#[cfg(target_family = "unix")]
use directories::{BaseDirs, ProjectDirs};

use std::fs::{create_dir_all, File};
use std::io::{ErrorKind, Read};
use std::path::PathBuf;
use std::process::Command;

#[cfg(target_family = "unix")]
use std::fs::read_dir;
#[cfg(target_family = "unix")]
use std::io::Write;

fn generate_resources() {
    let target_path = PathBuf::from("target").join("resources");
    create_dir_all(&target_path).unwrap();

    // Icons
    let input_path = PathBuf::from("assets").join("icons").join("hicolor");

    let mut compile_res = Command::new("glib-compile-resources");
    compile_res
        .arg("--generate")
        .arg(format!("--sourcedir={}", input_path.to_str().unwrap()))
        .arg(format!(
            "--target={}",
            target_path.join("icons.gresource").to_str().unwrap(),
        ))
        .arg(input_path.join("icons.gresource.xml").to_str().unwrap());

    match compile_res.status() {
        Ok(status) => if !status.success() {
            panic!(format!(
                "Failed to generate resources file for icons\n{:?}",
                compile_res,
            ));
        },
        Err(ref error) => match error.kind() {
            ErrorKind::NotFound => panic!(
                "Can't generate icon resources: command `glib-compile-resources` not available"
            ),
            _ => panic!("Error invoking `compile_res`: {}", error),
        },
    }

    // UI
    let input_path = PathBuf::from("assets").join("ui");

    let mut compile_res = Command::new("glib-compile-resources");
    compile_res
        .arg("--generate")
        .arg(format!("--sourcedir={}", input_path.to_str().unwrap()))
        .arg(format!(
            "--target={}",
            target_path.join("ui.gresource").to_str().unwrap(),
        ))
        .arg(input_path.join("ui.gresource.xml").to_str().unwrap());

    match compile_res.status() {
        Ok(status) => if !status.success() {
            panic!(format!(
                "Failed to generate resources file for the UI\n{:?}",
                compile_res,
            ));
        },
        Err(ref error) => match error.kind() {
            ErrorKind::NotFound => panic!(
                "Can't generate UI resources: command `glib-compile-resources` not available"
            ),
            _ => panic!("Error invoking `compile_res`: {}", error),
        },
    }
}

fn generate_translations() {
    if let Ok(mut linguas_file) = File::open(&PathBuf::from("po").join("LINGUAS")) {
        let mut linguas = String::new();
        linguas_file
            .read_to_string(&mut linguas)
            .expect("Couldn't read po/LINGUAS as string");

        for lingua in linguas.lines() {
            let mo_path = PathBuf::from("target")
                .join("locale")
                .join(lingua)
                .join("LC_MESSAGES");
            create_dir_all(&mo_path).unwrap();

            let mut msgfmt = Command::new("msgfmt");
            msgfmt
                .arg(format!(
                    "--output-file={}",
                    mo_path.join("media-toc.mo").to_str().unwrap()
                ))
                .arg("--directory=po")
                .arg(format!("{}.po", lingua));

            match msgfmt.status() {
                Ok(status) => if !status.success() {
                    panic!(format!(
                        "Failed to generate mo file for lingua {}\n{:?}",
                        lingua, msgfmt,
                    ));
                },
                Err(ref error) => match error.kind() {
                    ErrorKind::NotFound => {
                        eprintln!("Can't generate translations: command `msgfmt` not available");
                        return;
                    }
                    _ => panic!("Error invoking `msgfmt`: {}", error),
                },
            }
        }
    }
}

// FIXME: figure out macOS conventions for icons & translations
#[cfg(target_family = "unix")]
fn generate_install_script() {
    let base_dirs = BaseDirs::new();
    // Note: `base_dirs.executable_dir()` is `None` on macOS
    if let Some(exe_dir) = base_dirs.executable_dir() {
        let project_dirs = ProjectDirs::from("org", "fengalin", env!("CARGO_PKG_NAME"));
        let app_data_dir = project_dirs.data_dir();
        let data_dir = app_data_dir.parent().unwrap();

        match File::create(&PathBuf::from("target").join("install")) {
            Ok(mut install_file) => {
                install_file
                    .write_all(
                        format!("# User install script for {}\n", env!("CARGO_PKG_NAME"))
                            .as_bytes(),
                    )
                    .unwrap();

                install_file.write_all(b"\n# Install executable\n").unwrap();
                let exe_source_path = PathBuf::from("target")
                    .join("release")
                    .join(env!("CARGO_PKG_NAME"));
                install_file
                    .write_all(format!("mkdir -p {}\n", exe_dir.to_str().unwrap()).as_bytes())
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp {} {}\n",
                            exe_source_path.to_str().unwrap(),
                            exe_dir.join(env!("CARGO_PKG_NAME")).to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();

                install_file.write_all(b"\n# Install icons\n").unwrap();
                let icon_target_dir = data_dir.join("icons").join("hicolor");
                let mut entry_iter =
                    read_dir(PathBuf::from("assets").join("icons").join("hicolor")).unwrap();
                for entry in entry_iter {
                    let entry = entry.unwrap();
                    let entry_path = entry.path();
                    if entry_path.is_dir() {
                        let target_dir = icon_target_dir.join(&entry.file_name());
                        install_file
                            .write_all(
                                format!("mkdir -p {}\n", target_dir.to_str().unwrap()).as_bytes(),
                            )
                            .unwrap();

                        install_file
                            .write_all(
                                format!(
                                    "cp -r {:?}/* {:?}\n",
                                    entry_path.to_str().unwrap(),
                                    target_dir,
                                ).as_bytes(),
                            )
                            .unwrap();
                    }
                }

                install_file
                    .write_all(b"\n# Install translations\n")
                    .unwrap();
                install_file
                    .write_all(format!("mkdir -p {}\n", data_dir.to_str().unwrap()).as_bytes())
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp -r {} {}\n",
                            PathBuf::from("target").join("locale").to_str().unwrap(),
                            data_dir.to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();

                install_file
                    .write_all(b"\n# Install desktop file\n")
                    .unwrap();
                let desktop_target_dir = data_dir.join("applications");
                install_file
                    .write_all(
                        format!("mkdir -p {}\n", desktop_target_dir.to_str().unwrap()).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "cp {} {}\n",
                            PathBuf::from("assets")
                                .join(&format!("org.fengalin.{}.desktop", env!("CARGO_PKG_NAME")))
                                .to_str()
                                .unwrap(),
                            desktop_target_dir.to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
            }
            Err(err) => panic!("Couldn't create file `target/install`: {:?}", err),
        }
    }
}

// FIXME: figure out macOS conventions for icons & translations
#[cfg(target_family = "unix")]
fn generate_uninstall_script() {
    let base_dirs = BaseDirs::new();
    // Note: `base_dirs.executable_dir()` is `None` on macOS
    if let Some(exe_dir) = base_dirs.executable_dir() {
        let project_dirs = ProjectDirs::from("org", "fengalin", env!("CARGO_PKG_NAME"));
        let app_data_dir = project_dirs.data_dir();
        let data_dir = app_data_dir.parent().unwrap();

        match File::create(&PathBuf::from("target").join("uninstall")) {
            Ok(mut install_file) => {
                install_file
                    .write_all(
                        format!("# User uninstall script for {}\n", env!("CARGO_PKG_NAME"))
                            .as_bytes(),
                    )
                    .unwrap();

                install_file
                    .write_all(b"\n# Uninstall executable\n")
                    .unwrap();
                install_file
                    .write_all(
                        format!(
                            "rm {}\n",
                            exe_dir.join(env!("CARGO_PKG_NAME")).to_str().unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(format!("rmdir -p {}\n", exe_dir.to_str().unwrap()).as_bytes())
                    .unwrap();

                install_file.write_all(b"\n# Uninstall icons\n").unwrap();
                let icon_target_dir = data_dir.join("icons").join("hicolor");
                let mut entry_iter =
                    read_dir(PathBuf::from("assets").join("icons").join("hicolor")).unwrap();
                for entry in entry_iter {
                    let entry = entry.unwrap();
                    let entry_path = entry.path();
                    if entry_path.is_dir() {
                        let icon_target_dir = icon_target_dir
                            .join(entry_path.file_name().unwrap())
                            .join("apps");
                        let mut icons_subdir_iter = read_dir(entry_path.join("apps")).unwrap();
                        for icon_subdir_entry in icons_subdir_iter {
                            let icon_subdir_entry = icon_subdir_entry.unwrap();
                            install_file
                                .write_all(
                                    format!(
                                        "rm {}\n",
                                        icon_target_dir
                                            .join(icon_subdir_entry.file_name())
                                            .to_str()
                                            .unwrap(),
                                    ).as_bytes(),
                                )
                                .unwrap();
                        }

                        install_file
                            .write_all(
                                format!("rmdir -p {}\n", icon_target_dir.to_str().unwrap())
                                    .as_bytes(),
                            )
                            .unwrap();
                    }
                }

                if let Ok(mut linguas_file) = File::open(&PathBuf::from("po").join("LINGUAS")) {
                    let mut linguas = String::new();
                    linguas_file
                        .read_to_string(&mut linguas)
                        .expect("Couldn't read po/LINGUAS as string");

                    install_file
                        .write_all(b"\n# Uninstall translations\n")
                        .unwrap();
                    let locale_base_dir = data_dir.join("locale");
                    for lingua in linguas.lines() {
                        let lingua_dir = locale_base_dir.join(lingua).join("LC_MESSAGES");
                        install_file
                            .write_all(
                                format!(
                                    "rm {}\n",
                                    lingua_dir
                                        .join(&format!("{}.mo", env!("CARGO_PKG_NAME")))
                                        .to_str()
                                        .unwrap(),
                                ).as_bytes(),
                            )
                            .unwrap();

                        install_file
                            .write_all(
                                format!("rmdir -p {}\n", lingua_dir.to_str().unwrap()).as_bytes(),
                            )
                            .unwrap();
                    }
                }

                install_file
                    .write_all(b"\n# Uninstall desktop file\n")
                    .unwrap();
                let desktop_target_dir = data_dir.join("applications");
                install_file
                    .write_all(
                        format!(
                            "rm {}\n",
                            desktop_target_dir
                                .join(&format!("org.fengalin.{}.desktop", env!("CARGO_PKG_NAME")))
                                .to_str()
                                .unwrap(),
                        ).as_bytes(),
                    )
                    .unwrap();
                install_file
                    .write_all(
                        format!("rmdir -p {}\n", desktop_target_dir.to_str().unwrap()).as_bytes(),
                    )
                    .unwrap();
            }
            Err(err) => panic!("Couldn't create file `target/uninstall`: {:?}", err),
        }
    }
}

fn main() {
    generate_resources();
    generate_translations();

    #[cfg(target_family = "unix")]
    generate_install_script();

    #[cfg(target_family = "unix")]
    generate_uninstall_script();
}
<|endoftext|>


<commit_before>#![feature(use_extern_macros)]

extern crate error_chain;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::{Command, Stdio};

use error_chain::{bail, quick_main};

mod errors {
    use std::io;

    use error_chain::*;

    error_chain! {
        foreign_links {
            Io(io::Error);
        }
    }
}

use errors::*;

fn run() -> Result<()> {
    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());

    let status = Command::new("cp")
        .args(&["-r", "static", out_dir.to_str().unwrap()])
        .stderr(Stdio::inherit())
        .status()
        .chain_err(|| "failed to run `cp`")?;
    if !status.success() {
        bail!("failed to copy static files to the target directory");
    }

    let scss_out = out_dir.join("styles.css");
    let status = Command::new("sassc")
        .args(&["scss/main.scss", scss_out.to_str().unwrap()])
        .stderr(Stdio::inherit())
        .status()
        .chain_err(|| "failed to run `sassc`")?;
    if !status.success() {
        bail!("SCSS compilation failed");
    }

    let css_out = out_dir.join("static/css/");
    fs::create_dir_all(&css_out)?;
    let status = Command::new("postcss")
        .args(&["--use", "autoprefixer"])
        .arg(scss_out.to_str().unwrap())
        .args(&["-d", css_out.to_str().unwrap()])
        .status()
        .chain_err(|| "failed to run `postcss`")?;
    if !status.success() {
        bail!("failed postprocessing CSS");
    }

    Ok(())
}

quick_main!(run);
<commit_msg>fix(build): switch order of postcss args<commit_after>#![feature(use_extern_macros)]

extern crate error_chain;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::{Command, Stdio};

use error_chain::{bail, quick_main};

mod errors {
    use std::io;

    use error_chain::*;

    error_chain! {
        foreign_links {
            Io(io::Error);
        }
    }
}

use errors::*;

fn run() -> Result<()> {
    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());

    let status = Command::new("cp")
        .args(&["-r", "static", out_dir.to_str().unwrap()])
        .stderr(Stdio::inherit())
        .status()
        .chain_err(|| "failed to run `cp`")?;
    if !status.success() {
        bail!("failed to copy static files to the target directory");
    }

    let scss_out = out_dir.join("styles.css");
    let status = Command::new("sassc")
        .args(&["scss/main.scss", scss_out.to_str().unwrap()])
        .stderr(Stdio::inherit())
        .status()
        .chain_err(|| "failed to run `sassc`")?;
    if !status.success() {
        bail!("SCSS compilation failed");
    }

    let css_out = out_dir.join("static/css/");
    fs::create_dir_all(&css_out)?;
    let status = Command::new("postcss")
        .args(&["--use", "autoprefixer"])
        .args(&["-d", css_out.to_str().unwrap()])
        .arg(scss_out.to_str().unwrap())
        .status()
        .chain_err(|| "failed to run `postcss`")?;
    if !status.success() {
        bail!("failed postprocessing CSS");
    }

    Ok(())
}

quick_main!(run);
<|endoftext|>


<commit_before>use std::collections::BTreeMap;
use std::iter::Iterator;
use std::vec::Vec;

use rand::{seq::SliceRandom, thread_rng, Rng};
use test::{black_box, Bencher};

macro_rules! map_insert_rand_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let n: usize = $n;
            let mut map = $map::new();
            // setup
            let mut rng = thread_rng();

            for _ in 0..n {
                let i = rng.gen::<usize>() % n;
                map.insert(i, i);
            }

            // measure
            b.iter(|| {
                let k = rng.gen::<usize>() % n;
                map.insert(k, k);
                map.remove(&k);
            });
            black_box(map);
        }
    };
}

macro_rules! map_insert_seq_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;
            // setup
            for i in 0..n {
                map.insert(i * 2, i * 2);
            }

            // measure
            let mut i = 1;
            b.iter(|| {
                map.insert(i, i);
                map.remove(&i);
                i = (i + 2) % n;
            });
            black_box(map);
        }
    };
}

macro_rules! map_find_rand_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;

            // setup
            let mut rng = thread_rng();
            let mut keys: Vec<_> = (0..n).map(|_| rng.gen::<usize>() % n).collect();

            for &k in &keys {
                map.insert(k, k);
            }

            keys.shuffle(&mut rng);

            // measure
            let mut i = 0;
            b.iter(|| {
                let t = map.get(&keys[i]);
                i = (i + 1) % n;
                black_box(t);
            })
        }
    };
}

macro_rules! map_find_seq_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;

            // setup
            for i in 0..n {
                map.insert(i, i);
            }

            // measure
            let mut i = 0;
            b.iter(|| {
                let x = map.get(&i);
                i = (i + 1) % n;
                black_box(x);
            })
        }
    };
}

map_insert_rand_bench! {insert_rand_100,    100,    BTreeMap}
map_insert_rand_bench! {insert_rand_10_000, 10_000, BTreeMap}

map_insert_seq_bench! {insert_seq_100,    100,    BTreeMap}
map_insert_seq_bench! {insert_seq_10_000, 10_000, BTreeMap}

map_find_rand_bench! {find_rand_100,    100,    BTreeMap}
map_find_rand_bench! {find_rand_10_000, 10_000, BTreeMap}

map_find_seq_bench! {find_seq_100,    100,    BTreeMap}
map_find_seq_bench! {find_seq_10_000, 10_000, BTreeMap}

fn bench_iter(b: &mut Bencher, size: i32) {
    let mut map = BTreeMap::<i32, i32>::new();
    let mut rng = thread_rng();

    for _ in 0..size {
        map.insert(rng.gen(), rng.gen());
    }

    b.iter(|| {
        for entry in &map {
            black_box(entry);
        }
    });
}

#[bench]
pub fn iter_20(b: &mut Bencher) {
    bench_iter(b, 20);
}

#[bench]
pub fn iter_1000(b: &mut Bencher) {
    bench_iter(b, 1000);
}

#[bench]
pub fn iter_100000(b: &mut Bencher) {
    bench_iter(b, 100000);
}

fn bench_iter_mut(b: &mut Bencher, size: i32) {
    let mut map = BTreeMap::<i32, i32>::new();
    let mut rng = thread_rng();

    for _ in 0..size {
        map.insert(rng.gen(), rng.gen());
    }

    b.iter(|| {
        for kv in map.iter_mut() {
            black_box(kv);
        }
    });
}

#[bench]
pub fn iter_mut_20(b: &mut Bencher) {
    bench_iter_mut(b, 20);
}

#[bench]
pub fn iter_mut_1000(b: &mut Bencher) {
    bench_iter_mut(b, 1000);
}

#[bench]
pub fn iter_mut_100000(b: &mut Bencher) {
    bench_iter_mut(b, 100000);
}

fn bench_first_and_last(b: &mut Bencher, size: i32) {
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for _ in 0..10 {
            black_box(map.first_key_value());
            black_box(map.last_key_value());
        }
    });
}

#[bench]
pub fn first_and_last_0(b: &mut Bencher) {
    bench_first_and_last(b, 0);
}

#[bench]
pub fn first_and_last_100(b: &mut Bencher) {
    bench_first_and_last(b, 100);
}

#[bench]
pub fn first_and_last_10k(b: &mut Bencher) {
    bench_first_and_last(b, 10_000);
}
<commit_msg>Create benchmarks for BTreeMap::range<commit_after>use std::collections::BTreeMap;
use std::iter::Iterator;
use std::ops::Bound::{Excluded, Unbounded};
use std::vec::Vec;

use rand::{seq::SliceRandom, thread_rng, Rng};
use test::{black_box, Bencher};

macro_rules! map_insert_rand_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let n: usize = $n;
            let mut map = $map::new();
            // setup
            let mut rng = thread_rng();

            for _ in 0..n {
                let i = rng.gen::<usize>() % n;
                map.insert(i, i);
            }

            // measure
            b.iter(|| {
                let k = rng.gen::<usize>() % n;
                map.insert(k, k);
                map.remove(&k);
            });
            black_box(map);
        }
    };
}

macro_rules! map_insert_seq_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;
            // setup
            for i in 0..n {
                map.insert(i * 2, i * 2);
            }

            // measure
            let mut i = 1;
            b.iter(|| {
                map.insert(i, i);
                map.remove(&i);
                i = (i + 2) % n;
            });
            black_box(map);
        }
    };
}

macro_rules! map_find_rand_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;

            // setup
            let mut rng = thread_rng();
            let mut keys: Vec<_> = (0..n).map(|_| rng.gen::<usize>() % n).collect();

            for &k in &keys {
                map.insert(k, k);
            }

            keys.shuffle(&mut rng);

            // measure
            let mut i = 0;
            b.iter(|| {
                let t = map.get(&keys[i]);
                i = (i + 1) % n;
                black_box(t);
            })
        }
    };
}

macro_rules! map_find_seq_bench {
    ($name: ident, $n: expr, $map: ident) => {
        #[bench]
        pub fn $name(b: &mut Bencher) {
            let mut map = $map::new();
            let n: usize = $n;

            // setup
            for i in 0..n {
                map.insert(i, i);
            }

            // measure
            let mut i = 0;
            b.iter(|| {
                let x = map.get(&i);
                i = (i + 1) % n;
                black_box(x);
            })
        }
    };
}

map_insert_rand_bench! {insert_rand_100,    100,    BTreeMap}
map_insert_rand_bench! {insert_rand_10_000, 10_000, BTreeMap}

map_insert_seq_bench! {insert_seq_100,    100,    BTreeMap}
map_insert_seq_bench! {insert_seq_10_000, 10_000, BTreeMap}

map_find_rand_bench! {find_rand_100,    100,    BTreeMap}
map_find_rand_bench! {find_rand_10_000, 10_000, BTreeMap}

map_find_seq_bench! {find_seq_100,    100,    BTreeMap}
map_find_seq_bench! {find_seq_10_000, 10_000, BTreeMap}

fn bench_iter(b: &mut Bencher, size: i32) {
    let mut map = BTreeMap::<i32, i32>::new();
    let mut rng = thread_rng();

    for _ in 0..size {
        map.insert(rng.gen(), rng.gen());
    }

    b.iter(|| {
        for entry in &map {
            black_box(entry);
        }
    });
}

#[bench]
pub fn iter_20(b: &mut Bencher) {
    bench_iter(b, 20);
}

#[bench]
pub fn iter_1000(b: &mut Bencher) {
    bench_iter(b, 1000);
}

#[bench]
pub fn iter_100000(b: &mut Bencher) {
    bench_iter(b, 100000);
}

fn bench_iter_mut(b: &mut Bencher, size: i32) {
    let mut map = BTreeMap::<i32, i32>::new();
    let mut rng = thread_rng();

    for _ in 0..size {
        map.insert(rng.gen(), rng.gen());
    }

    b.iter(|| {
        for kv in map.iter_mut() {
            black_box(kv);
        }
    });
}

#[bench]
pub fn iter_mut_20(b: &mut Bencher) {
    bench_iter_mut(b, 20);
}

#[bench]
pub fn iter_mut_1000(b: &mut Bencher) {
    bench_iter_mut(b, 1000);
}

#[bench]
pub fn iter_mut_100000(b: &mut Bencher) {
    bench_iter_mut(b, 100000);
}

fn bench_first_and_last(b: &mut Bencher, size: i32) {
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for _ in 0..10 {
            black_box(map.first_key_value());
            black_box(map.last_key_value());
        }
    });
}

#[bench]
pub fn first_and_last_0(b: &mut Bencher) {
    bench_first_and_last(b, 0);
}

#[bench]
pub fn first_and_last_100(b: &mut Bencher) {
    bench_first_and_last(b, 100);
}

#[bench]
pub fn first_and_last_10k(b: &mut Bencher) {
    bench_first_and_last(b, 10_000);
}

#[bench]
pub fn range_excluded_excluded(b: &mut Bencher) {
    let size = 144;
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for first in 0..size {
            for last in first + 1..size {
                black_box(map.range((Excluded(first), Excluded(last))));
            }
        }
    });
}

#[bench]
pub fn range_excluded_unbounded(b: &mut Bencher) {
    let size = 144;
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for first in 0..size {
            black_box(map.range((Excluded(first), Unbounded)));
        }
    });
}

#[bench]
pub fn range_included_included(b: &mut Bencher) {
    let size = 144;
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for first in 0..size {
            for last in first..size {
                black_box(map.range(first..=last));
            }
        }
    });
}

#[bench]
pub fn range_included_unbounded(b: &mut Bencher) {
    let size = 144;
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| {
        for first in 0..size {
            black_box(map.range(first..));
        }
    });
}

#[bench]
pub fn range_unbounded_unbounded(b: &mut Bencher) {
    let size = 144;
    let map: BTreeMap<_, _> = (0..size).map(|i| (i, i)).collect();
    b.iter(|| map.range(..));
}
<|endoftext|>


<commit_before>// Copyright 2014 Pierre Talbot (IRCAM)

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

pub use syntax::ext::base::ExtCtxt;
pub use syntax::codemap::Span;

pub enum CompileErrorLevel
{
  Silent,
  Warn,
  Error
}

impl CompileErrorLevel
{
  pub fn issue<'a>(&self, cx: &'a ExtCtxt, span: Span, msg: &str)
  {
    match *self {
      Silent => (),
      Warn => cx.span_warn(span, msg),
      Error => cx.span_err(span, msg)
    }
  }

  pub fn is_silent(&self) -> bool
  {
    match *self {
      Silent => true,
      _ => false
    }
  }

  pub fn is_error(&self) -> bool
  {
    match *self {
      Error => true,
      _ => false
    }
  }
}

pub struct DuplicateAttribute
{
  level: CompileErrorLevel,
  extra_msg: Option<&'static str>
}

impl DuplicateAttribute
{
  pub fn new(level: CompileErrorLevel, extra_msg: Option<&'static str>)
    -> DuplicateAttribute
  {
    DuplicateAttribute {
      level: level,
      extra_msg: extra_msg
    }
  }

  pub fn simple(level: CompileErrorLevel) -> DuplicateAttribute
  {
    DuplicateAttribute::new(level, None)
  }

  pub fn issue<'a>(&self, cx: &'a ExtCtxt, span: Span, previous_span: Span) -> bool
  {
    let extra_msg = self.extra_msg.unwrap_or("");
    self.level.issue(cx, span,
      format!("Duplicate attribute. {}", extra_msg).as_slice());
    if !self.level.is_silent() {
      cx.span_note(previous_span,
        "Previous declaration here.");
    }
    self.level.is_error()
  }
}
<commit_msg>[DuplicateAttribute] Add a constructor for error duplicate.<commit_after>// Copyright 2014 Pierre Talbot (IRCAM)

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

pub use syntax::ext::base::ExtCtxt;
pub use syntax::codemap::Span;

pub enum CompileErrorLevel
{
  Silent,
  Warn,
  Error
}

impl CompileErrorLevel
{
  pub fn issue<'a>(&self, cx: &'a ExtCtxt, span: Span, msg: &str)
  {
    match *self {
      Silent => (),
      Warn => cx.span_warn(span, msg),
      Error => cx.span_err(span, msg)
    }
  }

  pub fn is_silent(&self) -> bool
  {
    match *self {
      Silent => true,
      _ => false
    }
  }

  pub fn is_error(&self) -> bool
  {
    match *self {
      Error => true,
      _ => false
    }
  }
}

pub struct DuplicateAttribute
{
  level: CompileErrorLevel,
  extra_msg: Option<&'static str>
}

impl DuplicateAttribute
{
  pub fn new(level: CompileErrorLevel, extra_msg: Option<&'static str>)
    -> DuplicateAttribute
  {
    DuplicateAttribute {
      level: level,
      extra_msg: extra_msg
    }
  }

  pub fn simple(level: CompileErrorLevel) -> DuplicateAttribute
  {
    DuplicateAttribute::new(level, None)
  }

  pub fn error(extra_msg: &'static str) -> DuplicateAttribute
  {
    DuplicateAttribute::new(Error, Some(extra_msg))
  }

  pub fn issue<'a>(&self, cx: &'a ExtCtxt, span: Span, previous_span: Span) -> bool
  {
    let extra_msg = self.extra_msg.unwrap_or("");
    self.level.issue(cx, span,
      format!("Duplicate attribute. {}", extra_msg).as_slice());
    if !self.level.is_silent() {
      cx.span_note(previous_span,
        "Previous declaration here.");
    }
    self.level.is_error()
  }
}
<|endoftext|>


<commit_before>use rustc_hir::def_id::DefId;
use rustc_middle::middle::privacy::AccessLevels;
use std::mem;

use crate::clean::{self, GetDefId, Item, ItemIdSet};
use crate::fold::{strip_item, DocFolder};

crate struct Stripper<'a> {
    crate retained: &'a mut ItemIdSet,
    crate access_levels: &'a AccessLevels<DefId>,
    crate update_retained: bool,
}

impl<'a> DocFolder for Stripper<'a> {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        match *i.kind {
            clean::StrippedItem(..) => {
                // We need to recurse into stripped modules to strip things
                // like impl methods but when doing so we must not add any
                // items to the `retained` set.
                debug!("Stripper: recursing into stripped {:?} {:?}", i.type_(), i.name);
                let old = mem::replace(&mut self.update_retained, false);
                let ret = self.fold_item_recur(i);
                self.update_retained = old;
                return Some(ret);
            }
            // These items can all get re-exported
            clean::OpaqueTyItem(..)
            | clean::TypedefItem(..)
            | clean::StaticItem(..)
            | clean::StructItem(..)
            | clean::EnumItem(..)
            | clean::TraitItem(..)
            | clean::FunctionItem(..)
            | clean::VariantItem(..)
            | clean::MethodItem(..)
            | clean::ForeignFunctionItem(..)
            | clean::ForeignStaticItem(..)
            | clean::ConstantItem(..)
            | clean::UnionItem(..)
            | clean::AssocConstItem(..)
            | clean::TraitAliasItem(..)
            | clean::ForeignTypeItem => {
                if i.def_id.is_local() {
                    if !self.access_levels.is_exported(i.def_id.expect_def_id()) {
                        debug!("Stripper: stripping {:?} {:?}", i.type_(), i.name);
                        return None;
                    }
                }
            }

            clean::StructFieldItem(..) => {
                if !i.visibility.is_public() {
                    return Some(strip_item(i));
                }
            }

            clean::ModuleItem(..) => {
                if i.def_id.is_local() && !i.visibility.is_public() {
                    debug!("Stripper: stripping module {:?}", i.name);
                    let old = mem::replace(&mut self.update_retained, false);
                    let ret = strip_item(self.fold_item_recur(i));
                    self.update_retained = old;
                    return Some(ret);
                }
            }

            // handled in the `strip-priv-imports` pass
            clean::ExternCrateItem { .. } | clean::ImportItem(..) => {}

            clean::ImplItem(..) => {}

            // tymethods/macros have no control over privacy
            clean::MacroItem(..) | clean::TyMethodItem(..) => {}

            // Proc-macros are always public
            clean::ProcMacroItem(..) => {}

            // Primitives are never stripped
            clean::PrimitiveItem(..) => {}

            // Associated types are never stripped
            clean::AssocTypeItem(..) => {}

            // Keywords are never stripped
            clean::KeywordItem(..) => {}
        }

        let fastreturn = match *i.kind {
            // nothing left to do for traits (don't want to filter their
            // methods out, visibility controlled by the trait)
            clean::TraitItem(..) => true,

            // implementations of traits are always public.
            clean::ImplItem(ref imp) if imp.trait_.is_some() => true,
            // Struct variant fields have inherited visibility
            clean::VariantItem(clean::Variant::Struct(..)) => true,
            _ => false,
        };

        let i = if fastreturn {
            if self.update_retained {
                self.retained.insert(i.def_id);
            }
            return Some(i);
        } else {
            self.fold_item_recur(i)
        };

        if self.update_retained {
            self.retained.insert(i.def_id);
        }
        Some(i)
    }
}

/// This stripper discards all impls which reference stripped items
crate struct ImplStripper<'a> {
    crate retained: &'a ItemIdSet,
}

impl<'a> DocFolder for ImplStripper<'a> {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        if let clean::ImplItem(ref imp) = *i.kind {
            // emptied none trait impls can be stripped
            if imp.trait_.is_none() && imp.items.is_empty() {
                return None;
            }
            if let Some(did) = imp.for_.def_id() {
                if did.is_local() && !imp.for_.is_generic() && !self.retained.contains(&did.into())
                {
                    debug!("ImplStripper: impl item for stripped type; removing");
                    return None;
                }
            }
            if let Some(did) = imp.trait_.def_id() {
                if did.is_local() && !self.retained.contains(&did.into()) {
                    debug!("ImplStripper: impl item for stripped trait; removing");
                    return None;
                }
            }
            if let Some(generics) = imp.trait_.as_ref().and_then(|t| t.generics()) {
                for typaram in generics {
                    if let Some(did) = typaram.def_id() {
                        if did.is_local() && !self.retained.contains(&did.into()) {
                            debug!(
                                "ImplStripper: stripped item in trait's generics; removing impl"
                            );
                            return None;
                        }
                    }
                }
            }
        }
        Some(self.fold_item_recur(i))
    }
}

/// This stripper discards all private import statements (`use`, `extern crate`)
crate struct ImportStripper;

impl DocFolder for ImportStripper {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        match *i.kind {
            clean::ExternCrateItem { .. } | clean::ImportItem(..) if !i.visibility.is_public() => {
                None
            }
            _ => Some(self.fold_item_recur(i)),
        }
    }
}
<commit_msg>Fix: don't document private macros by default<commit_after>use rustc_hir::def_id::DefId;
use rustc_middle::middle::privacy::AccessLevels;
use std::mem;

use crate::clean::{self, GetDefId, Item, ItemIdSet};
use crate::fold::{strip_item, DocFolder};

crate struct Stripper<'a> {
    crate retained: &'a mut ItemIdSet,
    crate access_levels: &'a AccessLevels<DefId>,
    crate update_retained: bool,
}

impl<'a> DocFolder for Stripper<'a> {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        match *i.kind {
            clean::StrippedItem(..) => {
                // We need to recurse into stripped modules to strip things
                // like impl methods but when doing so we must not add any
                // items to the `retained` set.
                debug!("Stripper: recursing into stripped {:?} {:?}", i.type_(), i.name);
                let old = mem::replace(&mut self.update_retained, false);
                let ret = self.fold_item_recur(i);
                self.update_retained = old;
                return Some(ret);
            }
            // These items can all get re-exported
            clean::OpaqueTyItem(..)
            | clean::TypedefItem(..)
            | clean::StaticItem(..)
            | clean::StructItem(..)
            | clean::EnumItem(..)
            | clean::TraitItem(..)
            | clean::FunctionItem(..)
            | clean::VariantItem(..)
            | clean::MethodItem(..)
            | clean::ForeignFunctionItem(..)
            | clean::ForeignStaticItem(..)
            | clean::ConstantItem(..)
            | clean::UnionItem(..)
            | clean::AssocConstItem(..)
            | clean::TraitAliasItem(..)
            | clean::MacroItem(..)
            | clean::ForeignTypeItem => {
                if i.def_id.is_local() {
                    if !self.access_levels.is_exported(i.def_id.expect_def_id()) {
                        debug!("Stripper: stripping {:?} {:?}", i.type_(), i.name);
                        return None;
                    }
                }
            }

            clean::StructFieldItem(..) => {
                if !i.visibility.is_public() {
                    return Some(strip_item(i));
                }
            }

            clean::ModuleItem(..) => {
                if i.def_id.is_local() && !i.visibility.is_public() {
                    debug!("Stripper: stripping module {:?}", i.name);
                    let old = mem::replace(&mut self.update_retained, false);
                    let ret = strip_item(self.fold_item_recur(i));
                    self.update_retained = old;
                    return Some(ret);
                }
            }

            // handled in the `strip-priv-imports` pass
            clean::ExternCrateItem { .. } | clean::ImportItem(..) => {}

            clean::ImplItem(..) => {}

            // tymethods have no control over privacy
            clean::TyMethodItem(..) => {}

            // Proc-macros are always public
            clean::ProcMacroItem(..) => {}

            // Primitives are never stripped
            clean::PrimitiveItem(..) => {}

            // Associated types are never stripped
            clean::AssocTypeItem(..) => {}

            // Keywords are never stripped
            clean::KeywordItem(..) => {}
        }

        let fastreturn = match *i.kind {
            // nothing left to do for traits (don't want to filter their
            // methods out, visibility controlled by the trait)
            clean::TraitItem(..) => true,

            // implementations of traits are always public.
            clean::ImplItem(ref imp) if imp.trait_.is_some() => true,
            // Struct variant fields have inherited visibility
            clean::VariantItem(clean::Variant::Struct(..)) => true,
            _ => false,
        };

        let i = if fastreturn {
            if self.update_retained {
                self.retained.insert(i.def_id);
            }
            return Some(i);
        } else {
            self.fold_item_recur(i)
        };

        if self.update_retained {
            self.retained.insert(i.def_id);
        }
        Some(i)
    }
}

/// This stripper discards all impls which reference stripped items
crate struct ImplStripper<'a> {
    crate retained: &'a ItemIdSet,
}

impl<'a> DocFolder for ImplStripper<'a> {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        if let clean::ImplItem(ref imp) = *i.kind {
            // emptied none trait impls can be stripped
            if imp.trait_.is_none() && imp.items.is_empty() {
                return None;
            }
            if let Some(did) = imp.for_.def_id() {
                if did.is_local() && !imp.for_.is_generic() && !self.retained.contains(&did.into())
                {
                    debug!("ImplStripper: impl item for stripped type; removing");
                    return None;
                }
            }
            if let Some(did) = imp.trait_.def_id() {
                if did.is_local() && !self.retained.contains(&did.into()) {
                    debug!("ImplStripper: impl item for stripped trait; removing");
                    return None;
                }
            }
            if let Some(generics) = imp.trait_.as_ref().and_then(|t| t.generics()) {
                for typaram in generics {
                    if let Some(did) = typaram.def_id() {
                        if did.is_local() && !self.retained.contains(&did.into()) {
                            debug!(
                                "ImplStripper: stripped item in trait's generics; removing impl"
                            );
                            return None;
                        }
                    }
                }
            }
        }
        Some(self.fold_item_recur(i))
    }
}

/// This stripper discards all private import statements (`use`, `extern crate`)
crate struct ImportStripper;

impl DocFolder for ImportStripper {
    fn fold_item(&mut self, i: Item) -> Option<Item> {
        match *i.kind {
            clean::ExternCrateItem { .. } | clean::ImportItem(..) if !i.visibility.is_public() => {
                None
            }
            _ => Some(self.fold_item_recur(i)),
        }
    }
}
<|endoftext|>


<commit_before>use crate::boxed::Box;

#[rustc_specialization_trait]
pub(super) unsafe trait IsZero {
    /// Whether this value is zero
    fn is_zero(&self) -> bool;
}

macro_rules! impl_is_zero {
    ($t:ty, $is_zero:expr) => {
        unsafe impl IsZero for $t {
            #[inline]
            fn is_zero(&self) -> bool {
                $is_zero(*self)
            }
        }
    };
}

impl_is_zero!(i16, |x| x == 0);
impl_is_zero!(i32, |x| x == 0);
impl_is_zero!(i64, |x| x == 0);
impl_is_zero!(i128, |x| x == 0);
impl_is_zero!(isize, |x| x == 0);

impl_is_zero!(u16, |x| x == 0);
impl_is_zero!(u32, |x| x == 0);
impl_is_zero!(u64, |x| x == 0);
impl_is_zero!(u128, |x| x == 0);
impl_is_zero!(usize, |x| x == 0);

impl_is_zero!(bool, |x| x == false);
impl_is_zero!(char, |x| x == '\0');

impl_is_zero!(f32, |x: f32| x.to_bits() == 0);
impl_is_zero!(f64, |x: f64| x.to_bits() == 0);

unsafe impl<T> IsZero for *const T {
    #[inline]
    fn is_zero(&self) -> bool {
        (*self).is_null()
    }
}

unsafe impl<T> IsZero for *mut T {
    #[inline]
    fn is_zero(&self) -> bool {
        (*self).is_null()
    }
}

// `Option<&T>` and `Option<Box<T>>` are guaranteed to represent `None` as null.
// For fat pointers, the bytes that would be the pointer metadata in the `Some`
// variant are padding in the `None` variant, so ignoring them and
// zero-initializing instead is ok.
// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of
// `SpecFromElem`.

unsafe impl<T: ?Sized> IsZero for Option<&T> {
    #[inline]
    fn is_zero(&self) -> bool {
        self.is_none()
    }
}

unsafe impl<T: ?Sized> IsZero for Option<Box<T>> {
    #[inline]
    fn is_zero(&self) -> bool {
        self.is_none()
    }
}

// `Option<num::NonZeroU32>` and similar have a representation guarantee that
// they're the same size as the corresponding `u32` type, as well as a guarantee
// that transmuting between `NonZeroU32` and `Option<num::NonZeroU32>` works.
// While the documentation officially makes in UB to transmute from `None`,
// we're the standard library so we can make extra inferences, and we know that
// the only niche available to represent `None` is the one that's all zeros.

macro_rules! impl_is_zero_option_of_nonzero {
    ($($t:ident,)+) => {$(
        unsafe impl IsZero for Option<core::num::$t> {
            #[inline]
            fn is_zero(&self) -> bool {
                self.is_none()
            }
        }
    )+};
}

impl_is_zero_option_of_nonzero!(
    NonZeroU8,
    NonZeroU16,
    NonZeroU32,
    NonZeroU64,
    NonZeroU128,
    NonZeroI8,
    NonZeroI16,
    NonZeroI32,
    NonZeroI64,
    NonZeroI128,
    NonZeroUsize,
    NonZeroIsize,
);
<commit_msg>Rollup merge of #85817 - r00ster91:patch-9, r=dtolnay<commit_after>use crate::boxed::Box;

#[rustc_specialization_trait]
pub(super) unsafe trait IsZero {
    /// Whether this value is zero
    fn is_zero(&self) -> bool;
}

macro_rules! impl_is_zero {
    ($t:ty, $is_zero:expr) => {
        unsafe impl IsZero for $t {
            #[inline]
            fn is_zero(&self) -> bool {
                $is_zero(*self)
            }
        }
    };
}

impl_is_zero!(i16, |x| x == 0);
impl_is_zero!(i32, |x| x == 0);
impl_is_zero!(i64, |x| x == 0);
impl_is_zero!(i128, |x| x == 0);
impl_is_zero!(isize, |x| x == 0);

impl_is_zero!(u16, |x| x == 0);
impl_is_zero!(u32, |x| x == 0);
impl_is_zero!(u64, |x| x == 0);
impl_is_zero!(u128, |x| x == 0);
impl_is_zero!(usize, |x| x == 0);

impl_is_zero!(bool, |x| x == false);
impl_is_zero!(char, |x| x == '\0');

impl_is_zero!(f32, |x: f32| x.to_bits() == 0);
impl_is_zero!(f64, |x: f64| x.to_bits() == 0);

unsafe impl<T> IsZero for *const T {
    #[inline]
    fn is_zero(&self) -> bool {
        (*self).is_null()
    }
}

unsafe impl<T> IsZero for *mut T {
    #[inline]
    fn is_zero(&self) -> bool {
        (*self).is_null()
    }
}

// `Option<&T>` and `Option<Box<T>>` are guaranteed to represent `None` as null.
// For fat pointers, the bytes that would be the pointer metadata in the `Some`
// variant are padding in the `None` variant, so ignoring them and
// zero-initializing instead is ok.
// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of
// `SpecFromElem`.

unsafe impl<T: ?Sized> IsZero for Option<&T> {
    #[inline]
    fn is_zero(&self) -> bool {
        self.is_none()
    }
}

unsafe impl<T: ?Sized> IsZero for Option<Box<T>> {
    #[inline]
    fn is_zero(&self) -> bool {
        self.is_none()
    }
}

// `Option<num::NonZeroU32>` and similar have a representation guarantee that
// they're the same size as the corresponding `u32` type, as well as a guarantee
// that transmuting between `NonZeroU32` and `Option<num::NonZeroU32>` works.
// While the documentation officially makes it UB to transmute from `None`,
// we're the standard library so we can make extra inferences, and we know that
// the only niche available to represent `None` is the one that's all zeros.

macro_rules! impl_is_zero_option_of_nonzero {
    ($($t:ident,)+) => {$(
        unsafe impl IsZero for Option<core::num::$t> {
            #[inline]
            fn is_zero(&self) -> bool {
                self.is_none()
            }
        }
    )+};
}

impl_is_zero_option_of_nonzero!(
    NonZeroU8,
    NonZeroU16,
    NonZeroU32,
    NonZeroU64,
    NonZeroU128,
    NonZeroI8,
    NonZeroI16,
    NonZeroI32,
    NonZeroI64,
    NonZeroI128,
    NonZeroUsize,
    NonZeroIsize,
);
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    errors::*,
    outcome_array::OutcomeArray,
    scheduler::{Scheduler, SchedulerTask, TaskGuard, TxnIndex, Version},
    task::{ExecutionStatus, ExecutorTask, Transaction, TransactionOutput},
    txn_last_input_output::{ReadDescriptor, TxnLastInputOutput},
};
use aptos_infallible::Mutex;
use mvhashmap::MVHashMap;
use num_cpus;
use once_cell::sync::Lazy;
use rayon::prelude::*;
use std::{
    collections::HashSet,
    hash::Hash,
    marker::PhantomData,
    sync::{
        atomic::{AtomicUsize, Ordering},
        Arc,
    },
    thread::spawn,
};

static RAYON_EXEC_POOL: Lazy<rayon::ThreadPool> = Lazy::new(|| {
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_cpus::get())
        .build()
        .unwrap()
});

/// A struct that is always used by a single thread performing an execution task. The struct is
/// passed to the VM and acts as a proxy to resolve reads first in the shared multi-version
/// data-structure. It also allows the caller to track the read-set and any dependencies.
///
/// TODO(issue 10177): MvHashMapView currently needs to be sync due to trait bounds, but should
/// not be. In this case, the read_dependency member can have a RefCell<bool> type and the
/// captured_reads member can have RefCell<Vec<ReadDescriptor<K>>> type.
pub struct MVHashMapView<'a, K, V> {
    versioned_map: &'a MVHashMap<K, V>,
    txn_idx: TxnIndex,
    scheduler: &'a Scheduler,
    captured_reads: Mutex<Vec<ReadDescriptor<K>>>,
}

impl<'a, K: PartialOrd + Send + Clone + Hash + Eq, V: Send + Sync> MVHashMapView<'a, K, V> {
    /// Drains the captured reads.
    pub fn take_reads(&self) -> Vec<ReadDescriptor<K>> {
        let mut reads = self.captured_reads.lock();
        std::mem::take(&mut reads)
    }

    /// Captures a read from the VM execution.
    pub fn read(&self, key: &K) -> Option<Arc<V>> {
        loop {
            match self.versioned_map.read(key, self.txn_idx) {
                Ok((version, v)) => {
                    let (txn_idx, incarnation) = version;
                    self.captured_reads.lock().push(ReadDescriptor::from(
                        key.clone(),
                        txn_idx,
                        incarnation,
                    ));
                    return Some(v);
                }
                Err(None) => {
                    self.captured_reads
                        .lock()
                        .push(ReadDescriptor::from_storage(key.clone()));
                    return None;
                }
                Err(Some(dep_idx)) => {
                    // `self.txn_idx` estimated to depend on a write from `dep_idx`.
                    match self.scheduler.wait_for_dependency(self.txn_idx, dep_idx) {
                        Some(dep_condition) => {
                            // Wait on a condition variable correpsonding to the encountered
                            // read dependency. Once the dep_idx finishes re-execution, scheduler
                            // will mark the dependency as resolved, and then the txn_idx will be
                            // scheduled for re-execution, which will re-awaken cvar here.
                            // A deadlock is not possible due to these condition variables:
                            // suppose all threads are waiting on read dependency, and consider
                            // one with lowest txn_idx. It observed a dependency, so some thread
                            // aborted dep_idx. If that abort returned execution task, by
                            // minimality (lower transactions aren't waiting), that thread would
                            // finish execution unblock txn_idx, contradiction. Otherwise,
                            // execution_idx in scheduler was lower at a time when at least the
                            // thread that aborted dep_idx was alive, and again, since lower txns
                            // than txn_idx are not blocked, so the execution of dep_idx will
                            // eventually finish and lead to unblocking txn_idx, contradiction.
                            let (lock, cvar) = &*dep_condition;
                            let mut dep_resolved = lock.lock();
                            while !*dep_resolved {
                                dep_resolved = cvar.wait(dep_resolved).unwrap();
                            }
                        }
                        None => continue,
                    }
                }
            };
        }
    }

    /// Return txn_idx associated with the MVHashMapView
    pub fn txn_idx(&self) -> TxnIndex {
        self.txn_idx
    }
}

pub struct ParallelTransactionExecutor<T: Transaction, E: ExecutorTask> {
    // number of active concurrent tasks, corresponding to the maximum number of rayon
    // threads that may be concurrently participating in parallel execution.
    concurrency_level: usize,
    phantom: PhantomData<(T, E)>,
}

impl<T, E> ParallelTransactionExecutor<T, E>
where
    T: Transaction,
    E: ExecutorTask<T = T>,
{
    /// The caller needs to ensure that concurrency_level > 1 (0 is illegal and 1 should
    /// be handled by sequential execution) and that concurrency_level <= num_cpus.
    pub fn new(concurrency_level: usize) -> Self {
        assert!(
            concurrency_level > 1 && concurrency_level <= num_cpus::get(),
            "Parallel execution concurrency level {} should be between 2 and number of CPUs",
            concurrency_level
        );
        Self {
            concurrency_level,
            phantom: PhantomData,
        }
    }

    fn execute<'a>(
        &self,
        version: Version,
        guard: TaskGuard<'a>,
        signature_verified_block: &[T],
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &'a Scheduler,
        executor: &E,
    ) -> SchedulerTask<'a> {
        let (idx_to_execute, incarnation) = version;
        let txn = &signature_verified_block[idx_to_execute];

        let state_view = MVHashMapView {
            versioned_map: versioned_data_cache,
            txn_idx: idx_to_execute,
            scheduler,
            captured_reads: Mutex::new(Vec::new()),
        };

        // VM execution.
        let execute_result = executor.execute_transaction(&state_view, txn);
        let mut prev_write_set: HashSet<T::Key> = last_input_output.write_set(idx_to_execute);

        // For tracking whether the recent execution wrote outside of the previous write set.
        let mut writes_outside = false;
        let mut apply_writes = |output: &<E as ExecutorTask>::Output| {
            let write_version = (idx_to_execute, incarnation);
            for (k, v) in output.get_writes().into_iter() {
                if !prev_write_set.remove(&k) {
                    writes_outside = true
                }
                versioned_data_cache.write(&k, write_version, v);
            }
        };

        let result = match execute_result {
            // These statuses are the results of speculative execution, so even for
            // SkipRest (skip the rest of transactions) and Abort (abort execution with
            // user defined error), no immediate action is taken. Instead the statuses
            // are recorded and (final statuses) are analyzed when the block is executed.
            ExecutionStatus::Success(output) => {
                // Apply the writes to the versioned_data_cache.
                apply_writes(&output);
                ExecutionStatus::Success(output)
            }
            ExecutionStatus::SkipRest(output) => {
                // Apply the writes and record status indicating skip.
                apply_writes(&output);
                ExecutionStatus::SkipRest(output)
            }
            ExecutionStatus::Abort(err) => {
                // Record the status indicating abort.
                ExecutionStatus::Abort(Error::UserError(err))
            }
        };

        // Remove entries from previous write set that were not overwritten.
        for k in &prev_write_set {
            versioned_data_cache.delete(k, idx_to_execute);
        }

        last_input_output.record(idx_to_execute, state_view.take_reads(), result);
        scheduler.finish_execution(idx_to_execute, incarnation, writes_outside, guard)
    }

    fn validate<'a>(
        &self,
        version_to_validate: Version,
        guard: TaskGuard<'a>,
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &'a Scheduler,
    ) -> SchedulerTask<'a> {
        let (idx_to_validate, incarnation) = version_to_validate;
        let read_set = last_input_output
            .read_set(idx_to_validate)
            .expect("Prior read-set must be recorded");

        let valid = read_set.iter().all(|r| {
            match versioned_data_cache.read(r.path(), idx_to_validate) {
                Ok((version, _)) => r.validate_version(version),
                Err(Some(_)) => false, // Dependency implies a validation failure.
                Err(None) => r.validate_storage(),
            }
        });

        let aborted = !valid && scheduler.try_abort(idx_to_validate, incarnation);

        if aborted {
            // Not valid and successfully aborted, mark the latest write-set as estimates.
            for k in &last_input_output.write_set(idx_to_validate) {
                versioned_data_cache.mark_estimate(k, idx_to_validate);
            }

            scheduler.finish_abort(idx_to_validate, incarnation, guard)
        } else {
            SchedulerTask::NoTask
        }
    }

    fn work_task_with_scope(
        &self,
        executor_arguments: &E::Argument,
        block: &[T],
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &Scheduler,
    ) {
        // Make executor for each task. TODO: fast concurrent executor.
        let executor = E::init(*executor_arguments);

        let mut scheduler_task = SchedulerTask::NoTask;
        loop {
            scheduler_task = match scheduler_task {
                SchedulerTask::ValidationTask(version_to_validate, guard) => self.validate(
                    version_to_validate,
                    guard,
                    last_input_output,
                    versioned_data_cache,
                    scheduler,
                ),
                SchedulerTask::ExecutionTask(version_to_execute, None, guard) => self.execute(
                    version_to_execute,
                    guard,
                    block,
                    last_input_output,
                    versioned_data_cache,
                    scheduler,
                    &executor,
                ),
                SchedulerTask::ExecutionTask(_, Some(condvar), _guard) => {
                    let (lock, cvar) = &*condvar;
                    // Mark dependency resolved.
                    *lock.lock() = true;
                    // Wake up the process waiting for dependency.
                    cvar.notify_one();

                    SchedulerTask::NoTask
                }
                SchedulerTask::NoTask => scheduler.next_task(),
                SchedulerTask::Done => {
                    break;
                }
            }
        }
    }

    pub fn execute_transactions_parallel(
        &self,
        executor_initial_arguments: E::Argument,
        signature_verified_block: Vec<T>,
    ) -> Result<Vec<E::Output>, E::Error> {
        if signature_verified_block.is_empty() {
            return Ok(vec![]);
        }

        let num_txns = signature_verified_block.len();
        let versioned_data_cache = MVHashMap::new();
        let outcomes = OutcomeArray::new(num_txns);
        let last_input_output = TxnLastInputOutput::new(num_txns);
        let scheduler = Scheduler::new(num_txns);

        RAYON_EXEC_POOL.scope(|s| {
            for _ in 0..self.concurrency_level {
                s.spawn(|_| {
                    self.work_task_with_scope(
                        &executor_initial_arguments,
                        &signature_verified_block,
                        &last_input_output,
                        &versioned_data_cache,
                        &scheduler,
                    );
                });
            }
        });

        // Extract outputs in parallel.
        let num_txns = scheduler.num_txn_to_execute();
        let valid_results_size = AtomicUsize::new(num_txns);
        let chunk_size = (num_txns + 4 * self.concurrency_level - 1) / (4 * self.concurrency_level);
        RAYON_EXEC_POOL.install(|| {
            (0..num_txns)
                .collect::<Vec<TxnIndex>>()
                .par_chunks(chunk_size)
                .map(|chunk| {
                    for idx in chunk.iter() {
                        let res = last_input_output.take_output(*idx);
                        if matches!(res, ExecutionStatus::SkipRest(_)) {
                            valid_results_size.fetch_min(*idx + 1, Ordering::SeqCst);
                        }
                        outcomes.set_result(*idx, res);
                    }
                })
                .collect::<()>();
        });

        spawn(move || {
            // Explicit async drops.
            drop(last_input_output);
            drop(signature_verified_block);
            drop(versioned_data_cache);
            drop(scheduler);
        });
        outcomes.get_all_results(valid_results_size.load(Ordering::SeqCst))
    }
}
<commit_msg>Add names to threads in parallel executor's thread pool. (#1756)<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    errors::*,
    outcome_array::OutcomeArray,
    scheduler::{Scheduler, SchedulerTask, TaskGuard, TxnIndex, Version},
    task::{ExecutionStatus, ExecutorTask, Transaction, TransactionOutput},
    txn_last_input_output::{ReadDescriptor, TxnLastInputOutput},
};
use aptos_infallible::Mutex;
use mvhashmap::MVHashMap;
use num_cpus;
use once_cell::sync::Lazy;
use rayon::prelude::*;
use std::{
    collections::HashSet,
    hash::Hash,
    marker::PhantomData,
    sync::{
        atomic::{AtomicUsize, Ordering},
        Arc,
    },
    thread::spawn,
};

static RAYON_EXEC_POOL: Lazy<rayon::ThreadPool> = Lazy::new(|| {
    rayon::ThreadPoolBuilder::new()
        .num_threads(num_cpus::get())
        .thread_name(|index| format!("parallel_executor_{}", index))
        .build()
        .unwrap()
});

/// A struct that is always used by a single thread performing an execution task. The struct is
/// passed to the VM and acts as a proxy to resolve reads first in the shared multi-version
/// data-structure. It also allows the caller to track the read-set and any dependencies.
///
/// TODO(issue 10177): MvHashMapView currently needs to be sync due to trait bounds, but should
/// not be. In this case, the read_dependency member can have a RefCell<bool> type and the
/// captured_reads member can have RefCell<Vec<ReadDescriptor<K>>> type.
pub struct MVHashMapView<'a, K, V> {
    versioned_map: &'a MVHashMap<K, V>,
    txn_idx: TxnIndex,
    scheduler: &'a Scheduler,
    captured_reads: Mutex<Vec<ReadDescriptor<K>>>,
}

impl<'a, K: PartialOrd + Send + Clone + Hash + Eq, V: Send + Sync> MVHashMapView<'a, K, V> {
    /// Drains the captured reads.
    pub fn take_reads(&self) -> Vec<ReadDescriptor<K>> {
        let mut reads = self.captured_reads.lock();
        std::mem::take(&mut reads)
    }

    /// Captures a read from the VM execution.
    pub fn read(&self, key: &K) -> Option<Arc<V>> {
        loop {
            match self.versioned_map.read(key, self.txn_idx) {
                Ok((version, v)) => {
                    let (txn_idx, incarnation) = version;
                    self.captured_reads.lock().push(ReadDescriptor::from(
                        key.clone(),
                        txn_idx,
                        incarnation,
                    ));
                    return Some(v);
                }
                Err(None) => {
                    self.captured_reads
                        .lock()
                        .push(ReadDescriptor::from_storage(key.clone()));
                    return None;
                }
                Err(Some(dep_idx)) => {
                    // `self.txn_idx` estimated to depend on a write from `dep_idx`.
                    match self.scheduler.wait_for_dependency(self.txn_idx, dep_idx) {
                        Some(dep_condition) => {
                            // Wait on a condition variable correpsonding to the encountered
                            // read dependency. Once the dep_idx finishes re-execution, scheduler
                            // will mark the dependency as resolved, and then the txn_idx will be
                            // scheduled for re-execution, which will re-awaken cvar here.
                            // A deadlock is not possible due to these condition variables:
                            // suppose all threads are waiting on read dependency, and consider
                            // one with lowest txn_idx. It observed a dependency, so some thread
                            // aborted dep_idx. If that abort returned execution task, by
                            // minimality (lower transactions aren't waiting), that thread would
                            // finish execution unblock txn_idx, contradiction. Otherwise,
                            // execution_idx in scheduler was lower at a time when at least the
                            // thread that aborted dep_idx was alive, and again, since lower txns
                            // than txn_idx are not blocked, so the execution of dep_idx will
                            // eventually finish and lead to unblocking txn_idx, contradiction.
                            let (lock, cvar) = &*dep_condition;
                            let mut dep_resolved = lock.lock();
                            while !*dep_resolved {
                                dep_resolved = cvar.wait(dep_resolved).unwrap();
                            }
                        }
                        None => continue,
                    }
                }
            };
        }
    }

    /// Return txn_idx associated with the MVHashMapView
    pub fn txn_idx(&self) -> TxnIndex {
        self.txn_idx
    }
}

pub struct ParallelTransactionExecutor<T: Transaction, E: ExecutorTask> {
    // number of active concurrent tasks, corresponding to the maximum number of rayon
    // threads that may be concurrently participating in parallel execution.
    concurrency_level: usize,
    phantom: PhantomData<(T, E)>,
}

impl<T, E> ParallelTransactionExecutor<T, E>
where
    T: Transaction,
    E: ExecutorTask<T = T>,
{
    /// The caller needs to ensure that concurrency_level > 1 (0 is illegal and 1 should
    /// be handled by sequential execution) and that concurrency_level <= num_cpus.
    pub fn new(concurrency_level: usize) -> Self {
        assert!(
            concurrency_level > 1 && concurrency_level <= num_cpus::get(),
            "Parallel execution concurrency level {} should be between 2 and number of CPUs",
            concurrency_level
        );
        Self {
            concurrency_level,
            phantom: PhantomData,
        }
    }

    fn execute<'a>(
        &self,
        version: Version,
        guard: TaskGuard<'a>,
        signature_verified_block: &[T],
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &'a Scheduler,
        executor: &E,
    ) -> SchedulerTask<'a> {
        let (idx_to_execute, incarnation) = version;
        let txn = &signature_verified_block[idx_to_execute];

        let state_view = MVHashMapView {
            versioned_map: versioned_data_cache,
            txn_idx: idx_to_execute,
            scheduler,
            captured_reads: Mutex::new(Vec::new()),
        };

        // VM execution.
        let execute_result = executor.execute_transaction(&state_view, txn);
        let mut prev_write_set: HashSet<T::Key> = last_input_output.write_set(idx_to_execute);

        // For tracking whether the recent execution wrote outside of the previous write set.
        let mut writes_outside = false;
        let mut apply_writes = |output: &<E as ExecutorTask>::Output| {
            let write_version = (idx_to_execute, incarnation);
            for (k, v) in output.get_writes().into_iter() {
                if !prev_write_set.remove(&k) {
                    writes_outside = true
                }
                versioned_data_cache.write(&k, write_version, v);
            }
        };

        let result = match execute_result {
            // These statuses are the results of speculative execution, so even for
            // SkipRest (skip the rest of transactions) and Abort (abort execution with
            // user defined error), no immediate action is taken. Instead the statuses
            // are recorded and (final statuses) are analyzed when the block is executed.
            ExecutionStatus::Success(output) => {
                // Apply the writes to the versioned_data_cache.
                apply_writes(&output);
                ExecutionStatus::Success(output)
            }
            ExecutionStatus::SkipRest(output) => {
                // Apply the writes and record status indicating skip.
                apply_writes(&output);
                ExecutionStatus::SkipRest(output)
            }
            ExecutionStatus::Abort(err) => {
                // Record the status indicating abort.
                ExecutionStatus::Abort(Error::UserError(err))
            }
        };

        // Remove entries from previous write set that were not overwritten.
        for k in &prev_write_set {
            versioned_data_cache.delete(k, idx_to_execute);
        }

        last_input_output.record(idx_to_execute, state_view.take_reads(), result);
        scheduler.finish_execution(idx_to_execute, incarnation, writes_outside, guard)
    }

    fn validate<'a>(
        &self,
        version_to_validate: Version,
        guard: TaskGuard<'a>,
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &'a Scheduler,
    ) -> SchedulerTask<'a> {
        let (idx_to_validate, incarnation) = version_to_validate;
        let read_set = last_input_output
            .read_set(idx_to_validate)
            .expect("Prior read-set must be recorded");

        let valid = read_set.iter().all(|r| {
            match versioned_data_cache.read(r.path(), idx_to_validate) {
                Ok((version, _)) => r.validate_version(version),
                Err(Some(_)) => false, // Dependency implies a validation failure.
                Err(None) => r.validate_storage(),
            }
        });

        let aborted = !valid && scheduler.try_abort(idx_to_validate, incarnation);

        if aborted {
            // Not valid and successfully aborted, mark the latest write-set as estimates.
            for k in &last_input_output.write_set(idx_to_validate) {
                versioned_data_cache.mark_estimate(k, idx_to_validate);
            }

            scheduler.finish_abort(idx_to_validate, incarnation, guard)
        } else {
            SchedulerTask::NoTask
        }
    }

    fn work_task_with_scope(
        &self,
        executor_arguments: &E::Argument,
        block: &[T],
        last_input_output: &TxnLastInputOutput<
            <T as Transaction>::Key,
            <E as ExecutorTask>::Output,
            <E as ExecutorTask>::Error,
        >,
        versioned_data_cache: &MVHashMap<<T as Transaction>::Key, <T as Transaction>::Value>,
        scheduler: &Scheduler,
    ) {
        // Make executor for each task. TODO: fast concurrent executor.
        let executor = E::init(*executor_arguments);

        let mut scheduler_task = SchedulerTask::NoTask;
        loop {
            scheduler_task = match scheduler_task {
                SchedulerTask::ValidationTask(version_to_validate, guard) => self.validate(
                    version_to_validate,
                    guard,
                    last_input_output,
                    versioned_data_cache,
                    scheduler,
                ),
                SchedulerTask::ExecutionTask(version_to_execute, None, guard) => self.execute(
                    version_to_execute,
                    guard,
                    block,
                    last_input_output,
                    versioned_data_cache,
                    scheduler,
                    &executor,
                ),
                SchedulerTask::ExecutionTask(_, Some(condvar), _guard) => {
                    let (lock, cvar) = &*condvar;
                    // Mark dependency resolved.
                    *lock.lock() = true;
                    // Wake up the process waiting for dependency.
                    cvar.notify_one();

                    SchedulerTask::NoTask
                }
                SchedulerTask::NoTask => scheduler.next_task(),
                SchedulerTask::Done => {
                    break;
                }
            }
        }
    }

    pub fn execute_transactions_parallel(
        &self,
        executor_initial_arguments: E::Argument,
        signature_verified_block: Vec<T>,
    ) -> Result<Vec<E::Output>, E::Error> {
        if signature_verified_block.is_empty() {
            return Ok(vec![]);
        }

        let num_txns = signature_verified_block.len();
        let versioned_data_cache = MVHashMap::new();
        let outcomes = OutcomeArray::new(num_txns);
        let last_input_output = TxnLastInputOutput::new(num_txns);
        let scheduler = Scheduler::new(num_txns);

        RAYON_EXEC_POOL.scope(|s| {
            for _ in 0..self.concurrency_level {
                s.spawn(|_| {
                    self.work_task_with_scope(
                        &executor_initial_arguments,
                        &signature_verified_block,
                        &last_input_output,
                        &versioned_data_cache,
                        &scheduler,
                    );
                });
            }
        });

        // Extract outputs in parallel.
        let num_txns = scheduler.num_txn_to_execute();
        let valid_results_size = AtomicUsize::new(num_txns);
        let chunk_size = (num_txns + 4 * self.concurrency_level - 1) / (4 * self.concurrency_level);
        RAYON_EXEC_POOL.install(|| {
            (0..num_txns)
                .collect::<Vec<TxnIndex>>()
                .par_chunks(chunk_size)
                .map(|chunk| {
                    for idx in chunk.iter() {
                        let res = last_input_output.take_output(*idx);
                        if matches!(res, ExecutionStatus::SkipRest(_)) {
                            valid_results_size.fetch_min(*idx + 1, Ordering::SeqCst);
                        }
                        outcomes.set_result(*idx, res);
                    }
                })
                .collect::<()>();
        });

        spawn(move || {
            // Explicit async drops.
            drop(last_input_output);
            drop(signature_verified_block);
            drop(versioned_data_cache);
            drop(scheduler);
        });
        outcomes.get_all_results(valid_results_size.load(Ordering::SeqCst))
    }
}
<|endoftext|>