<commit_before>//! Utilities for managing and calling external binaries

use binary::{
    eyre::{bail, Result},
    Binary, BinaryInstallation, BinaryTrait,
};
use once_cell::sync::Lazy;
use std::collections::{BTreeMap, HashMap};
use tokio::sync::RwLock;

///! A module for locating, running and installing third party binaries.
///!
///! Binaries may be used as runtimes for plugins (e.g. Node.js, Python) or
///! are used in created (e.g Pandoc by the `codec-pandoc` crate).

/// A global store of binaries
///
/// This is an immutable, lazily initialized list of "registered" binaries
/// that Stencila knows how to install, get the version for etc. However,
/// the functions below can be used for any other binary as well.
#[allow(unused_mut)]
static BINARIES: Lazy<BTreeMap<String, Box<dyn BinaryTrait>>> = Lazy::new(|| {
    let mut map: BTreeMap<String, Box<dyn BinaryTrait>> = BTreeMap::new();

    macro_rules! binary_new {
        ($feat:literal, $bin:expr) => {
            #[cfg(feature = $feat)]
            {
                map.insert($bin.spec().name.to_lowercase(), Box::new($bin));
            }
        };
    }

    binary_new!("binary-chrome", binary_chrome::ChromeBinary {});
    binary_new!("binary-chromium", binary_chromium::ChromiumBinary {});
    binary_new!("binary-node", binary_node::NodeBinary {});
    binary_new!("binary-pack", binary_pack::PackBinary {});
    binary_new!("binary-pandoc", binary_pandoc::PandocBinary {});
    binary_new!("binary-python", binary_python::PythonBinary {});
    binary_new!("binary-rscript", binary_rscript::RscriptBinary {});

    map
});

/// Get a registered binary by matching by name and or aliases (case insensitively)
#[allow(clippy::borrowed_box)]
fn registered(name: &str) -> Option<&Box<dyn BinaryTrait>> {
    let name = name.to_lowercase();

    let binary = BINARIES.get(&name);
    if binary.is_some() {
        return binary;
    }

    for binary in BINARIES.values() {
        for alias in binary.spec().aliases {
            if alias.to_lowercase() == name {
                return Some(binary);
            }
        }
    }

    None
}

/// A cache of installations used to memoize calls to `installation`.
static INSTALLATIONS: Lazy<RwLock<HashMap<String, BinaryInstallation>>> =
    Lazy::new(|| RwLock::new(HashMap::new()));

/// Get an installation
///
/// This is a relatively expensive function, even if the binary is already installed,
/// because it searches the file system and executes commands to get their version.
/// Therefore, this function memoizes installations in `INSTALLATIONS` for each `name` and `semver`.
/// Each cached result is removed if the binary is installed or uninstalled.
pub async fn installation(name: &str, semver: &str) -> Result<BinaryInstallation> {
    let name_semver = [name, "@", semver].concat();

    if let Some(installation) = INSTALLATIONS.read().await.get(&name_semver) {
        return Ok(installation.clone());
    }

    let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(name));
    let binary = registered(name).unwrap_or(&unregistered);

    let semver = if semver == "*" {
        None
    } else {
        Some(semver.into())
    };

    if let Some(installation) = binary.installed(semver)? {
        INSTALLATIONS
            .write()
            .await
            .insert(name_semver, installation.clone());
        Ok(installation)
    } else {
        bail!("No matching installation found")
    }
}

/// Is a binary installation meeting semantic versioning requirements installed?
pub async fn installed(name: &str, semver: &str) -> bool {
    installation(name, semver).await.is_ok()
}

/// Install a binary
pub async fn install(name: &str, semver: &str) -> Result<BinaryInstallation> {
    let name_semver = [name, "@", semver].concat();
    let semver = if semver == "*" {
        None
    } else {
        Some(semver.into())
    };

    let binary = match registered(name) {
        Some(binary) => binary,
        None => bail!("Unable to automatically install binary `{}`", name),
    };
    binary.install(semver.clone(), None, None).await?;

    if let Some(installation) = binary.installed(semver)? {
        let mut installations = INSTALLATIONS.write().await;
        installations.insert(name_semver, installation.clone());
        Ok(installation)
    } else {
        bail!("Failed to automatically install binary `{}`", name)
    }
}

/// Get a binary installation meeting semantic versioning requirements.
///
/// If the binary is already available, or automatic installs are configured, returns
/// a `BinaryInstallation` that can be used to run commands. Otherwise, errors
/// with a message that the required binary is not yet installed, or failed to install.
pub async fn require(name: &str, semver: &str) -> Result<BinaryInstallation> {
    if let Ok(installation) = installation(name, semver).await {
        return Ok(installation);
    }

    // TODO: Use an env var to set this?
    let auto = true;
    if auto {
        install(name, semver).await
    } else {
        bail!("Required binary `{}` is not installed", name)
    }
}

/// Get a binary installation meeting one of the semantic versioning requirements.
///
/// If none are installed, will install the first in the list (assuming auto-install
/// is configured as on).
pub async fn require_any(binaries: &[(&str, &str)]) -> Result<BinaryInstallation> {
    for (name, semver) in binaries {
        if let Ok(installation) = installation(name, semver).await {
            return Ok(installation);
        }
    }
    match binaries.get(0) {
        Some((name, semver)) => require(name, semver).await,
        None => bail!("No name/semver pairs provided"),
    }
}

#[cfg(feature = "cli")]
pub mod commands {
    use super::*;
    use cli_utils::structopt::StructOpt;
    use cli_utils::{async_trait::async_trait, result, Result, Run};

    #[derive(Debug, StructOpt)]
    #[structopt(
        about = "Manage helper binaries",
        setting = structopt::clap::AppSettings::ColoredHelp,
        setting = structopt::clap::AppSettings::VersionlessSubcommands
    )]
    pub struct Command {
        #[structopt(subcommand)]
        pub action: Action,
    }

    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder
    )]
    pub enum Action {
        Show(Show),
        Installable(Installable),
        Install(Install),
        Uninstall(Uninstall),
        Run(Run_),
    }

    #[async_trait]
    impl Run for Command {
        async fn run(&self) -> Result {
            let Self { action } = self;
            match action {
                Action::Show(action) => action.run().await,
                Action::Installable(action) => action.run().await,
                Action::Install(action) => action.run().await,
                Action::Uninstall(action) => action.run().await,
                Action::Run(action) => action.run().await,
            }
        }
    }

    /// Show information on a binary
    ///
    /// Searches for the binary on your path and in Stencila's "binaries"
    /// folder for versions that are installed. Use the `semver` argument
    /// to show the latest version that meets the semantic version requirement.
    ///
    /// This command should find any binary that is on your PATH
    /// (i.e. including those not in the `stencila binaries installable` list).
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Show {
        /// The name of the binary e.g. pandoc
        pub name: String,

        /// The semantic version requirement for the binary e.g. >2
        ///
        /// If this is supplied and the binary does not meet the semver
        /// requirement nothing will be shown
        pub semver: Option<String>,
    }

    #[async_trait]
    impl Run for Show {
        async fn run(&self) -> Result {
            // Try to get registered binary (because has potential aliases and extracting versions) but fall
            // back to unregistered for others
            let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(&self.name));
            let binary = registered(&self.name).unwrap_or(&unregistered);
            if self.semver.is_some() {
                if let Ok(installation) = binary.installed(self.semver.clone()) {
                    result::value(installation)
                } else {
                    tracing::info!(
                        "No matching binary found. Perhaps use `stencila binaries install`?"
                    );
                    result::nothing()
                }
            } else {
                let installations = binary.installations();
                result::value(installations)
            }
        }
    }

    /// List binaries (and their versions) that can be installed using Stencila
    ///
    /// The returned list is a list of the binaries/versions that Stencila knows how to install.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Installable {}

    #[async_trait]
    impl Run for Installable {
        async fn run(&self) -> Result {
            let list: Vec<serde_json::Value> = BINARIES
                .values()
                .map(|binary| {
                    let spec = binary.spec();
                    serde_json::json!({
                        "name": spec.name,
                        "versions": spec.installable
                    })
                })
                .collect();
            result::value(list)
        }
    }

    /// Install a binary
    ///
    /// Installs the latest version of the binary, that also meets any
    /// semantic version requirement supplied, into the Stencila "binaries"
    /// folder.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Install {
        /// The name of the binary (must be a registered binary name)
        pub name: String,

        /// The semantic version requirement (the most recent matching version will be installed)
        pub semver: Option<String>,

        /// The operating system to install for (defaults to the current)
        #[structopt(short, long, possible_values = &OS_VALUES )]
        pub os: Option<String>,

        /// The architecture to install for (defaults to the current)
        #[structopt(short, long, possible_values = &ARCH_VALUES)]
        pub arch: Option<String>,
    }

    const OS_VALUES: [&str; 3] = ["macos", "windows", "linux"];
    const ARCH_VALUES: [&str; 3] = ["x86", "x86_64", "arm"];

    #[async_trait]
    impl Run for Install {
        async fn run(&self) -> Result {
            match registered(&self.name) {
                Some(binary) => {
                    binary
                        .install(self.semver.clone(), self.os.clone(), self.arch.clone())
                        .await?;
                    tracing::info!("📦 Installed {}", self.name);
                }
                None => {
                    tracing::error!("Stencila is unable to install `{}`", self.name);
                }
            }
            result::nothing()
        }
    }

    /// Uninstall a binary
    ///
    /// Removes the binary (optionally, just a specific version) from the Stencila
    /// "binaries" folder. No other installations of the binary on the system will
    /// will be removed.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Uninstall {
        /// The name of the binary (must be a registered binary name)
        pub name: String,

        /// The specific version of the binary to uninstall
        ///
        /// If this is not provided, all versions will be removed.
        pub version: Option<String>,
    }
    #[async_trait]
    impl Run for Uninstall {
        async fn run(&self) -> Result {
            // Fallback to unregistered since that is sufficient for uninstall
            let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(&self.name));
            let binary = registered(&self.name).unwrap_or(&unregistered);
            binary.uninstall(self.version.clone()).await?;
            result::nothing()
        }
    }

    /// Run a command using a binary
    ///
    /// Pass arguments and options to the binary after the `--` flag.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Run_ {
        /// The name of the binary e.g. node
        pub name: String,

        /// The semantic version requirement e.g. 16
        pub semver: Option<String>,

        /// The arguments and options to pass to the binary
        #[structopt(raw(true))]
        pub args: Vec<String>,
    }

    #[async_trait]
    impl Run for Run_ {
        async fn run(&self) -> Result {
            let installation = require(
                &self.name,
                &self.semver.clone().unwrap_or_else(|| "*".to_string()),
            )
            .await?;
            let output = installation.run(&self.args).await?;

            use std::io::Write;
            std::io::stdout().write_all(output.stdout.as_ref())?;
            std::io::stderr().write_all(output.stderr.as_ref())?;

            result::nothing()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // End to end CLI test that install, show and uninstall
    // the latest version of each binary. Intended as a coarse
    // tests of installation of each binary. These tests are
    // tagged with #[ignore] because they are slow, so in development
    // you don't want to run them, and because if they are run in
    // parallel with other tests that use `require()` they can cause deadlocks
    // and other on-disk conflicts.

    // Run this test at the start of CI tests using
    //   cargo test binaries::tests::install -- --ignored --nocapture
    #[cfg(feature = "cli")]
    #[tokio::test]
    #[ignore]
    async fn install() -> Result<()> {
        use super::commands::{Install, Installable, Show};
        use cli_utils::Run;
        use test_utils::assert_json_eq;

        Installable {}.run().await?;

        for name in BINARIES.keys() {
            eprintln!("Testing {}", name);

            Install {
                name: name.clone(),
                semver: None,
                os: None,
                arch: None,
            }
            .run()
            .await?;

            let display = Show {
                name: name.clone(),
                semver: None,
            }
            .run()
            .await?;

            let value = if let Some(value) = display.value {
                value
            } else {
                bail!("Expected value")
            };
            assert_json_eq!(value.get("name"), Some(name.clone()));
            assert!(!value
                .get("installs")
                .expect("To have installs")
                .as_array()
                .expect("To be array")
                .is_empty());
        }

        Ok(())
    }

    // Run this test at the end of CI tests using
    //   cargo test binaries::tests::uninstall -- --ignored --nocapture
    #[cfg(feature = "cli")]
    #[tokio::test]
    #[ignore]
    async fn uninstall() -> Result<()> {
        use super::commands::Uninstall;
        use cli_utils::Run;

        for name in BINARIES.keys() {
            Uninstall {
                name: name.clone(),
                version: None,
            }
            .run()
            .await?;
        }

        Ok(())
    }
}
<commit_msg>feat(Binaries): Add `require_sync` function<commit_after>//! Utilities for managing and calling external binaries

use binary::{
    eyre::{bail, Result},
    Binary, BinaryInstallation, BinaryTrait,
};
use once_cell::sync::Lazy;
use std::collections::{BTreeMap, HashMap};
use tokio::sync::RwLock;

///! A module for locating, running and installing third party binaries.
///!
///! Binaries may be used as runtimes for plugins (e.g. Node.js, Python) or
///! are used in created (e.g Pandoc by the `codec-pandoc` crate).

/// A global store of binaries
///
/// This is an immutable, lazily initialized list of "registered" binaries
/// that Stencila knows how to install, get the version for etc. However,
/// the functions below can be used for any other binary as well.
#[allow(unused_mut)]
static BINARIES: Lazy<BTreeMap<String, Box<dyn BinaryTrait>>> = Lazy::new(|| {
    let mut map: BTreeMap<String, Box<dyn BinaryTrait>> = BTreeMap::new();

    macro_rules! binary_new {
        ($feat:literal, $bin:expr) => {
            #[cfg(feature = $feat)]
            {
                map.insert($bin.spec().name.to_lowercase(), Box::new($bin));
            }
        };
    }

    binary_new!("binary-chrome", binary_chrome::ChromeBinary {});
    binary_new!("binary-chromium", binary_chromium::ChromiumBinary {});
    binary_new!("binary-node", binary_node::NodeBinary {});
    binary_new!("binary-pack", binary_pack::PackBinary {});
    binary_new!("binary-pandoc", binary_pandoc::PandocBinary {});
    binary_new!("binary-python", binary_python::PythonBinary {});
    binary_new!("binary-rscript", binary_rscript::RscriptBinary {});

    map
});

/// Get a registered binary by matching by name and or aliases (case insensitively)
#[allow(clippy::borrowed_box)]
fn registered(name: &str) -> Option<&Box<dyn BinaryTrait>> {
    let name = name.to_lowercase();

    let binary = BINARIES.get(&name);
    if binary.is_some() {
        return binary;
    }

    for binary in BINARIES.values() {
        for alias in binary.spec().aliases {
            if alias.to_lowercase() == name {
                return Some(binary);
            }
        }
    }

    None
}

/// A cache of installations used to memoize calls to `installation`.
static INSTALLATIONS: Lazy<RwLock<HashMap<String, BinaryInstallation>>> =
    Lazy::new(|| RwLock::new(HashMap::new()));

/// Get an installation
///
/// This is a relatively expensive function, even if the binary is already installed,
/// because it searches the file system and executes commands to get their version.
/// Therefore, this function memoizes installations in `INSTALLATIONS` for each `name` and `semver`.
/// Each cached result is removed if the binary is installed or uninstalled.
pub async fn installation(name: &str, semver: &str) -> Result<BinaryInstallation> {
    let name_semver = [name, "@", semver].concat();

    if let Some(installation) = INSTALLATIONS.read().await.get(&name_semver) {
        return Ok(installation.clone());
    }

    let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(name));
    let binary = registered(name).unwrap_or(&unregistered);

    let semver = if semver == "*" {
        None
    } else {
        Some(semver.into())
    };

    if let Some(installation) = binary.installed(semver)? {
        INSTALLATIONS
            .write()
            .await
            .insert(name_semver, installation.clone());
        Ok(installation)
    } else {
        bail!("No matching installation found")
    }
}

/// Is a binary installation meeting semantic versioning requirements installed?
pub async fn installed(name: &str, semver: &str) -> bool {
    installation(name, semver).await.is_ok()
}

/// Install a binary
pub async fn install(name: &str, semver: &str) -> Result<BinaryInstallation> {
    let name_semver = [name, "@", semver].concat();
    let semver = if semver == "*" {
        None
    } else {
        Some(semver.into())
    };

    let binary = match registered(name) {
        Some(binary) => binary,
        None => bail!("Unable to automatically install binary `{}`", name),
    };
    binary.install(semver.clone(), None, None).await?;

    if let Some(installation) = binary.installed(semver)? {
        let mut installations = INSTALLATIONS.write().await;
        installations.insert(name_semver, installation.clone());
        Ok(installation)
    } else {
        bail!("Failed to automatically install binary `{}`", name)
    }
}

/// Get a binary installation meeting semantic versioning requirements.
///
/// If the binary is already available, or automatic installs are configured, returns
/// a `BinaryInstallation` that can be used to run commands. Otherwise, errors
/// with a message that the required binary is not yet installed, or failed to install.
pub async fn require(name: &str, semver: &str) -> Result<BinaryInstallation> {
    if let Ok(installation) = installation(name, semver).await {
        return Ok(installation);
    }

    // TODO: Use an env var to set this?
    let auto = true;
    if auto {
        install(name, semver).await
    } else {
        bail!("Required binary `{}` is not installed", name)
    }
}

/// Synchronous version of `require`
pub fn require_sync(name: &str, semver: &str) -> Result<BinaryInstallation> {
    let name = name.to_string();
    let semver = semver.to_string();
    let (sender, receiver) = std::sync::mpsc::channel();
    tokio::spawn(async move {
        let result = require(&name, &semver).await;
        sender.send(result)
    });
    receiver.recv()?
}

/// Get a binary installation meeting one of the semantic versioning requirements.
///
/// If none are installed, will install the first in the list (assuming auto-install
/// is configured as on).
pub async fn require_any(binaries: &[(&str, &str)]) -> Result<BinaryInstallation> {
    for (name, semver) in binaries {
        if let Ok(installation) = installation(name, semver).await {
            return Ok(installation);
        }
    }
    match binaries.get(0) {
        Some((name, semver)) => require(name, semver).await,
        None => bail!("No name/semver pairs provided"),
    }
}

#[cfg(feature = "cli")]
pub mod commands {
    use super::*;
    use cli_utils::structopt::StructOpt;
    use cli_utils::{async_trait::async_trait, result, Result, Run};

    #[derive(Debug, StructOpt)]
    #[structopt(
        about = "Manage helper binaries",
        setting = structopt::clap::AppSettings::ColoredHelp,
        setting = structopt::clap::AppSettings::VersionlessSubcommands
    )]
    pub struct Command {
        #[structopt(subcommand)]
        pub action: Action,
    }

    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder
    )]
    pub enum Action {
        Show(Show),
        Installable(Installable),
        Install(Install),
        Uninstall(Uninstall),
        Run(Run_),
    }

    #[async_trait]
    impl Run for Command {
        async fn run(&self) -> Result {
            let Self { action } = self;
            match action {
                Action::Show(action) => action.run().await,
                Action::Installable(action) => action.run().await,
                Action::Install(action) => action.run().await,
                Action::Uninstall(action) => action.run().await,
                Action::Run(action) => action.run().await,
            }
        }
    }

    /// Show information on a binary
    ///
    /// Searches for the binary on your path and in Stencila's "binaries"
    /// folder for versions that are installed. Use the `semver` argument
    /// to show the latest version that meets the semantic version requirement.
    ///
    /// This command should find any binary that is on your PATH
    /// (i.e. including those not in the `stencila binaries installable` list).
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Show {
        /// The name of the binary e.g. pandoc
        pub name: String,

        /// The semantic version requirement for the binary e.g. >2
        ///
        /// If this is supplied and the binary does not meet the semver
        /// requirement nothing will be shown
        pub semver: Option<String>,
    }

    #[async_trait]
    impl Run for Show {
        async fn run(&self) -> Result {
            // Try to get registered binary (because has potential aliases and extracting versions) but fall
            // back to unregistered for others
            let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(&self.name));
            let binary = registered(&self.name).unwrap_or(&unregistered);
            if self.semver.is_some() {
                if let Ok(installation) = binary.installed(self.semver.clone()) {
                    result::value(installation)
                } else {
                    tracing::info!(
                        "No matching binary found. Perhaps use `stencila binaries install`?"
                    );
                    result::nothing()
                }
            } else {
                let installations = binary.installations();
                result::value(installations)
            }
        }
    }

    /// List binaries (and their versions) that can be installed using Stencila
    ///
    /// The returned list is a list of the binaries/versions that Stencila knows how to install.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Installable {}

    #[async_trait]
    impl Run for Installable {
        async fn run(&self) -> Result {
            let list: Vec<serde_json::Value> = BINARIES
                .values()
                .map(|binary| {
                    let spec = binary.spec();
                    serde_json::json!({
                        "name": spec.name,
                        "versions": spec.installable
                    })
                })
                .collect();
            result::value(list)
        }
    }

    /// Install a binary
    ///
    /// Installs the latest version of the binary, that also meets any
    /// semantic version requirement supplied, into the Stencila "binaries"
    /// folder.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Install {
        /// The name of the binary (must be a registered binary name)
        pub name: String,

        /// The semantic version requirement (the most recent matching version will be installed)
        pub semver: Option<String>,

        /// The operating system to install for (defaults to the current)
        #[structopt(short, long, possible_values = &OS_VALUES )]
        pub os: Option<String>,

        /// The architecture to install for (defaults to the current)
        #[structopt(short, long, possible_values = &ARCH_VALUES)]
        pub arch: Option<String>,
    }

    const OS_VALUES: [&str; 3] = ["macos", "windows", "linux"];
    const ARCH_VALUES: [&str; 3] = ["x86", "x86_64", "arm"];

    #[async_trait]
    impl Run for Install {
        async fn run(&self) -> Result {
            match registered(&self.name) {
                Some(binary) => {
                    binary
                        .install(self.semver.clone(), self.os.clone(), self.arch.clone())
                        .await?;
                    tracing::info!("📦 Installed {}", self.name);
                }
                None => {
                    tracing::error!("Stencila is unable to install `{}`", self.name);
                }
            }
            result::nothing()
        }
    }

    /// Uninstall a binary
    ///
    /// Removes the binary (optionally, just a specific version) from the Stencila
    /// "binaries" folder. No other installations of the binary on the system will
    /// will be removed.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Uninstall {
        /// The name of the binary (must be a registered binary name)
        pub name: String,

        /// The specific version of the binary to uninstall
        ///
        /// If this is not provided, all versions will be removed.
        pub version: Option<String>,
    }
    #[async_trait]
    impl Run for Uninstall {
        async fn run(&self) -> Result {
            // Fallback to unregistered since that is sufficient for uninstall
            let unregistered: Box<dyn BinaryTrait> = Box::new(Binary::unregistered(&self.name));
            let binary = registered(&self.name).unwrap_or(&unregistered);
            binary.uninstall(self.version.clone()).await?;
            result::nothing()
        }
    }

    /// Run a command using a binary
    ///
    /// Pass arguments and options to the binary after the `--` flag.
    #[derive(Debug, StructOpt)]
    #[structopt(
        setting = structopt::clap::AppSettings::DeriveDisplayOrder,
        setting = structopt::clap::AppSettings::ColoredHelp
    )]
    pub struct Run_ {
        /// The name of the binary e.g. node
        pub name: String,

        /// The semantic version requirement e.g. 16
        pub semver: Option<String>,

        /// The arguments and options to pass to the binary
        #[structopt(raw(true))]
        pub args: Vec<String>,
    }

    #[async_trait]
    impl Run for Run_ {
        async fn run(&self) -> Result {
            let installation = require(
                &self.name,
                &self.semver.clone().unwrap_or_else(|| "*".to_string()),
            )
            .await?;
            let output = installation.run(&self.args).await?;

            use std::io::Write;
            std::io::stdout().write_all(output.stdout.as_ref())?;
            std::io::stderr().write_all(output.stderr.as_ref())?;

            result::nothing()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // End to end CLI test that install, show and uninstall
    // the latest version of each binary. Intended as a coarse
    // tests of installation of each binary. These tests are
    // tagged with #[ignore] because they are slow, so in development
    // you don't want to run them, and because if they are run in
    // parallel with other tests that use `require()` they can cause deadlocks
    // and other on-disk conflicts.

    // Run this test at the start of CI tests using
    //   cargo test binaries::tests::install -- --ignored --nocapture
    #[cfg(feature = "cli")]
    #[tokio::test]
    #[ignore]
    async fn install() -> Result<()> {
        use super::commands::{Install, Installable, Show};
        use cli_utils::Run;
        use test_utils::assert_json_eq;

        Installable {}.run().await?;

        for name in BINARIES.keys() {
            eprintln!("Testing {}", name);

            Install {
                name: name.clone(),
                semver: None,
                os: None,
                arch: None,
            }
            .run()
            .await?;

            let display = Show {
                name: name.clone(),
                semver: None,
            }
            .run()
            .await?;

            let value = if let Some(value) = display.value {
                value
            } else {
                bail!("Expected value")
            };
            assert_json_eq!(value.get("name"), Some(name.clone()));
            assert!(!value
                .get("installs")
                .expect("To have installs")
                .as_array()
                .expect("To be array")
                .is_empty());
        }

        Ok(())
    }

    // Run this test at the end of CI tests using
    //   cargo test binaries::tests::uninstall -- --ignored --nocapture
    #[cfg(feature = "cli")]
    #[tokio::test]
    #[ignore]
    async fn uninstall() -> Result<()> {
        use super::commands::Uninstall;
        use cli_utils::Run;

        for name in BINARIES.keys() {
            Uninstall {
                name: name.clone(),
                version: None,
            }
            .run()
            .await?;
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use kernel_micro::{include_file, MicroKernel};

/// A microkernel for R
pub fn new() -> MicroKernel {
    MicroKernel::new(
        "r-micro",
        &["r"],
        &["linux", "macos", "windows"],
        ("Rscript", "*"),
        &["{{script}}"],
        include_file!("r-kernel.r"),
        &[include_file!("r-codec.r")],
        "{{name}} <- decode_value(\"{{json}}\")",
        "{{name}}",
    )
}

#[cfg(test)]
mod tests {
    use super::*;
    use kernel::{
        eyre::{bail, Result},
        stencila_schema::Node,
        KernelTrait,
    };
    use once_cell::sync::Lazy;
    use test_utils::{assert_json_eq, print_logs, serde_json::json};
    use tokio::sync::Mutex;

    // Make sure that only one test runs at any one time
    // This is to avoid `install.packages` being run in parallel.
    // Previous the `serial_test` crate was used for this but did not
    // seem to provide necessary guarantee.
    static QUEUE: Lazy<Mutex<()>> = Lazy::new(Mutex::default);

    async fn skip_or_kernel() -> Result<MicroKernel> {
        let mut kernel = new();
        if !kernel.available().await {
            eprintln!("R not available on this machine");
            bail!("Skipping")
        } else {
            QUEUE.lock().await;
            kernel.start().await?;
        }

        Ok(kernel)
    }

    // Run these tests serially to avoid parallel installation of dependencies
    // which may interfere with each other.

    /// Tests of basic functionality
    /// This test is replicated in all the microkernels.
    /// Other test should be written for language specific quirks and regressions.
    #[tokio::test]
        async fn basics() -> Result<()> {
        print_logs();

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        // Assign a variable and output it
        let (outputs, messages) = kernel.exec("a = 2\na").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[2]]);

        // Print the variable twice and then output it
        let (outputs, messages) = kernel.exec("print(a)\nprint(a)\na").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[2], [2], [2]]);

        // Syntax error
        let (outputs, messages) = kernel.exec("bad ^ # syntax").await?;
        assert_json_eq!(messages[0].error_type, "SyntaxError");
        assert_json_eq!(
            messages[0].error_message,
            "<text>:2:0: unexpected end of input\n1: bad ^ # syntax\n   ^"
        );
        assert_json_eq!(outputs, json!([]));

        // Runtime error
        let (outputs, messages) = kernel.exec("foo").await?;
        assert_json_eq!(messages[0].error_type, "RuntimeError");
        assert_json_eq!(messages[0].error_message, "object 'foo' not found");
        assert_json_eq!(outputs, json!([]));

        // Set and get another variable
        kernel.set("b", Node::Integer(3)).await?;
        let b = kernel.get("b").await?;
        assert_json_eq!(b, [3]);

        // Use both variables
        let (outputs, messages) = kernel.exec("a*b").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[6]]);

        Ok(())
    }

    /// Test that an assignment on the last line does not generate an output
    #[tokio::test]
        async fn assignment_no_output() -> Result<()> {
        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        let (outputs, messages) = kernel.exec("a <- 1").await?;
        assert!(messages.is_empty());
        assert!(outputs.is_empty());

        let (outputs, messages) = kernel.exec("b = 2").await?;
        assert!(messages.is_empty());
        assert!(outputs.is_empty());

        let (outputs, messages) = kernel.exec("print(a)\nprint(b)\na_b <- a + b").await?;
        assert!(messages.is_empty());
        assert_json_eq!(outputs, [[1], [2]]);

        Ok(())
    }

    #[tokio::test]
        async fn encode_general() -> Result<()> {
        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        // Null, booleans, integers, numbers, strings
        let (outputs, messages) = kernel
            .exec("list(NULL, TRUE, FALSE, 1, 1.23456789, 'str')")
            .await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(
            outputs,
            json!([[null, [true], [false], [1], [1.23456789], ["str"]]])
        );

        // Arrays
        let (outputs, messages) = kernel.exec("1:5").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[1, 2, 3, 4, 5]]);

        // Objects
        let (outputs, messages) = kernel.exec("list(a=1, b=list(c=2))").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, json!([{"type": "Entity"}]));
        // TODO: correct output when deserialization of Node::Object is working
        //assert_json_eq!(outputs, json!([{"a": [1], "b": {"c": [2]}}]));

        // Matrix
        let (outputs, messages) = kernel.exec("matrix(c(1:4), 2, 2)").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[[1, 3], [2, 4]]]);

        Ok(())
    }

    #[tokio::test]
        async fn encode_dataframes() -> Result<()> {
        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        let (outputs, messages) = kernel
            .exec(
                r#"data.frame(
    a = 1:2,
    b = c(TRUE, FALSE),
    c = c("x", "y"),
    d = factor(c("X", "Y"), levels = c("X", "Y", "Z")),
    stringsAsFactors = FALSE
)"#,
            )
            .await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec!["a", "b", "c", "d"]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({ "type": "BooleanValidator"})
        );
        assert_json_eq!(
            dt.columns[2].validator.as_ref().unwrap().items_validator,
            json!({ "type": "StringValidator"})
        );
        assert_json_eq!(
            dt.columns[3].validator.as_ref().unwrap().items_validator,
            json!({
                "type": "EnumValidator",
                "values": ["X", "Y", "Z"]
            })
        );

        let (outputs, messages) = kernel.exec("mtcars").await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec![
                "name", "mpg", "cyl", "disp", "hp", "drat", "wt", "qsec", "vs", "am", "gear",
                "carb"
            ]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "StringValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );

        let (outputs, messages) = kernel.exec("chickwts").await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec!["weight", "feed"]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({
                "type": "EnumValidator",
                "values": ["casein", "horsebean", "linseed", "meatmeal", "soybean", "sunflower"]
            })
        );

        Ok(())
    }

    #[tokio::test]
        async fn encode_plots() -> Result<()> {
        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        for code in ["plot(1)", "hist(rnorm(1000), breaks=30)"] {
            let (outputs, messages) = kernel.exec(code).await?;
            assert_json_eq!(messages, json!([]));
            let image = match &outputs[0] {
                Node::ImageObject(dt) => dt.clone(),
                _ => bail!("unexpected type {:?}", outputs[0]),
            };
            assert!(image.content_url.starts_with("data:image/png;base64,"));
        }

        Ok(())
    }
}
<commit_msg>test(R microkernel): Hold on to queue lock until end of each test<commit_after>use kernel_micro::{include_file, MicroKernel};

/// A microkernel for R
pub fn new() -> MicroKernel {
    MicroKernel::new(
        "r-micro",
        &["r"],
        &["linux", "macos", "windows"],
        ("Rscript", "*"),
        &["{{script}}"],
        include_file!("r-kernel.r"),
        &[include_file!("r-codec.r")],
        "{{name}} <- decode_value(\"{{json}}\")",
        "{{name}}",
    )
}

#[cfg(test)]
mod tests {
    use super::*;
    use kernel::{
        eyre::{bail, Result},
        stencila_schema::Node,
        KernelTrait,
    };
    use once_cell::sync::Lazy;
    use test_utils::{assert_json_eq, print_logs, serde_json::json};
    use tokio::sync::Mutex;

    // Make sure that only one test runs at any one time
    // This is to avoid `install.packages` being run in parallel.
    // Previous the `serial_test` crate was used for this but did not
    // seem to provide necessary guarantee.
    static QUEUE: Lazy<Mutex<()>> = Lazy::new(Mutex::default);

    async fn skip_or_kernel() -> Result<MicroKernel> {
        let mut kernel = new();
        if !kernel.available().await {
            eprintln!("R not available on this machine");
            bail!("Skipping")
        } else {
            kernel.start().await?;
        }

        Ok(kernel)
    }

    // Run these tests serially to avoid parallel installation of dependencies
    // which may interfere with each other.

    /// Tests of basic functionality
    /// This test is replicated in all the microkernels.
    /// Other test should be written for language specific quirks and regressions.
    #[tokio::test]
    async fn basics() -> Result<()> {
        let _guard = QUEUE.lock().await;

        print_logs();

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        // Assign a variable and output it
        let (outputs, messages) = kernel.exec("a = 2\na").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[2]]);

        // Print the variable twice and then output it
        let (outputs, messages) = kernel.exec("print(a)\nprint(a)\na").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[2], [2], [2]]);

        // Syntax error
        let (outputs, messages) = kernel.exec("bad ^ # syntax").await?;
        assert_json_eq!(messages[0].error_type, "SyntaxError");
        assert_json_eq!(
            messages[0].error_message,
            "<text>:2:0: unexpected end of input\n1: bad ^ # syntax\n   ^"
        );
        assert_json_eq!(outputs, json!([]));

        // Runtime error
        let (outputs, messages) = kernel.exec("foo").await?;
        assert_json_eq!(messages[0].error_type, "RuntimeError");
        assert_json_eq!(messages[0].error_message, "object 'foo' not found");
        assert_json_eq!(outputs, json!([]));

        // Set and get another variable
        kernel.set("b", Node::Integer(3)).await?;
        let b = kernel.get("b").await?;
        assert_json_eq!(b, [3]);

        // Use both variables
        let (outputs, messages) = kernel.exec("a*b").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[6]]);

        Ok(())
    }

    /// Test that an assignment on the last line does not generate an output
    #[tokio::test]
    async fn assignment_no_output() -> Result<()> {
        let _guard = QUEUE.lock().await;

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        let (outputs, messages) = kernel.exec("a <- 1").await?;
        assert!(messages.is_empty());
        assert!(outputs.is_empty());

        let (outputs, messages) = kernel.exec("b = 2").await?;
        assert!(messages.is_empty());
        assert!(outputs.is_empty());

        let (outputs, messages) = kernel.exec("print(a)\nprint(b)\na_b <- a + b").await?;
        assert!(messages.is_empty());
        assert_json_eq!(outputs, [[1], [2]]);

        Ok(())
    }

    #[tokio::test]
    async fn encode_general() -> Result<()> {
        let _guard = QUEUE.lock().await;

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        // Null, booleans, integers, numbers, strings
        let (outputs, messages) = kernel
            .exec("list(NULL, TRUE, FALSE, 1, 1.23456789, 'str')")
            .await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(
            outputs,
            json!([[null, [true], [false], [1], [1.23456789], ["str"]]])
        );

        // Arrays
        let (outputs, messages) = kernel.exec("1:5").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[1, 2, 3, 4, 5]]);

        // Objects
        let (outputs, messages) = kernel.exec("list(a=1, b=list(c=2))").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, json!([{"type": "Entity"}]));
        // TODO: correct output when deserialization of Node::Object is working
        //assert_json_eq!(outputs, json!([{"a": [1], "b": {"c": [2]}}]));

        // Matrix
        let (outputs, messages) = kernel.exec("matrix(c(1:4), 2, 2)").await?;
        assert_json_eq!(messages, json!([]));
        assert_json_eq!(outputs, [[[1, 3], [2, 4]]]);

        Ok(())
    }

    #[tokio::test]
    async fn encode_dataframes() -> Result<()> {
        let _guard = QUEUE.lock().await;

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        let (outputs, messages) = kernel
            .exec(
                r#"data.frame(
    a = 1:2,
    b = c(TRUE, FALSE),
    c = c("x", "y"),
    d = factor(c("X", "Y"), levels = c("X", "Y", "Z")),
    stringsAsFactors = FALSE
)"#,
            )
            .await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec!["a", "b", "c", "d"]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({ "type": "BooleanValidator"})
        );
        assert_json_eq!(
            dt.columns[2].validator.as_ref().unwrap().items_validator,
            json!({ "type": "StringValidator"})
        );
        assert_json_eq!(
            dt.columns[3].validator.as_ref().unwrap().items_validator,
            json!({
                "type": "EnumValidator",
                "values": ["X", "Y", "Z"]
            })
        );

        let (outputs, messages) = kernel.exec("mtcars").await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec![
                "name", "mpg", "cyl", "disp", "hp", "drat", "wt", "qsec", "vs", "am", "gear",
                "carb"
            ]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "StringValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );

        let (outputs, messages) = kernel.exec("chickwts").await?;
        assert_json_eq!(messages, json!([]));
        let dt = match &outputs[0] {
            Node::Datatable(dt) => dt.clone(),
            _ => bail!("unexpected type {:?}", outputs[0]),
        };
        assert_eq!(
            dt.columns
                .iter()
                .map(|column| column.name.as_str())
                .collect::<Vec<&str>>(),
            vec!["weight", "feed"]
        );
        assert_json_eq!(
            dt.columns[0].validator.as_ref().unwrap().items_validator,
            json!({ "type": "NumberValidator"})
        );
        assert_json_eq!(
            dt.columns[1].validator.as_ref().unwrap().items_validator,
            json!({
                "type": "EnumValidator",
                "values": ["casein", "horsebean", "linseed", "meatmeal", "soybean", "sunflower"]
            })
        );

        Ok(())
    }

    #[tokio::test]
    async fn encode_plots() -> Result<()> {
        let _guard = QUEUE.lock().await;

        let mut kernel = match skip_or_kernel().await {
            Ok(kernel) => kernel,
            Err(..) => return Ok(()),
        };

        for code in ["plot(1)", "hist(rnorm(1000), breaks=30)"] {
            let (outputs, messages) = kernel.exec(code).await?;
            assert_json_eq!(messages, json!([]));
            let image = match &outputs[0] {
                Node::ImageObject(dt) => dt.clone(),
                _ => bail!("unexpected type {:?}", outputs[0]),
            };
            assert!(image.content_url.starts_with("data:image/png;base64,"));
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use memchr::{memchr, memrchr};
use regex::bytes::{Regex, RegexBuilder};
use syntax;

use literals::LiteralSets;
use nonl;
use syntax::Expr;
use word_boundary::strip_unicode_word_boundaries;
use Result;

/// A matched line.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct Match {
    start: usize,
    end: usize,
}

impl Match {
    /// Create a new empty match value.
    pub fn new() -> Match {
        Match::default()
    }

    /// Return the starting byte offset of the line that matched.
    #[inline]
    pub fn start(&self) -> usize {
        self.start
    }

    /// Return the ending byte offset of the line that matched.
    #[inline]
    pub fn end(&self) -> usize {
        self.end
    }
}

/// A fast line oriented regex searcher.
#[derive(Clone, Debug)]
pub struct Grep {
    re: Regex,
    required: Option<Regex>,
    opts: Options,
}

/// A builder for a grep searcher.
#[derive(Clone, Debug)]
pub struct GrepBuilder {
    pattern: String,
    opts: Options,
}

#[derive(Clone, Debug)]
struct Options {
    case_insensitive: bool,
    case_smart: bool,
    line_terminator: u8,
    size_limit: usize,
    dfa_size_limit: usize,
}

impl Default for Options {
    fn default() -> Options {
        Options {
            case_insensitive: false,
            case_smart: false,
            line_terminator: b'\n',
            size_limit: 10 * (1 << 20),
            dfa_size_limit: 10 * (1 << 20),
        }
    }
}

impl GrepBuilder {
    /// Create a new builder for line searching.
    ///
    /// The pattern given should be a regular expression. The precise syntax
    /// supported is documented on the regex crate.
    pub fn new(pattern: &str) -> GrepBuilder {
        GrepBuilder {
            pattern: pattern.to_string(),
            opts: Options::default(),
        }
    }

    /// Set the line terminator.
    ///
    /// The line terminator can be any ASCII character and serves to delineate
    /// the match boundaries in the text searched.
    ///
    /// This panics if `ascii_byte` is greater than `0x7F` (i.e., not ASCII).
    pub fn line_terminator(mut self, ascii_byte: u8) -> GrepBuilder {
        assert!(ascii_byte <= 0x7F);
        self.opts.line_terminator = ascii_byte;
        self
    }

    /// Set the case sensitive flag (`i`) on the regex.
    pub fn case_insensitive(mut self, yes: bool) -> GrepBuilder {
        self.opts.case_insensitive = yes;
        self
    }

    /// Whether to enable smart case search or not (disabled by default).
    ///
    /// Smart case uses case insensitive search if the regex is contains all
    /// lowercase literal characters. Otherwise, a case sensitive search is
    /// used instead.
    ///
    /// Enabling the case_insensitive flag overrides this.
    pub fn case_smart(mut self, yes: bool) -> GrepBuilder {
        self.opts.case_smart = yes;
        self
    }

    /// Set the approximate size limit of the compiled regular expression.
    ///
    /// This roughly corresponds to the number of bytes occupied by a
    /// single compiled program. If the program exceeds this number, then a
    /// compilation error is returned.
    pub fn size_limit(mut self, limit: usize) -> GrepBuilder {
        self.opts.size_limit = limit;
        self
    }

    /// Set the approximate size of the cache used by the DFA.
    ///
    /// This roughly corresponds to the number of bytes that the DFA will use
    /// while searching.
    ///
    /// Note that this is a per thread limit. There is no way to set a global
    /// limit. In particular, if a regex is used from multiple threads
    /// simulanteously, then each thread may use up to the number of bytes
    /// specified here.
    pub fn dfa_size_limit(mut self, limit: usize) -> GrepBuilder {
        self.opts.dfa_size_limit = limit;
        self
    }

    /// Create a line searcher.
    ///
    /// If there was a problem parsing or compiling the regex with the given
    /// options, then an error is returned.
    pub fn build(self) -> Result<Grep> {
        let expr = try!(self.parse());
        let literals = LiteralSets::create(&expr);
        let re = try!(self.regex(&expr));
        let required = literals.to_regex().or_else(|| {
            let expr = match strip_unicode_word_boundaries(&expr) {
                None => return None,
                Some(expr) => expr,
            };
            debug!("Stripped Unicode word boundaries. New AST:\n{:?}", expr);
            self.regex(&expr).ok()
        });
        Ok(Grep {
            re: re,
            required: required,
            opts: self.opts,
        })
    }

    /// Creates a new regex from the given expression with the current
    /// configuration.
    fn regex(&self, expr: &Expr) -> Result<Regex> {
        let casei =
            self.opts.case_insensitive
            || (self.opts.case_smart && !has_uppercase_literal(expr));
        RegexBuilder::new(&expr.to_string())
            .case_insensitive(casei)
            .multi_line(true)
            .unicode(true)
            .size_limit(self.opts.size_limit)
            .dfa_size_limit(self.opts.dfa_size_limit)
            .compile()
            .map_err(From::from)
    }

    /// Parses the underlying pattern and ensures the pattern can never match
    /// the line terminator.
    fn parse(&self) -> Result<syntax::Expr> {
        let expr =
            try!(syntax::ExprBuilder::new()
                 .allow_bytes(true)
                 .unicode(true)
                 .case_insensitive(self.opts.case_insensitive)
                 .parse(&self.pattern));
        debug!("regex ast:\n{:#?}", expr);
        Ok(try!(nonl::remove(expr, self.opts.line_terminator)))
    }
}

impl Grep {
    /// Returns a reference to the underlying regex used by the searcher.
    pub fn regex(&self) -> &Regex {
        &self.re
    }

    /// Returns an iterator over all matches in the given buffer.
    pub fn iter<'b, 's>(&'s self, buf: &'b [u8]) -> Iter<'b, 's> {
        Iter {
            searcher: self,
            buf: buf,
            start: 0,
        }
    }

    /// Fills in the next line that matches in the given buffer starting at
    /// the position given.
    ///
    /// If no match could be found, `false` is returned, otherwise, `true` is
    /// returned.
    pub fn read_match(
        &self,
        mat: &mut Match,
        buf: &[u8],
        mut start: usize,
    ) -> bool {
        if start >= buf.len() {
            return false;
        }
        if let Some(ref req) = self.required {
            while start < buf.len() {
                let e = match req.shortest_match(&buf[start..]) {
                    None => return false,
                    Some(e) => start + e,
                };
                let (prevnl, nextnl) = self.find_line(buf, e, e);
                match self.re.shortest_match(&buf[prevnl..nextnl]) {
                    None => {
                        start = nextnl;
                        continue;
                    }
                    Some(_) => {
                        self.fill_match(mat, prevnl, nextnl);
                        return true;
                    }
                }
            }
            false
        } else {
            let e = match self.re.shortest_match(&buf[start..]) {
                None => return false,
                Some(e) => start + e,
            };
            let (s, e) = self.find_line(buf, e, e);
            self.fill_match(mat, s, e);
            true
        }
    }

    fn fill_match(&self, mat: &mut Match, start: usize, end: usize) {
        mat.start = start;
        mat.end = end;
    }

    fn find_line(&self, buf: &[u8], s: usize, e: usize) -> (usize, usize) {
        (self.find_line_start(buf, s), self.find_line_end(buf, e))
    }

    fn find_line_start(&self, buf: &[u8], pos: usize) -> usize {
        memrchr(self.opts.line_terminator, &buf[0..pos]).map_or(0, |i| i + 1)
    }

    fn find_line_end(&self, buf: &[u8], pos: usize) -> usize {
        memchr(self.opts.line_terminator, &buf[pos..])
            .map_or(buf.len(), |i| pos + i + 1)
    }
}

/// An iterator over all matches in a particular buffer.
///
/// `'b` refers to the lifetime of the buffer, and `'s` refers to the lifetime
/// of the searcher.
pub struct Iter<'b, 's> {
    searcher: &'s Grep,
    buf: &'b [u8],
    start: usize,
}

impl<'b, 's> Iterator for Iter<'b, 's> {
    type Item = Match;

    fn next(&mut self) -> Option<Match> {
        let mut mat = Match::default();
        if !self.searcher.read_match(&mut mat, self.buf, self.start) {
            self.start = self.buf.len();
            return None;
        }
        self.start = mat.end;
        Some(mat)
    }
}

fn has_uppercase_literal(expr: &Expr) -> bool {
    use syntax::Expr::*;
    match *expr {
        Literal { ref chars, casei } => {
            casei || chars.iter().any(|c| c.is_uppercase())
        }
        LiteralBytes { ref bytes, casei } => {
            casei || bytes.iter().any(|&b| b'A' <= b && b <= b'Z')
        }
        Group { ref e, .. } => has_uppercase_literal(e),
        Repeat { ref e, .. } => has_uppercase_literal(e),
        Concat(ref es) => es.iter().any(has_uppercase_literal),
        Alternate(ref es) => es.iter().any(has_uppercase_literal),
        _ => false,
    }
}

#[cfg(test)]
mod tests {
    #![allow(unused_imports)]

    use memchr::{memchr, memrchr};
    use regex::bytes::Regex;

    use super::{GrepBuilder, Match};

    static SHERLOCK: &'static [u8] = include_bytes!("./data/sherlock.txt");

    #[allow(dead_code)]
    fn s(bytes: &[u8]) -> String {
        String::from_utf8(bytes.to_vec()).unwrap()
    }

    fn find_lines(pat: &str, haystack: &[u8]) -> Vec<Match> {
        let re = Regex::new(pat).unwrap();
        let mut lines = vec![];
        for (s, e) in re.find_iter(haystack) {
            let start = memrchr(b'\n', &haystack[..s])
                        .map_or(0, |i| i + 1);
            let end = memchr(b'\n', &haystack[e..])
                      .map_or(haystack.len(), |i| e + i + 1);
            lines.push(Match {
                start: start,
                end: end,
            });
        }
        lines
    }

    fn grep_lines(pat: &str, haystack: &[u8]) -> Vec<Match> {
        let g = GrepBuilder::new(pat).build().unwrap();
        g.iter(haystack).collect()
    }

    #[test]
    fn buffered_literal() {
        let expected = find_lines("Sherlock Holmes", SHERLOCK);
        let got = grep_lines("Sherlock Holmes", SHERLOCK);
        assert_eq!(expected.len(), got.len());
        assert_eq!(expected, got);
    }
}
<commit_msg>Fix debug expression statement.<commit_after>use memchr::{memchr, memrchr};
use regex::bytes::{Regex, RegexBuilder};
use syntax;

use literals::LiteralSets;
use nonl;
use syntax::Expr;
use word_boundary::strip_unicode_word_boundaries;
use Result;

/// A matched line.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct Match {
    start: usize,
    end: usize,
}

impl Match {
    /// Create a new empty match value.
    pub fn new() -> Match {
        Match::default()
    }

    /// Return the starting byte offset of the line that matched.
    #[inline]
    pub fn start(&self) -> usize {
        self.start
    }

    /// Return the ending byte offset of the line that matched.
    #[inline]
    pub fn end(&self) -> usize {
        self.end
    }
}

/// A fast line oriented regex searcher.
#[derive(Clone, Debug)]
pub struct Grep {
    re: Regex,
    required: Option<Regex>,
    opts: Options,
}

/// A builder for a grep searcher.
#[derive(Clone, Debug)]
pub struct GrepBuilder {
    pattern: String,
    opts: Options,
}

#[derive(Clone, Debug)]
struct Options {
    case_insensitive: bool,
    case_smart: bool,
    line_terminator: u8,
    size_limit: usize,
    dfa_size_limit: usize,
}

impl Default for Options {
    fn default() -> Options {
        Options {
            case_insensitive: false,
            case_smart: false,
            line_terminator: b'\n',
            size_limit: 10 * (1 << 20),
            dfa_size_limit: 10 * (1 << 20),
        }
    }
}

impl GrepBuilder {
    /// Create a new builder for line searching.
    ///
    /// The pattern given should be a regular expression. The precise syntax
    /// supported is documented on the regex crate.
    pub fn new(pattern: &str) -> GrepBuilder {
        GrepBuilder {
            pattern: pattern.to_string(),
            opts: Options::default(),
        }
    }

    /// Set the line terminator.
    ///
    /// The line terminator can be any ASCII character and serves to delineate
    /// the match boundaries in the text searched.
    ///
    /// This panics if `ascii_byte` is greater than `0x7F` (i.e., not ASCII).
    pub fn line_terminator(mut self, ascii_byte: u8) -> GrepBuilder {
        assert!(ascii_byte <= 0x7F);
        self.opts.line_terminator = ascii_byte;
        self
    }

    /// Set the case sensitive flag (`i`) on the regex.
    pub fn case_insensitive(mut self, yes: bool) -> GrepBuilder {
        self.opts.case_insensitive = yes;
        self
    }

    /// Whether to enable smart case search or not (disabled by default).
    ///
    /// Smart case uses case insensitive search if the regex is contains all
    /// lowercase literal characters. Otherwise, a case sensitive search is
    /// used instead.
    ///
    /// Enabling the case_insensitive flag overrides this.
    pub fn case_smart(mut self, yes: bool) -> GrepBuilder {
        self.opts.case_smart = yes;
        self
    }

    /// Set the approximate size limit of the compiled regular expression.
    ///
    /// This roughly corresponds to the number of bytes occupied by a
    /// single compiled program. If the program exceeds this number, then a
    /// compilation error is returned.
    pub fn size_limit(mut self, limit: usize) -> GrepBuilder {
        self.opts.size_limit = limit;
        self
    }

    /// Set the approximate size of the cache used by the DFA.
    ///
    /// This roughly corresponds to the number of bytes that the DFA will use
    /// while searching.
    ///
    /// Note that this is a per thread limit. There is no way to set a global
    /// limit. In particular, if a regex is used from multiple threads
    /// simulanteously, then each thread may use up to the number of bytes
    /// specified here.
    pub fn dfa_size_limit(mut self, limit: usize) -> GrepBuilder {
        self.opts.dfa_size_limit = limit;
        self
    }

    /// Create a line searcher.
    ///
    /// If there was a problem parsing or compiling the regex with the given
    /// options, then an error is returned.
    pub fn build(self) -> Result<Grep> {
        let expr = try!(self.parse());
        let literals = LiteralSets::create(&expr);
        let re = try!(self.regex(&expr));
        let required = literals.to_regex().or_else(|| {
            let expr = match strip_unicode_word_boundaries(&expr) {
                None => return None,
                Some(expr) => expr,
            };
            debug!("Stripped Unicode word boundaries. New AST:\n{:?}", expr);
            self.regex(&expr).ok()
        });
        Ok(Grep {
            re: re,
            required: required,
            opts: self.opts,
        })
    }

    /// Creates a new regex from the given expression with the current
    /// configuration.
    fn regex(&self, expr: &Expr) -> Result<Regex> {
        let casei =
            self.opts.case_insensitive
            || (self.opts.case_smart && !has_uppercase_literal(expr));
        RegexBuilder::new(&expr.to_string())
            .case_insensitive(casei)
            .multi_line(true)
            .unicode(true)
            .size_limit(self.opts.size_limit)
            .dfa_size_limit(self.opts.dfa_size_limit)
            .compile()
            .map_err(From::from)
    }

    /// Parses the underlying pattern and ensures the pattern can never match
    /// the line terminator.
    fn parse(&self) -> Result<syntax::Expr> {
        let expr =
            try!(syntax::ExprBuilder::new()
                 .allow_bytes(true)
                 .unicode(true)
                 .case_insensitive(self.opts.case_insensitive)
                 .parse(&self.pattern));
        let expr = try!(nonl::remove(expr, self.opts.line_terminator));
        debug!("regex ast:\n{:#?}", expr);
        Ok(expr)
    }
}

impl Grep {
    /// Returns a reference to the underlying regex used by the searcher.
    pub fn regex(&self) -> &Regex {
        &self.re
    }

    /// Returns an iterator over all matches in the given buffer.
    pub fn iter<'b, 's>(&'s self, buf: &'b [u8]) -> Iter<'b, 's> {
        Iter {
            searcher: self,
            buf: buf,
            start: 0,
        }
    }

    /// Fills in the next line that matches in the given buffer starting at
    /// the position given.
    ///
    /// If no match could be found, `false` is returned, otherwise, `true` is
    /// returned.
    pub fn read_match(
        &self,
        mat: &mut Match,
        buf: &[u8],
        mut start: usize,
    ) -> bool {
        if start >= buf.len() {
            return false;
        }
        if let Some(ref req) = self.required {
            while start < buf.len() {
                let e = match req.shortest_match(&buf[start..]) {
                    None => return false,
                    Some(e) => start + e,
                };
                let (prevnl, nextnl) = self.find_line(buf, e, e);
                match self.re.shortest_match(&buf[prevnl..nextnl]) {
                    None => {
                        start = nextnl;
                        continue;
                    }
                    Some(_) => {
                        self.fill_match(mat, prevnl, nextnl);
                        return true;
                    }
                }
            }
            false
        } else {
            let e = match self.re.shortest_match(&buf[start..]) {
                None => return false,
                Some(e) => start + e,
            };
            let (s, e) = self.find_line(buf, e, e);
            self.fill_match(mat, s, e);
            true
        }
    }

    fn fill_match(&self, mat: &mut Match, start: usize, end: usize) {
        mat.start = start;
        mat.end = end;
    }

    fn find_line(&self, buf: &[u8], s: usize, e: usize) -> (usize, usize) {
        (self.find_line_start(buf, s), self.find_line_end(buf, e))
    }

    fn find_line_start(&self, buf: &[u8], pos: usize) -> usize {
        memrchr(self.opts.line_terminator, &buf[0..pos]).map_or(0, |i| i + 1)
    }

    fn find_line_end(&self, buf: &[u8], pos: usize) -> usize {
        memchr(self.opts.line_terminator, &buf[pos..])
            .map_or(buf.len(), |i| pos + i + 1)
    }
}

/// An iterator over all matches in a particular buffer.
///
/// `'b` refers to the lifetime of the buffer, and `'s` refers to the lifetime
/// of the searcher.
pub struct Iter<'b, 's> {
    searcher: &'s Grep,
    buf: &'b [u8],
    start: usize,
}

impl<'b, 's> Iterator for Iter<'b, 's> {
    type Item = Match;

    fn next(&mut self) -> Option<Match> {
        let mut mat = Match::default();
        if !self.searcher.read_match(&mut mat, self.buf, self.start) {
            self.start = self.buf.len();
            return None;
        }
        self.start = mat.end;
        Some(mat)
    }
}

fn has_uppercase_literal(expr: &Expr) -> bool {
    use syntax::Expr::*;
    match *expr {
        Literal { ref chars, casei } => {
            casei || chars.iter().any(|c| c.is_uppercase())
        }
        LiteralBytes { ref bytes, casei } => {
            casei || bytes.iter().any(|&b| b'A' <= b && b <= b'Z')
        }
        Group { ref e, .. } => has_uppercase_literal(e),
        Repeat { ref e, .. } => has_uppercase_literal(e),
        Concat(ref es) => es.iter().any(has_uppercase_literal),
        Alternate(ref es) => es.iter().any(has_uppercase_literal),
        _ => false,
    }
}

#[cfg(test)]
mod tests {
    #![allow(unused_imports)]

    use memchr::{memchr, memrchr};
    use regex::bytes::Regex;

    use super::{GrepBuilder, Match};

    static SHERLOCK: &'static [u8] = include_bytes!("./data/sherlock.txt");

    #[allow(dead_code)]
    fn s(bytes: &[u8]) -> String {
        String::from_utf8(bytes.to_vec()).unwrap()
    }

    fn find_lines(pat: &str, haystack: &[u8]) -> Vec<Match> {
        let re = Regex::new(pat).unwrap();
        let mut lines = vec![];
        for (s, e) in re.find_iter(haystack) {
            let start = memrchr(b'\n', &haystack[..s])
                        .map_or(0, |i| i + 1);
            let end = memchr(b'\n', &haystack[e..])
                      .map_or(haystack.len(), |i| e + i + 1);
            lines.push(Match {
                start: start,
                end: end,
            });
        }
        lines
    }

    fn grep_lines(pat: &str, haystack: &[u8]) -> Vec<Match> {
        let g = GrepBuilder::new(pat).build().unwrap();
        g.iter(haystack).collect()
    }

    #[test]
    fn buffered_literal() {
        let expected = find_lines("Sherlock Holmes", SHERLOCK);
        let got = grep_lines("Sherlock Holmes", SHERLOCK);
        assert_eq!(expected.len(), got.len());
        assert_eq!(expected, got);
    }
}
<|endoftext|>


<commit_before>use syntax_pos::symbol::Symbol;
use back::write::create_informational_target_machine;
use llvm;
use rustc::session::Session;
use rustc::session::config::PrintRequest;
use rustc_target::spec::MergeFunctions;
use libc::c_int;
use std::ffi::CString;
use syntax::feature_gate::UnstableFeatures;

use std::str;
use std::slice;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Once;

static POISONED: AtomicBool = AtomicBool::new(false);
static INIT: Once = Once::new();

pub(crate) fn init(sess: &Session) {
    unsafe {
        // Before we touch LLVM, make sure that multithreading is enabled.
        INIT.call_once(|| {
            if llvm::LLVMStartMultithreaded() != 1 {
                // use an extra bool to make sure that all future usage of LLVM
                // cannot proceed despite the Once not running more than once.
                POISONED.store(true, Ordering::SeqCst);
            }

            configure_llvm(sess);
        });

        if POISONED.load(Ordering::SeqCst) {
            bug!("couldn't enable multi-threaded LLVM");
        }
    }
}

fn require_inited() {
    INIT.call_once(|| bug!("llvm is not initialized"));
    if POISONED.load(Ordering::SeqCst) {
        bug!("couldn't enable multi-threaded LLVM");
    }
}

unsafe fn configure_llvm(sess: &Session) {
    let n_args = sess.opts.cg.llvm_args.len();
    let mut llvm_c_strs = Vec::with_capacity(n_args + 1);
    let mut llvm_args = Vec::with_capacity(n_args + 1);

    llvm::LLVMRustInstallFatalErrorHandler();

    {
        let mut add = |arg: &str| {
            let s = CString::new(arg).unwrap();
            llvm_args.push(s.as_ptr());
            llvm_c_strs.push(s);
        };
        add("rustc"); // fake program name
        if sess.time_llvm_passes() { add("-time-passes"); }
        if sess.print_llvm_passes() { add("-debug-pass=Structure"); }
        if sess.opts.debugging_opts.disable_instrumentation_preinliner {
            add("-disable-preinline");
        }
        if llvm::LLVMRustIsRustLLVM() {
            match sess.opts.debugging_opts.merge_functions
                  .unwrap_or(sess.target.target.options.merge_functions) {
                MergeFunctions::Disabled |
                MergeFunctions::Trampolines => {}
                MergeFunctions::Aliases => {
                    add("-mergefunc-use-aliases");
                }
            }
        }

        // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes
        // during inlining. Unfortunately these may block other optimizations.
        add("-preserve-alignment-assumptions-during-inlining=false");

        for arg in &sess.opts.cg.llvm_args {
            add(&(*arg));
        }
    }

    llvm::LLVMInitializePasses();

    ::rustc_llvm::initialize_available_targets();

    llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int,
                                 llvm_args.as_ptr());
}

// WARNING: the features after applying `to_llvm_feature` must be known
// to LLVM or the feature detection code will walk past the end of the feature
// array, leading to crashes.

const ARM_WHITELIST: &[(&str, Option<&str>)] = &[
    ("aclass", Some("arm_target_feature")),
    ("mclass", Some("arm_target_feature")),
    ("rclass", Some("arm_target_feature")),
    ("dsp", Some("arm_target_feature")),
    ("neon", Some("arm_target_feature")),
    ("v5te", Some("arm_target_feature")),
    ("v6", Some("arm_target_feature")),
    ("v6k", Some("arm_target_feature")),
    ("v6t2", Some("arm_target_feature")),
    ("v7", Some("arm_target_feature")),
    ("vfp2", Some("arm_target_feature")),
    ("vfp3", Some("arm_target_feature")),
    ("vfp4", Some("arm_target_feature")),
];

const AARCH64_WHITELIST: &[(&str, Option<&str>)] = &[
    ("fp", Some("aarch64_target_feature")),
    ("neon", Some("aarch64_target_feature")),
    ("sve", Some("aarch64_target_feature")),
    ("crc", Some("aarch64_target_feature")),
    ("crypto", Some("aarch64_target_feature")),
    ("ras", Some("aarch64_target_feature")),
    ("lse", Some("aarch64_target_feature")),
    ("rdm", Some("aarch64_target_feature")),
    ("fp16", Some("aarch64_target_feature")),
    ("rcpc", Some("aarch64_target_feature")),
    ("dotprod", Some("aarch64_target_feature")),
    ("v8.1a", Some("aarch64_target_feature")),
    ("v8.2a", Some("aarch64_target_feature")),
    ("v8.3a", Some("aarch64_target_feature")),
];

const X86_WHITELIST: &[(&str, Option<&str>)] = &[
    ("adx", Some("adx_target_feature")),
    ("aes", None),
    ("avx", None),
    ("avx2", None),
    ("avx512bw", Some("avx512_target_feature")),
    ("avx512cd", Some("avx512_target_feature")),
    ("avx512dq", Some("avx512_target_feature")),
    ("avx512er", Some("avx512_target_feature")),
    ("avx512f", Some("avx512_target_feature")),
    ("avx512ifma", Some("avx512_target_feature")),
    ("avx512pf", Some("avx512_target_feature")),
    ("avx512vbmi", Some("avx512_target_feature")),
    ("avx512vl", Some("avx512_target_feature")),
    ("avx512vpopcntdq", Some("avx512_target_feature")),
    ("bmi1", None),
    ("bmi2", None),
    ("cmpxchg16b", Some("cmpxchg16b_target_feature")),
    ("fma", None),
    ("fxsr", None),
    ("lzcnt", None),
    ("mmx", Some("mmx_target_feature")),
    ("movbe", Some("movbe_target_feature")),
    ("pclmulqdq", None),
    ("popcnt", None),
    ("rdrand", None),
    ("rdseed", None),
    ("sha", None),
    ("sse", None),
    ("sse2", None),
    ("sse3", None),
    ("sse4.1", None),
    ("sse4.2", None),
    ("sse4a", Some("sse4a_target_feature")),
    ("ssse3", None),
    ("tbm", Some("tbm_target_feature")),
    ("xsave", None),
    ("xsavec", None),
    ("xsaveopt", None),
    ("xsaves", None),
];

const HEXAGON_WHITELIST: &[(&str, Option<&str>)] = &[
    ("hvx", Some("hexagon_target_feature")),
    ("hvx-double", Some("hexagon_target_feature")),
];

const POWERPC_WHITELIST: &[(&str, Option<&str>)] = &[
    ("altivec", Some("powerpc_target_feature")),
    ("power8-altivec", Some("powerpc_target_feature")),
    ("power9-altivec", Some("powerpc_target_feature")),
    ("power8-vector", Some("powerpc_target_feature")),
    ("power9-vector", Some("powerpc_target_feature")),
    ("vsx", Some("powerpc_target_feature")),
];

const MIPS_WHITELIST: &[(&str, Option<&str>)] = &[
    ("fp64", Some("mips_target_feature")),
    ("msa", Some("mips_target_feature")),
];

const WASM_WHITELIST: &[(&str, Option<&str>)] = &[
    ("simd128", Some("wasm_target_feature")),
    ("atomics", Some("wasm_target_feature")),
];

/// When rustdoc is running, provide a list of all known features so that all their respective
/// primitives may be documented.
///
/// IMPORTANT: If you're adding another whitelist to the above lists, make sure to add it to this
/// iterator!
pub fn all_known_features() -> impl Iterator<Item=(&'static str, Option<&'static str>)> {
    ARM_WHITELIST.iter().cloned()
        .chain(AARCH64_WHITELIST.iter().cloned())
        .chain(X86_WHITELIST.iter().cloned())
        .chain(HEXAGON_WHITELIST.iter().cloned())
        .chain(POWERPC_WHITELIST.iter().cloned())
        .chain(MIPS_WHITELIST.iter().cloned())
        .chain(WASM_WHITELIST.iter().cloned())
}

pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str {
    let arch = if sess.target.target.arch == "x86_64" {
        "x86"
    } else {
        &*sess.target.target.arch
    };
    match (arch, s) {
        ("x86", "pclmulqdq") => "pclmul",
        ("x86", "rdrand") => "rdrnd",
        ("x86", "bmi1") => "bmi",
        ("x86", "cmpxchg16b") => "cx16",
        ("aarch64", "fp") => "fp-armv8",
        ("aarch64", "fp16") => "fullfp16",
        (_, s) => s,
    }
}

pub fn target_features(sess: &Session) -> Vec<Symbol> {
    let target_machine = create_informational_target_machine(sess, true);
    target_feature_whitelist(sess)
        .iter()
        .filter_map(|&(feature, gate)| {
            if UnstableFeatures::from_environment().is_nightly_build() || gate.is_none() {
                Some(feature)
            } else {
                None
            }
        })
        .filter(|feature| {
            let llvm_feature = to_llvm_feature(sess, feature);
            let cstr = CString::new(llvm_feature).unwrap();
            unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) }
        })
        .map(|feature| Symbol::intern(feature)).collect()
}

pub fn target_feature_whitelist(sess: &Session)
    -> &'static [(&'static str, Option<&'static str>)]
{
    match &*sess.target.target.arch {
        "arm" => ARM_WHITELIST,
        "aarch64" => AARCH64_WHITELIST,
        "x86" | "x86_64" => X86_WHITELIST,
        "hexagon" => HEXAGON_WHITELIST,
        "mips" | "mips64" => MIPS_WHITELIST,
        "powerpc" | "powerpc64" => POWERPC_WHITELIST,
        // wasm32 on emscripten does not support these target features
        "wasm32" if !sess.target.target.options.is_like_emscripten => WASM_WHITELIST,
        _ => &[],
    }
}

pub fn print_version() {
    // Can be called without initializing LLVM
    unsafe {
        println!("LLVM version: {}.{}",
                 llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor());
    }
}

pub fn get_major_version() -> u32 {
    unsafe { llvm::LLVMRustVersionMajor() }
}

pub fn print_passes() {
    // Can be called without initializing LLVM
    unsafe { llvm::LLVMRustPrintPasses(); }
}

pub(crate) fn print(req: PrintRequest, sess: &Session) {
    require_inited();
    let tm = create_informational_target_machine(sess, true);
    unsafe {
        match req {
            PrintRequest::TargetCPUs => llvm::LLVMRustPrintTargetCPUs(tm),
            PrintRequest::TargetFeatures => llvm::LLVMRustPrintTargetFeatures(tm),
            _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req),
        }
    }
}

pub fn target_cpu(sess: &Session) -> &str {
    let name = match sess.opts.cg.target_cpu {
        Some(ref s) => &**s,
        None => &*sess.target.target.options.cpu
    };
    if name != "native" {
        return name
    }

    unsafe {
        let mut len = 0;
        let ptr = llvm::LLVMRustGetHostCPUName(&mut len);
        str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap()
    }
}
<commit_msg>Whitelist the ARM v8 target-feature<commit_after>use syntax_pos::symbol::Symbol;
use back::write::create_informational_target_machine;
use llvm;
use rustc::session::Session;
use rustc::session::config::PrintRequest;
use rustc_target::spec::MergeFunctions;
use libc::c_int;
use std::ffi::CString;
use syntax::feature_gate::UnstableFeatures;

use std::str;
use std::slice;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Once;

static POISONED: AtomicBool = AtomicBool::new(false);
static INIT: Once = Once::new();

pub(crate) fn init(sess: &Session) {
    unsafe {
        // Before we touch LLVM, make sure that multithreading is enabled.
        INIT.call_once(|| {
            if llvm::LLVMStartMultithreaded() != 1 {
                // use an extra bool to make sure that all future usage of LLVM
                // cannot proceed despite the Once not running more than once.
                POISONED.store(true, Ordering::SeqCst);
            }

            configure_llvm(sess);
        });

        if POISONED.load(Ordering::SeqCst) {
            bug!("couldn't enable multi-threaded LLVM");
        }
    }
}

fn require_inited() {
    INIT.call_once(|| bug!("llvm is not initialized"));
    if POISONED.load(Ordering::SeqCst) {
        bug!("couldn't enable multi-threaded LLVM");
    }
}

unsafe fn configure_llvm(sess: &Session) {
    let n_args = sess.opts.cg.llvm_args.len();
    let mut llvm_c_strs = Vec::with_capacity(n_args + 1);
    let mut llvm_args = Vec::with_capacity(n_args + 1);

    llvm::LLVMRustInstallFatalErrorHandler();

    {
        let mut add = |arg: &str| {
            let s = CString::new(arg).unwrap();
            llvm_args.push(s.as_ptr());
            llvm_c_strs.push(s);
        };
        add("rustc"); // fake program name
        if sess.time_llvm_passes() { add("-time-passes"); }
        if sess.print_llvm_passes() { add("-debug-pass=Structure"); }
        if sess.opts.debugging_opts.disable_instrumentation_preinliner {
            add("-disable-preinline");
        }
        if llvm::LLVMRustIsRustLLVM() {
            match sess.opts.debugging_opts.merge_functions
                  .unwrap_or(sess.target.target.options.merge_functions) {
                MergeFunctions::Disabled |
                MergeFunctions::Trampolines => {}
                MergeFunctions::Aliases => {
                    add("-mergefunc-use-aliases");
                }
            }
        }

        // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes
        // during inlining. Unfortunately these may block other optimizations.
        add("-preserve-alignment-assumptions-during-inlining=false");

        for arg in &sess.opts.cg.llvm_args {
            add(&(*arg));
        }
    }

    llvm::LLVMInitializePasses();

    ::rustc_llvm::initialize_available_targets();

    llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int,
                                 llvm_args.as_ptr());
}

// WARNING: the features after applying `to_llvm_feature` must be known
// to LLVM or the feature detection code will walk past the end of the feature
// array, leading to crashes.

const ARM_WHITELIST: &[(&str, Option<&str>)] = &[
    ("aclass", Some("arm_target_feature")),
    ("mclass", Some("arm_target_feature")),
    ("rclass", Some("arm_target_feature")),
    ("dsp", Some("arm_target_feature")),
    ("neon", Some("arm_target_feature")),
    ("v5te", Some("arm_target_feature")),
    ("v6", Some("arm_target_feature")),
    ("v6k", Some("arm_target_feature")),
    ("v6t2", Some("arm_target_feature")),
    ("v7", Some("arm_target_feature")),
    ("v8", Some("arm_target_feature")),
    ("vfp2", Some("arm_target_feature")),
    ("vfp3", Some("arm_target_feature")),
    ("vfp4", Some("arm_target_feature")),
];

const AARCH64_WHITELIST: &[(&str, Option<&str>)] = &[
    ("fp", Some("aarch64_target_feature")),
    ("neon", Some("aarch64_target_feature")),
    ("sve", Some("aarch64_target_feature")),
    ("crc", Some("aarch64_target_feature")),
    ("crypto", Some("aarch64_target_feature")),
    ("ras", Some("aarch64_target_feature")),
    ("lse", Some("aarch64_target_feature")),
    ("rdm", Some("aarch64_target_feature")),
    ("fp16", Some("aarch64_target_feature")),
    ("rcpc", Some("aarch64_target_feature")),
    ("dotprod", Some("aarch64_target_feature")),
    ("v8.1a", Some("aarch64_target_feature")),
    ("v8.2a", Some("aarch64_target_feature")),
    ("v8.3a", Some("aarch64_target_feature")),
];

const X86_WHITELIST: &[(&str, Option<&str>)] = &[
    ("adx", Some("adx_target_feature")),
    ("aes", None),
    ("avx", None),
    ("avx2", None),
    ("avx512bw", Some("avx512_target_feature")),
    ("avx512cd", Some("avx512_target_feature")),
    ("avx512dq", Some("avx512_target_feature")),
    ("avx512er", Some("avx512_target_feature")),
    ("avx512f", Some("avx512_target_feature")),
    ("avx512ifma", Some("avx512_target_feature")),
    ("avx512pf", Some("avx512_target_feature")),
    ("avx512vbmi", Some("avx512_target_feature")),
    ("avx512vl", Some("avx512_target_feature")),
    ("avx512vpopcntdq", Some("avx512_target_feature")),
    ("bmi1", None),
    ("bmi2", None),
    ("cmpxchg16b", Some("cmpxchg16b_target_feature")),
    ("fma", None),
    ("fxsr", None),
    ("lzcnt", None),
    ("mmx", Some("mmx_target_feature")),
    ("movbe", Some("movbe_target_feature")),
    ("pclmulqdq", None),
    ("popcnt", None),
    ("rdrand", None),
    ("rdseed", None),
    ("sha", None),
    ("sse", None),
    ("sse2", None),
    ("sse3", None),
    ("sse4.1", None),
    ("sse4.2", None),
    ("sse4a", Some("sse4a_target_feature")),
    ("ssse3", None),
    ("tbm", Some("tbm_target_feature")),
    ("xsave", None),
    ("xsavec", None),
    ("xsaveopt", None),
    ("xsaves", None),
];

const HEXAGON_WHITELIST: &[(&str, Option<&str>)] = &[
    ("hvx", Some("hexagon_target_feature")),
    ("hvx-double", Some("hexagon_target_feature")),
];

const POWERPC_WHITELIST: &[(&str, Option<&str>)] = &[
    ("altivec", Some("powerpc_target_feature")),
    ("power8-altivec", Some("powerpc_target_feature")),
    ("power9-altivec", Some("powerpc_target_feature")),
    ("power8-vector", Some("powerpc_target_feature")),
    ("power9-vector", Some("powerpc_target_feature")),
    ("vsx", Some("powerpc_target_feature")),
];

const MIPS_WHITELIST: &[(&str, Option<&str>)] = &[
    ("fp64", Some("mips_target_feature")),
    ("msa", Some("mips_target_feature")),
];

const WASM_WHITELIST: &[(&str, Option<&str>)] = &[
    ("simd128", Some("wasm_target_feature")),
    ("atomics", Some("wasm_target_feature")),
];

/// When rustdoc is running, provide a list of all known features so that all their respective
/// primitives may be documented.
///
/// IMPORTANT: If you're adding another whitelist to the above lists, make sure to add it to this
/// iterator!
pub fn all_known_features() -> impl Iterator<Item=(&'static str, Option<&'static str>)> {
    ARM_WHITELIST.iter().cloned()
        .chain(AARCH64_WHITELIST.iter().cloned())
        .chain(X86_WHITELIST.iter().cloned())
        .chain(HEXAGON_WHITELIST.iter().cloned())
        .chain(POWERPC_WHITELIST.iter().cloned())
        .chain(MIPS_WHITELIST.iter().cloned())
        .chain(WASM_WHITELIST.iter().cloned())
}

pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str {
    let arch = if sess.target.target.arch == "x86_64" {
        "x86"
    } else {
        &*sess.target.target.arch
    };
    match (arch, s) {
        ("x86", "pclmulqdq") => "pclmul",
        ("x86", "rdrand") => "rdrnd",
        ("x86", "bmi1") => "bmi",
        ("x86", "cmpxchg16b") => "cx16",
        ("aarch64", "fp") => "fp-armv8",
        ("aarch64", "fp16") => "fullfp16",
        (_, s) => s,
    }
}

pub fn target_features(sess: &Session) -> Vec<Symbol> {
    let target_machine = create_informational_target_machine(sess, true);
    target_feature_whitelist(sess)
        .iter()
        .filter_map(|&(feature, gate)| {
            if UnstableFeatures::from_environment().is_nightly_build() || gate.is_none() {
                Some(feature)
            } else {
                None
            }
        })
        .filter(|feature| {
            let llvm_feature = to_llvm_feature(sess, feature);
            let cstr = CString::new(llvm_feature).unwrap();
            unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) }
        })
        .map(|feature| Symbol::intern(feature)).collect()
}

pub fn target_feature_whitelist(sess: &Session)
    -> &'static [(&'static str, Option<&'static str>)]
{
    match &*sess.target.target.arch {
        "arm" => ARM_WHITELIST,
        "aarch64" => AARCH64_WHITELIST,
        "x86" | "x86_64" => X86_WHITELIST,
        "hexagon" => HEXAGON_WHITELIST,
        "mips" | "mips64" => MIPS_WHITELIST,
        "powerpc" | "powerpc64" => POWERPC_WHITELIST,
        // wasm32 on emscripten does not support these target features
        "wasm32" if !sess.target.target.options.is_like_emscripten => WASM_WHITELIST,
        _ => &[],
    }
}

pub fn print_version() {
    // Can be called without initializing LLVM
    unsafe {
        println!("LLVM version: {}.{}",
                 llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor());
    }
}

pub fn get_major_version() -> u32 {
    unsafe { llvm::LLVMRustVersionMajor() }
}

pub fn print_passes() {
    // Can be called without initializing LLVM
    unsafe { llvm::LLVMRustPrintPasses(); }
}

pub(crate) fn print(req: PrintRequest, sess: &Session) {
    require_inited();
    let tm = create_informational_target_machine(sess, true);
    unsafe {
        match req {
            PrintRequest::TargetCPUs => llvm::LLVMRustPrintTargetCPUs(tm),
            PrintRequest::TargetFeatures => llvm::LLVMRustPrintTargetFeatures(tm),
            _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req),
        }
    }
}

pub fn target_cpu(sess: &Session) -> &str {
    let name = match sess.opts.cg.target_cpu {
        Some(ref s) => &**s,
        None => &*sess.target.target.options.cpu
    };
    if name != "native" {
        return name
    }

    unsafe {
        let mut len = 0;
        let ptr = llvm::LLVMRustGetHostCPUName(&mut len);
        str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap()
    }
}
<|endoftext|>