<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{cargo::Cargo, config::CargoConfig, installer::install_if_needed, Result};
use anyhow::anyhow;
use log::{info, warn};
use serde::Deserialize;
use std::{
    env::var_os,
    path::{Path, PathBuf},
    process::Command,
};

/// The number of directories between the project root and the root of this crate.
pub const X_DEPTH: usize = 2;

/// Returns the project root. TODO: switch uses to XCoreContext::project_root instead)
pub fn project_root() -> &'static Path {
    Path::new(&env!("CARGO_MANIFEST_DIR"))
        .ancestors()
        .nth(X_DEPTH)
        .unwrap()
}

pub fn locate_project(cargo_config: &CargoConfig) -> Result<PathBuf> {
    #[derive(Deserialize)]
    struct LocateProject {
        root: PathBuf,
    };

    let output = Cargo::new(cargo_config, "locate-project", false).run_with_output()?;
    Ok(serde_json::from_slice::<LocateProject>(&output)?.root)
}

pub fn project_is_root(cargo_config: &CargoConfig) -> Result<bool> {
    let mut project = locate_project(cargo_config)?;
    project.pop();

    Ok(project == project_root())
}

fn stop_sccache_server() {
    let mut sccache = Command::new("sccache");
    sccache.arg("--stop-server");
    let result = sccache.output();
    if let Ok(output) = result {
        if output.status.success() {
            info!("Stopped already running sccache.");
        }
    }
}

pub fn apply_sccache_if_possible<'a>(
    cargo_config: &'a CargoConfig,
) -> Result<Vec<(&'a str, Option<String>)>> {
    let mut envs = vec![];

    if var_os("SKIP_SCCACHE").is_none() && cargo_config.sccache.is_some() {
        if let Some(sccach_config) = &cargo_config.sccache {
            // Are we work on items in the right location:
            // See: https://github.com/mozilla/sccache#known-caveats
            let correct_location = var_os("CARGO_HOME")
                .unwrap_or_default()
                .to_str()
                .unwrap_or_default()
                == sccach_config.required_cargo_home
                && sccach_config.required_git_home == project_root().to_str().unwrap_or_default();
            if !correct_location {
                warn!("You will not benefit from sccache in this build!!!");
                warn!(
                    "To get the best experience, please move your libra source code to {} and your set your CARGO_HOME to be {}, simply export it in your .profile or .bash_rc",
                    &sccach_config.required_git_home, &sccach_config.required_cargo_home
                );
                warn!(
                    "Current libra root is '{}',  and current CARGO_HOME is '{}'",
                    project_root().to_str().unwrap_or_default(),
                    var_os("CARGO_HOME").unwrap_or_default().to_string_lossy()
                );
            } else {
                if !install_if_needed(cargo_config, "sccache", &sccach_config.installer) {
                    return Err(anyhow!("Failed to install sccache, bailing"));
                }
                stop_sccache_server();
                envs.push(("RUSTC_WRAPPER", Some("sccache".to_owned())));
                envs.push(("CARGO_INCREMENTAL", Some("false".to_owned())));
                envs.push(("SCCACHE_BUCKET", Some(sccach_config.bucket.to_owned())));
                if let Some(ssl) = &sccach_config.ssl {
                    envs.push((
                        "SCCACHE_S3_USE_SSL",
                        if *ssl {
                            Some("true".to_owned())
                        } else {
                            Some("false".to_owned())
                        },
                    ));
                }

                if let Some(url) = &sccach_config.endpoint {
                    envs.push(("SCCACHE_ENDPOINT", Some(url.to_owned())));
                }

                if let Some(extra_envs) = &sccach_config.envs {
                    for (key, value) in extra_envs {
                        envs.push((key, Some(value.to_owned())));
                    }
                }

                if let Some(region) = &sccach_config.region {
                    envs.push(("SCCACHE_REGION", Some(region.to_owned())));
                }

                if let Some(prefix) = &sccach_config.prefix {
                    envs.push(("SCCACHE_S3_KEY_PREFIX", Some(prefix.to_owned())));
                }
                let access_key_id = if let Some(val) = var_os("SCCACHE_AWS_ACCESS_KEY_ID") {
                    Some(val.to_string_lossy().to_string())
                } else {
                    None
                };
                let access_key_secret = if let Some(val) = var_os("SCCACHE_AWS_SECRET_ACCESS_KEY") {
                    Some(val.to_string_lossy().to_string())
                } else {
                    None
                };
                if (access_key_id.is_none() || access_key_secret.is_some())
                    && sccach_config.public.unwrap_or(true)
                {
                    envs.push(("SCCACHE_S3_PUBLIC", Some("true".to_owned())))
                }
                //Note: that this is also used to _unset_ AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY
                envs.push(("AWS_ACCESS_KEY_ID", access_key_id));
                envs.push(("AWS_SECRET_ACCESS_KEY", access_key_secret));
            }
        }
    }
    Ok(envs)
}
<commit_msg>[sccache] fixing shameful logic bug<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{cargo::Cargo, config::CargoConfig, installer::install_if_needed, Result};
use anyhow::anyhow;
use log::{info, warn};
use serde::Deserialize;
use std::{
    env::var_os,
    path::{Path, PathBuf},
    process::Command,
};

/// The number of directories between the project root and the root of this crate.
pub const X_DEPTH: usize = 2;

/// Returns the project root. TODO: switch uses to XCoreContext::project_root instead)
pub fn project_root() -> &'static Path {
    Path::new(&env!("CARGO_MANIFEST_DIR"))
        .ancestors()
        .nth(X_DEPTH)
        .unwrap()
}

pub fn locate_project(cargo_config: &CargoConfig) -> Result<PathBuf> {
    #[derive(Deserialize)]
    struct LocateProject {
        root: PathBuf,
    };

    let output = Cargo::new(cargo_config, "locate-project", false).run_with_output()?;
    Ok(serde_json::from_slice::<LocateProject>(&output)?.root)
}

pub fn project_is_root(cargo_config: &CargoConfig) -> Result<bool> {
    let mut project = locate_project(cargo_config)?;
    project.pop();

    Ok(project == project_root())
}

fn stop_sccache_server() {
    let mut sccache = Command::new("sccache");
    sccache.arg("--stop-server");
    let result = sccache.output();
    if let Ok(output) = result {
        if output.status.success() {
            info!("Stopped already running sccache.");
        }
    }
}

pub fn apply_sccache_if_possible<'a>(
    cargo_config: &'a CargoConfig,
) -> Result<Vec<(&'a str, Option<String>)>> {
    let mut envs = vec![];

    if var_os("SKIP_SCCACHE").is_none() && cargo_config.sccache.is_some() {
        if let Some(sccach_config) = &cargo_config.sccache {
            // Are we work on items in the right location:
            // See: https://github.com/mozilla/sccache#known-caveats
            let correct_location = var_os("CARGO_HOME")
                .unwrap_or_default()
                .to_str()
                .unwrap_or_default()
                == sccach_config.required_cargo_home
                && sccach_config.required_git_home == project_root().to_str().unwrap_or_default();
            if !correct_location {
                warn!("You will not benefit from sccache in this build!!!");
                warn!(
                    "To get the best experience, please move your libra source code to {} and your set your CARGO_HOME to be {}, simply export it in your .profile or .bash_rc",
                    &sccach_config.required_git_home, &sccach_config.required_cargo_home
                );
                warn!(
                    "Current libra root is '{}',  and current CARGO_HOME is '{}'",
                    project_root().to_str().unwrap_or_default(),
                    var_os("CARGO_HOME").unwrap_or_default().to_string_lossy()
                );
            } else {
                if !install_if_needed(cargo_config, "sccache", &sccach_config.installer) {
                    return Err(anyhow!("Failed to install sccache, bailing"));
                }
                stop_sccache_server();
                envs.push(("RUSTC_WRAPPER", Some("sccache".to_owned())));
                envs.push(("CARGO_INCREMENTAL", Some("false".to_owned())));
                envs.push(("SCCACHE_BUCKET", Some(sccach_config.bucket.to_owned())));
                if let Some(ssl) = &sccach_config.ssl {
                    envs.push((
                        "SCCACHE_S3_USE_SSL",
                        if *ssl {
                            Some("true".to_owned())
                        } else {
                            Some("false".to_owned())
                        },
                    ));
                }

                if let Some(url) = &sccach_config.endpoint {
                    envs.push(("SCCACHE_ENDPOINT", Some(url.to_owned())));
                }

                if let Some(extra_envs) = &sccach_config.envs {
                    for (key, value) in extra_envs {
                        envs.push((key, Some(value.to_owned())));
                    }
                }

                if let Some(region) = &sccach_config.region {
                    envs.push(("SCCACHE_REGION", Some(region.to_owned())));
                }

                if let Some(prefix) = &sccach_config.prefix {
                    envs.push(("SCCACHE_S3_KEY_PREFIX", Some(prefix.to_owned())));
                }
                let access_key_id = if let Some(val) = var_os("SCCACHE_AWS_ACCESS_KEY_ID") {
                    Some(val.to_string_lossy().to_string())
                } else {
                    None
                };
                let access_key_secret = if let Some(val) = var_os("SCCACHE_AWS_SECRET_ACCESS_KEY") {
                    Some(val.to_string_lossy().to_string())
                } else {
                    None
                };
                // if either the access or secret key is not set, attempt to perform a public read.
                // do not set this flag if attempting to write, as it will prevent the use of the aws creds.
                if (access_key_id.is_none() || access_key_secret.is_none())
                    && sccach_config.public.unwrap_or(true)
                {
                    envs.push(("SCCACHE_S3_PUBLIC", Some("true".to_owned())))
                }
                //Note: that this is also used to _unset_ AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY
                envs.push(("AWS_ACCESS_KEY_ID", access_key_id));
                envs.push(("AWS_SECRET_ACCESS_KEY", access_key_secret));
            }
        }
    }
    Ok(envs)
}
<|endoftext|>


<commit_before>//! cargo_check provides the functionality needed to run `cargo check` or
//! another compatible command (f.x. clippy) in a background thread and provide
//! LSP diagnostics based on the output of the command.

use std::{
    fmt,
    io::{self, BufReader},
    ops,
    path::PathBuf,
    process::{self, Command, Stdio},
    time::Duration,
};

use crossbeam_channel::{never, select, unbounded, Receiver, Sender};

pub use cargo_metadata::diagnostic::{
    Applicability, Diagnostic, DiagnosticLevel, DiagnosticSpan, DiagnosticSpanMacroExpansion,
};

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FlycheckConfig {
    CargoCommand {
        command: String,
        all_targets: bool,
        all_features: bool,
        features: Vec<String>,
        extra_args: Vec<String>,
    },
    CustomCommand {
        command: String,
        args: Vec<String>,
    },
}

impl fmt::Display for FlycheckConfig {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            FlycheckConfig::CargoCommand { command, .. } => write!(f, "cargo {}", command),
            FlycheckConfig::CustomCommand { command, args } => {
                write!(f, "{} {}", command, args.join(" "))
            }
        }
    }
}

/// Flycheck wraps the shared state and communication machinery used for
/// running `cargo check` (or other compatible command) and providing
/// diagnostics based on the output.
/// The spawned thread is shut down when this struct is dropped.
#[derive(Debug)]
pub struct FlycheckHandle {
    // XXX: drop order is significant
    cmd_send: Sender<Restart>,
    handle: jod_thread::JoinHandle,
}

impl FlycheckHandle {
    pub fn spawn(
        sender: Box<dyn Fn(Message) + Send>,
        config: FlycheckConfig,
        workspace_root: PathBuf,
    ) -> FlycheckHandle {
        let (cmd_send, cmd_recv) = unbounded::<Restart>();
        let handle = jod_thread::spawn(move || {
            FlycheckActor::new(sender, config, workspace_root).run(cmd_recv);
        });
        FlycheckHandle { cmd_send, handle }
    }

    /// Schedule a re-start of the cargo check worker.
    pub fn update(&self) {
        self.cmd_send.send(Restart).unwrap();
    }
}

#[derive(Debug)]
pub enum Message {
    /// Request adding a diagnostic with fixes included to a file
    AddDiagnostic { workspace_root: PathBuf, diagnostic: Diagnostic },

    /// Request check progress notification to client
    Progress(Progress),
}

#[derive(Debug)]
pub enum Progress {
    DidStart,
    DidCheckCrate(String),
    DidFinish,
    DidCancel,
}

struct Restart;

struct FlycheckActor {
    sender: Box<dyn Fn(Message) + Send>,
    config: FlycheckConfig,
    workspace_root: PathBuf,
    /// WatchThread exists to wrap around the communication needed to be able to
    /// run `cargo check` without blocking. Currently the Rust standard library
    /// doesn't provide a way to read sub-process output without blocking, so we
    /// have to wrap sub-processes output handling in a thread and pass messages
    /// back over a channel.
    // XXX: drop order is significant
    check_process: Option<(Receiver<cargo_metadata::Message>, jod_thread::JoinHandle)>,
}

enum Event {
    Restart(Restart),
    CheckEvent(Option<cargo_metadata::Message>),
}

impl FlycheckActor {
    fn new(
        sender: Box<dyn Fn(Message) + Send>,
        config: FlycheckConfig,
        workspace_root: PathBuf,
    ) -> FlycheckActor {
        FlycheckActor { sender, config, workspace_root, check_process: None }
    }
    fn next_event(&self, inbox: &Receiver<Restart>) -> Option<Event> {
        let check_chan = self.check_process.as_ref().map(|(chan, _thread)| chan);
        select! {
            recv(inbox) -> msg => msg.ok().map(Event::Restart),
            recv(check_chan.unwrap_or(&never())) -> msg => Some(Event::CheckEvent(msg.ok())),
        }
    }
    fn run(&mut self, inbox: Receiver<Restart>) {
        while let Some(event) = self.next_event(&inbox) {
            match event {
                Event::Restart(Restart) => {
                    while let Ok(Restart) = inbox.recv_timeout(Duration::from_millis(50)) {}
                    self.cancel_check_process();
                    self.check_process = Some(self.start_check_process());
                    self.send(Message::Progress(Progress::DidStart));
                }
                Event::CheckEvent(None) => {
                    // Watcher finished, replace it with a never channel to
                    // avoid busy-waiting.
                    assert!(self.check_process.take().is_some());
                    self.send(Message::Progress(Progress::DidFinish));
                }
                Event::CheckEvent(Some(message)) => match message {
                    cargo_metadata::Message::CompilerArtifact(msg) => {
                        self.send(Message::Progress(Progress::DidCheckCrate(msg.target.name)));
                    }

                    cargo_metadata::Message::CompilerMessage(msg) => {
                        self.send(Message::AddDiagnostic {
                            workspace_root: self.workspace_root.clone(),
                            diagnostic: msg.message,
                        });
                    }

                    cargo_metadata::Message::BuildScriptExecuted(_)
                    | cargo_metadata::Message::BuildFinished(_)
                    | cargo_metadata::Message::TextLine(_)
                    | cargo_metadata::Message::Unknown => {}
                },
            }
        }
        // If we rerun the thread, we need to discard the previous check results first
        self.cancel_check_process();
    }
    fn cancel_check_process(&mut self) {
        if self.check_process.take().is_some() {
            self.send(Message::Progress(Progress::DidCancel));
        }
    }
    fn start_check_process(&self) -> (Receiver<cargo_metadata::Message>, jod_thread::JoinHandle) {
        let mut cmd = match &self.config {
            FlycheckConfig::CargoCommand {
                command,
                all_targets,
                all_features,
                extra_args,
                features,
            } => {
                let mut cmd = Command::new(ra_toolchain::cargo());
                cmd.arg(command);
                cmd.args(&["--workspace", "--message-format=json", "--manifest-path"])
                    .arg(self.workspace_root.join("Cargo.toml"));
                if *all_targets {
                    cmd.arg("--all-targets");
                }
                if *all_features {
                    cmd.arg("--all-features");
                } else if !features.is_empty() {
                    cmd.arg("--features");
                    cmd.arg(features.join(" "));
                }
                cmd.args(extra_args);
                cmd
            }
            FlycheckConfig::CustomCommand { command, args } => {
                let mut cmd = Command::new(command);
                cmd.args(args);
                cmd
            }
        };
        cmd.current_dir(&self.workspace_root);

        let (message_send, message_recv) = unbounded();
        let thread = jod_thread::spawn(move || {
            // If we trigger an error here, we will do so in the loop instead,
            // which will break out of the loop, and continue the shutdown
            let res = run_cargo(cmd, &mut |message| {
                // Skip certain kinds of messages to only spend time on what's useful
                match &message {
                    cargo_metadata::Message::CompilerArtifact(artifact) if artifact.fresh => {
                        return true
                    }
                    cargo_metadata::Message::BuildScriptExecuted(_)
                    | cargo_metadata::Message::Unknown => return true,
                    _ => {}
                }

                // if the send channel was closed, we want to shutdown
                message_send.send(message).is_ok()
            });

            if let Err(err) = res {
                // FIXME: make the `message_send` to be `Sender<Result<CheckEvent, CargoError>>`
                // to display user-caused misconfiguration errors instead of just logging them here
                log::error!("Cargo watcher failed {:?}", err);
            }
        });
        (message_recv, thread)
    }

    fn send(&self, check_task: Message) {
        (self.sender)(check_task)
    }
}

fn run_cargo(
    mut command: Command,
    on_message: &mut dyn FnMut(cargo_metadata::Message) -> bool,
) -> io::Result<()> {
    let child =
        command.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null()).spawn()?;
    let mut child = ChildKiller(child);

    // We manually read a line at a time, instead of using serde's
    // stream deserializers, because the deserializer cannot recover
    // from an error, resulting in it getting stuck, because we try to
    // be resillient against failures.
    //
    // Because cargo only outputs one JSON object per line, we can
    // simply skip a line if it doesn't parse, which just ignores any
    // erroneus output.
    let stdout = BufReader::new(child.stdout.take().unwrap());
    let mut read_at_least_one_message = false;
    for message in cargo_metadata::Message::parse_stream(stdout) {
        let message = match message {
            Ok(message) => message,
            Err(err) => {
                log::error!("Invalid json from cargo check, ignoring ({})", err);
                continue;
            }
        };

        read_at_least_one_message = true;

        if !on_message(message) {
            break;
        }
    }

    // It is okay to ignore the result, as it only errors if the process is already dead
    let _ = child.kill();

    let exit_status = child.wait()?;
    if !exit_status.success() && !read_at_least_one_message {
        // FIXME: Read the stderr to display the reason, see `read2()` reference in PR comment:
        // https://github.com/rust-analyzer/rust-analyzer/pull/3632#discussion_r395605298
        return Err(io::Error::new(
            io::ErrorKind::Other,
            format!(
                "the command produced no valid metadata (exit code: {:?}): {:?}",
                exit_status, command
            ),
        ));
    }

    Ok(())
}

struct ChildKiller(process::Child);

impl ops::Deref for ChildKiller {
    type Target = process::Child;
    fn deref(&self) -> &process::Child {
        &self.0
    }
}

impl ops::DerefMut for ChildKiller {
    fn deref_mut(&mut self) -> &mut process::Child {
        &mut self.0
    }
}

impl Drop for ChildKiller {
    fn drop(&mut self) {
        let _ = self.0.kill();
    }
}
<commit_msg>Rename<commit_after>//! cargo_check provides the functionality needed to run `cargo check` or
//! another compatible command (f.x. clippy) in a background thread and provide
//! LSP diagnostics based on the output of the command.

use std::{
    fmt,
    io::{self, BufReader},
    ops,
    path::PathBuf,
    process::{self, Command, Stdio},
    time::Duration,
};

use crossbeam_channel::{never, select, unbounded, Receiver, Sender};

pub use cargo_metadata::diagnostic::{
    Applicability, Diagnostic, DiagnosticLevel, DiagnosticSpan, DiagnosticSpanMacroExpansion,
};

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FlycheckConfig {
    CargoCommand {
        command: String,
        all_targets: bool,
        all_features: bool,
        features: Vec<String>,
        extra_args: Vec<String>,
    },
    CustomCommand {
        command: String,
        args: Vec<String>,
    },
}

impl fmt::Display for FlycheckConfig {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            FlycheckConfig::CargoCommand { command, .. } => write!(f, "cargo {}", command),
            FlycheckConfig::CustomCommand { command, args } => {
                write!(f, "{} {}", command, args.join(" "))
            }
        }
    }
}

/// Flycheck wraps the shared state and communication machinery used for
/// running `cargo check` (or other compatible command) and providing
/// diagnostics based on the output.
/// The spawned thread is shut down when this struct is dropped.
#[derive(Debug)]
pub struct FlycheckHandle {
    // XXX: drop order is significant
    cmd_send: Sender<Restart>,
    thread: jod_thread::JoinHandle,
}

impl FlycheckHandle {
    pub fn spawn(
        sender: Box<dyn Fn(Message) + Send>,
        config: FlycheckConfig,
        workspace_root: PathBuf,
    ) -> FlycheckHandle {
        let (cmd_send, cmd_recv) = unbounded::<Restart>();
        let thread = jod_thread::spawn(move || {
            FlycheckActor::new(sender, config, workspace_root).run(cmd_recv);
        });
        FlycheckHandle { cmd_send, thread }
    }

    /// Schedule a re-start of the cargo check worker.
    pub fn update(&self) {
        self.cmd_send.send(Restart).unwrap();
    }
}

#[derive(Debug)]
pub enum Message {
    /// Request adding a diagnostic with fixes included to a file
    AddDiagnostic { workspace_root: PathBuf, diagnostic: Diagnostic },

    /// Request check progress notification to client
    Progress(Progress),
}

#[derive(Debug)]
pub enum Progress {
    DidStart,
    DidCheckCrate(String),
    DidFinish,
    DidCancel,
}

struct Restart;

struct FlycheckActor {
    sender: Box<dyn Fn(Message) + Send>,
    config: FlycheckConfig,
    workspace_root: PathBuf,
    /// WatchThread exists to wrap around the communication needed to be able to
    /// run `cargo check` without blocking. Currently the Rust standard library
    /// doesn't provide a way to read sub-process output without blocking, so we
    /// have to wrap sub-processes output handling in a thread and pass messages
    /// back over a channel.
    // XXX: drop order is significant
    check_process: Option<(Receiver<cargo_metadata::Message>, jod_thread::JoinHandle)>,
}

enum Event {
    Restart(Restart),
    CheckEvent(Option<cargo_metadata::Message>),
}

impl FlycheckActor {
    fn new(
        sender: Box<dyn Fn(Message) + Send>,
        config: FlycheckConfig,
        workspace_root: PathBuf,
    ) -> FlycheckActor {
        FlycheckActor { sender, config, workspace_root, check_process: None }
    }
    fn next_event(&self, inbox: &Receiver<Restart>) -> Option<Event> {
        let check_chan = self.check_process.as_ref().map(|(chan, _thread)| chan);
        select! {
            recv(inbox) -> msg => msg.ok().map(Event::Restart),
            recv(check_chan.unwrap_or(&never())) -> msg => Some(Event::CheckEvent(msg.ok())),
        }
    }
    fn run(&mut self, inbox: Receiver<Restart>) {
        while let Some(event) = self.next_event(&inbox) {
            match event {
                Event::Restart(Restart) => {
                    while let Ok(Restart) = inbox.recv_timeout(Duration::from_millis(50)) {}
                    self.cancel_check_process();
                    self.check_process = Some(self.start_check_process());
                    self.send(Message::Progress(Progress::DidStart));
                }
                Event::CheckEvent(None) => {
                    // Watcher finished, replace it with a never channel to
                    // avoid busy-waiting.
                    assert!(self.check_process.take().is_some());
                    self.send(Message::Progress(Progress::DidFinish));
                }
                Event::CheckEvent(Some(message)) => match message {
                    cargo_metadata::Message::CompilerArtifact(msg) => {
                        self.send(Message::Progress(Progress::DidCheckCrate(msg.target.name)));
                    }

                    cargo_metadata::Message::CompilerMessage(msg) => {
                        self.send(Message::AddDiagnostic {
                            workspace_root: self.workspace_root.clone(),
                            diagnostic: msg.message,
                        });
                    }

                    cargo_metadata::Message::BuildScriptExecuted(_)
                    | cargo_metadata::Message::BuildFinished(_)
                    | cargo_metadata::Message::TextLine(_)
                    | cargo_metadata::Message::Unknown => {}
                },
            }
        }
        // If we rerun the thread, we need to discard the previous check results first
        self.cancel_check_process();
    }
    fn cancel_check_process(&mut self) {
        if self.check_process.take().is_some() {
            self.send(Message::Progress(Progress::DidCancel));
        }
    }
    fn start_check_process(&self) -> (Receiver<cargo_metadata::Message>, jod_thread::JoinHandle) {
        let mut cmd = match &self.config {
            FlycheckConfig::CargoCommand {
                command,
                all_targets,
                all_features,
                extra_args,
                features,
            } => {
                let mut cmd = Command::new(ra_toolchain::cargo());
                cmd.arg(command);
                cmd.args(&["--workspace", "--message-format=json", "--manifest-path"])
                    .arg(self.workspace_root.join("Cargo.toml"));
                if *all_targets {
                    cmd.arg("--all-targets");
                }
                if *all_features {
                    cmd.arg("--all-features");
                } else if !features.is_empty() {
                    cmd.arg("--features");
                    cmd.arg(features.join(" "));
                }
                cmd.args(extra_args);
                cmd
            }
            FlycheckConfig::CustomCommand { command, args } => {
                let mut cmd = Command::new(command);
                cmd.args(args);
                cmd
            }
        };
        cmd.current_dir(&self.workspace_root);

        let (message_send, message_recv) = unbounded();
        let thread = jod_thread::spawn(move || {
            // If we trigger an error here, we will do so in the loop instead,
            // which will break out of the loop, and continue the shutdown
            let res = run_cargo(cmd, &mut |message| {
                // Skip certain kinds of messages to only spend time on what's useful
                match &message {
                    cargo_metadata::Message::CompilerArtifact(artifact) if artifact.fresh => {
                        return true
                    }
                    cargo_metadata::Message::BuildScriptExecuted(_)
                    | cargo_metadata::Message::Unknown => return true,
                    _ => {}
                }

                // if the send channel was closed, we want to shutdown
                message_send.send(message).is_ok()
            });

            if let Err(err) = res {
                // FIXME: make the `message_send` to be `Sender<Result<CheckEvent, CargoError>>`
                // to display user-caused misconfiguration errors instead of just logging them here
                log::error!("Cargo watcher failed {:?}", err);
            }
        });
        (message_recv, thread)
    }

    fn send(&self, check_task: Message) {
        (self.sender)(check_task)
    }
}

fn run_cargo(
    mut command: Command,
    on_message: &mut dyn FnMut(cargo_metadata::Message) -> bool,
) -> io::Result<()> {
    let child =
        command.stdout(Stdio::piped()).stderr(Stdio::null()).stdin(Stdio::null()).spawn()?;
    let mut child = ChildKiller(child);

    // We manually read a line at a time, instead of using serde's
    // stream deserializers, because the deserializer cannot recover
    // from an error, resulting in it getting stuck, because we try to
    // be resillient against failures.
    //
    // Because cargo only outputs one JSON object per line, we can
    // simply skip a line if it doesn't parse, which just ignores any
    // erroneus output.
    let stdout = BufReader::new(child.stdout.take().unwrap());
    let mut read_at_least_one_message = false;
    for message in cargo_metadata::Message::parse_stream(stdout) {
        let message = match message {
            Ok(message) => message,
            Err(err) => {
                log::error!("Invalid json from cargo check, ignoring ({})", err);
                continue;
            }
        };

        read_at_least_one_message = true;

        if !on_message(message) {
            break;
        }
    }

    // It is okay to ignore the result, as it only errors if the process is already dead
    let _ = child.kill();

    let exit_status = child.wait()?;
    if !exit_status.success() && !read_at_least_one_message {
        // FIXME: Read the stderr to display the reason, see `read2()` reference in PR comment:
        // https://github.com/rust-analyzer/rust-analyzer/pull/3632#discussion_r395605298
        return Err(io::Error::new(
            io::ErrorKind::Other,
            format!(
                "the command produced no valid metadata (exit code: {:?}): {:?}",
                exit_status, command
            ),
        ));
    }

    Ok(())
}

struct ChildKiller(process::Child);

impl ops::Deref for ChildKiller {
    type Target = process::Child;
    fn deref(&self) -> &process::Child {
        &self.0
    }
}

impl ops::DerefMut for ChildKiller {
    fn deref_mut(&mut self) -> &mut process::Child {
        &mut self.0
    }
}

impl Drop for ChildKiller {
    fn drop(&mut self) {
        let _ = self.0.kill();
    }
}
<|endoftext|>


<commit_before>use chill;
use chill::{IntoDatabasePath, IntoViewPath};
use {DecodedTrack, User, Error, ImpossibleError};

mod fs;
mod jamendo;

pub struct Index;

impl Index {
    pub fn take_result<'a, P>(db: &'a chill::Client, db_path: P, result: Vec<DecodedTrack>) -> Result<(), Error> where P: IntoDatabasePath<'a> {
        let db_path = db_path.into_database_path()?;

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_tracks"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let tracks = res.rows();

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_albums"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let albums = res.rows();

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_artists"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let artists = res.rows();

        for track in result {

        }

        Ok(())
    }
}

pub trait Indexer<T> {
    fn index(&chill::Client, &User, &T) -> Result<(), Error>;
}
<commit_msg>feat(index): implement take_result<commit_after>use chill;
use chill::{IntoDatabasePath, IntoViewPath, DocumentId, DocumentIdRef};
use {DecodedTrack, User, Error, Artist, Album, Track, ImpossibleError};

mod fs;
mod jamendo;

pub struct Index;

impl Index {
    pub fn take_result<'a, P>(db: &'a chill::Client, db_path: P, result: Vec<DecodedTrack>) -> Result<(), Error> where P: IntoDatabasePath<'a> {
        let db_path = db_path.into_database_path()?;

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_tracks"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let mut tracks: Vec<(DocumentId, String)> = res.rows().iter()
            .map(|a| (DocumentId::from(a.document_path().document_id()), a.key().clone())).collect();

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_albums"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let mut albums: Vec<(DocumentId, String)> = res.rows().iter()
            .map(|a| (DocumentId::from(a.document_path().document_id()), a.key().clone())).collect();

        let res = db.execute_view::<String, Option<String>, _>((db_path, "cloudfm", "all_artists"))?.run()?;
        let res = res.as_unreduced().ok_or(ImpossibleError::ViewReduced)?;
        let mut artists: Vec<(DocumentId, String)> = res.rows().iter()
            .map(|a| (DocumentId::from(a.document_path().document_id()), a.key().clone())).collect();

        for track in result {

            // is_new works around https://github.com/rust-lang/rfcs/issues/811
            // vec can't be mutated inside match block
            let (artist_id, is_new) = match artists.iter().find(|&&(ref i, ref a)| a == &track.artist) {
                Some(&(ref i, _)) => (i.clone(), false),
                None => {
                    let (id, _) = db.create_document(db_path, &Artist { name: track.artist.clone() })?.run()?;
                    (id, true)
                }
            };
            if is_new {
                artists.push((artist_id.clone(), track.artist.clone()));
            };

            let (album_id, is_new) = match albums.iter().find(|&&(ref i, ref a)| a == &track.album) {
                Some(&(ref i, _)) => (i.clone(), false),
                None => {
                    let (id, _) = db.create_document(db_path, &Album { name: track.album.clone(), artist: artist_id.clone() })?.run()?;
                    (id, true)
                }
            };
            if is_new {
                albums.push((album_id.clone(), track.album.clone()));
            };

            let new_track = match tracks.iter().find(|&&(ref i, ref t)| t == &track.name) {
                Some(&(ref i, _)) => None,
                None => {
                    let track = Track {
                        name: track.name,
                        number: track.number,
                        uri: track.uri,
                        artist: artist_id,
                        album: album_id,
                    };
                    let (id, _) = db.create_document(db_path, &track)?.run()?;
                    Some((id, track.name))
                }
            };
            if let Some(t) = new_track {
                tracks.push(t);
            }

        }

        Ok(())
    }
}

pub trait Indexer<T> {
    fn index(&chill::Client, &User, &T) -> Result<(), Error>;
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use devtools_traits::{ConsoleMessage, LogLevel, ScriptToDevtoolsControlMsg};
use dom::bindings::codegen::Bindings::ConsoleBinding;
use dom::bindings::codegen::Bindings::ConsoleBinding::ConsoleMethods;
use dom::bindings::global::{GlobalField, GlobalRef};
use dom::bindings::js::Root;
use dom::bindings::reflector::{Reflector, reflect_dom_object};
use util::str::DOMString;

// https://developer.mozilla.org/en-US/docs/Web/API/Console
#[dom_struct]
pub struct Console {
    reflector_: Reflector,
    global: GlobalField,
}

impl Console {
    fn new_inherited(global: GlobalRef) -> Console {
        Console {
            reflector_: Reflector::new(),
            global: GlobalField::from_rooted(&global),
        }
    }

    pub fn new(global: GlobalRef) -> Root<Console> {
        reflect_dom_object(box Console::new_inherited(global),
                           global,
                           ConsoleBinding::Wrap)
    }
}

impl ConsoleMethods for Console {
    // https://developer.mozilla.org/en-US/docs/Web/API/Console/log
    fn Log(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Log, message));
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console
    fn Debug(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Debug, message));
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/info
    fn Info(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Info, message));
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/warn
    fn Warn(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Warn, message));
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/error
    fn Error(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Error, message));
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/assert
    fn Assert(&self, condition: bool, message: Option<DOMString>) {
        if !condition {
            let message = message.unwrap_or_else(|| DOMString::from("no message"));
            println!("Assertion failed: {}", message);
            propagate_console_msg(&self, prepare_message(LogLevel::Error, message));
        }
    }
}

fn prepare_message(logLevel: LogLevel, message: DOMString) -> ConsoleMessage {
    // TODO: Sending fake values for filename, lineNumber and columnNumber in LogMessage; adjust later
    ConsoleMessage {
        message: String::from(message),
        logLevel: logLevel,
        filename: "test".to_owned(),
        lineNumber: 1,
        columnNumber: 1,
    }
}

fn propagate_console_msg(console: &&Console, console_message: ConsoleMessage) {
    let global = console.global.root();
    let pipelineId = global.r().pipeline();
    global.r().devtools_chan().as_ref().map(|chan| {
        chan.send(ScriptToDevtoolsControlMsg::ConsoleAPI(pipelineId,
                                                         console_message.clone(),
                                                         global.r().get_worker_id()))
            .unwrap();
    });
}
<commit_msg>Auto merge of #9226 - nox:clean-console, r=KiChjang<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use devtools_traits::{ConsoleMessage, LogLevel, ScriptToDevtoolsControlMsg};
use dom::bindings::codegen::Bindings::ConsoleBinding;
use dom::bindings::codegen::Bindings::ConsoleBinding::ConsoleMethods;
use dom::bindings::global::{GlobalRef, global_root_from_reflector};
use dom::bindings::js::Root;
use dom::bindings::reflector::{Reflector, reflect_dom_object};
use util::str::DOMString;

// https://developer.mozilla.org/en-US/docs/Web/API/Console
#[dom_struct]
pub struct Console {
    reflector_: Reflector,
}

impl Console {
    fn new_inherited() -> Console {
        Console {
            reflector_: Reflector::new(),
        }
    }

    pub fn new(global: GlobalRef) -> Root<Console> {
        reflect_dom_object(box Console::new_inherited(),
                           global,
                           ConsoleBinding::Wrap)
    }

    fn send_to_devtools(&self, level: LogLevel, message: DOMString) {
        let global = global_root_from_reflector(self);
        let global = global.r();
        if let Some(chan) = global.devtools_chan() {
            let console_message = prepare_message(level, message);
            let devtools_message = ScriptToDevtoolsControlMsg::ConsoleAPI(
                global.pipeline(),
                console_message,
                global.get_worker_id());
            chan.send(devtools_message).unwrap();
        }
    }
}

impl ConsoleMethods for Console {
    // https://developer.mozilla.org/en-US/docs/Web/API/Console/log
    fn Log(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            self.send_to_devtools(LogLevel::Log, message);
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console
    fn Debug(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            self.send_to_devtools(LogLevel::Debug, message);
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/info
    fn Info(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            self.send_to_devtools(LogLevel::Info, message);
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/warn
    fn Warn(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            self.send_to_devtools(LogLevel::Warn, message);
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/error
    fn Error(&self, messages: Vec<DOMString>) {
        for message in messages {
            println!("{}", message);
            self.send_to_devtools(LogLevel::Error, message);
        }
    }

    // https://developer.mozilla.org/en-US/docs/Web/API/Console/assert
    fn Assert(&self, condition: bool, message: Option<DOMString>) {
        if !condition {
            let message = message.unwrap_or_else(|| DOMString::from("no message"));
            println!("Assertion failed: {}", message);
            self.send_to_devtools(LogLevel::Error, message);
        }
    }
}

fn prepare_message(logLevel: LogLevel, message: DOMString) -> ConsoleMessage {
    // TODO: Sending fake values for filename, lineNumber and columnNumber in LogMessage; adjust later
    ConsoleMessage {
        message: String::from(message),
        logLevel: logLevel,
        filename: "test".to_owned(),
        lineNumber: 1,
        columnNumber: 1,
    }
}
<|endoftext|>


<commit_before>//! Utilities for defining a JSON parser.

use util::Id;

use std::cell::RefCell;
use std::collections::{ HashMap, HashSet };
use std::error::Error as StdError;
use std::fmt::{ Display, Debug, Error as FmtError, Formatter };
use std::hash::Hash;
use std::rc::Rc;
use std::sync::Arc;

use serde_json::error;
use serde::ser::{ Serialize, Serializer };
use serde_json;
pub use serde_json::value::Value as JSON;
use serde::de::Error;

/// Utility function: Make sure that we have consumed all the fields of an object.
pub fn check_fields(path: Path, json: &JSON) -> Result<(), ParseError> {
    if let JSON::Object(ref obj) = *json {
        if obj.is_empty() {
            Ok(())
        } else {
            Err(ParseError::unknown_fields(obj.keys().cloned().collect(), &path))
        }
    } else {
        Ok(())
    }
}

/// A path in the JSON tree. Used for displaying error messages.
#[derive(Clone, Debug)]
pub struct Path {
    buf: Rc<RefCell<String>>,
    len: usize,
}
impl Path {
    /// Create an empty Path.
    pub fn new() -> Self {
        Path {
            buf: Rc::new(RefCell::new(String::new())),
            len: 0,
        }
    }

    /// Push a suffix after a path.
    pub fn push_str<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        let buf = self.buf.clone();
        let len;
        {
            let mut str = buf.borrow_mut();
            str.push_str(suffix);
            len = str.len();
        }
        let path = Path {
            buf: buf,
            len: len,
        };
        let result = cb(path);
        {
            let mut str = self.buf.borrow_mut();
            str.truncate(self.len)
        }
        result
    }

    pub fn push_index<F, T>(&self, index: usize, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!("[{}]", index), cb)
    }

    pub fn push<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!(".{}", suffix), cb)
    }

    pub fn to_string(&self) -> String {
        let mut str = self.buf.borrow().clone();
        str.truncate(self.len);
        str
    }
}

/// An error during parsing.
#[derive(Debug)]
pub enum ParseError {
    JSON(JSONError),
    MissingField {
        name: String,
        at: String,
    },
    UnknownFields {
        names: Vec<String>,
        at: String,
    },
    TypeError {
        name: String,
        at: String,
        expected: String,
    },
    EmptyObject {
        at: String
    },
    UnknownConstant {
        at: String,
        constant: String,
    }
}

impl Display for ParseError {
    fn fmt(&self, formatter: &mut Formatter) -> Result<(), FmtError> {
        (self as &Debug).fmt(formatter)
    }
}
impl StdError for ParseError {
    fn description(&self) -> &str {
        "Error while parsing to JSON"
    }
}

impl ParseError {
    pub fn missing_field(name: &str, at: &Path) -> Self {
        ParseError::MissingField {
            name: name.to_owned(),
            at: at.to_string(),
        }
    }
    pub fn type_error(name: &str, at: &Path, expected: &str) -> Self {
        ParseError::TypeError {
            name: name.to_owned(),
            at: at.to_string(),
            expected: expected.to_owned()
        }
    }
    pub fn unknown_fields(names: Vec<String>, at: &Path) -> Self {
        ParseError::UnknownFields {
            names: names,
            at: at.to_string()
        }
    }
    pub fn unknown_constant(value: &str, at: &Path) -> Self {
        ParseError::UnknownConstant {
            constant: value.to_owned(),
            at: at.to_string()
        }
    }
    pub fn empty_object(at: &Path) -> Self {
        ParseError::EmptyObject {
            at: at.to_string(),
        }
    }
    pub fn json(error: error::Error) -> Self {
        ParseError::JSON(JSONError(error))
    }
}

#[derive(Debug)]
pub struct JSONError(error::Error);

impl Serialize for JSONError {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{:?}", self))
    }
}

/// An object that knows how to parse values from JSON into type T.
///
/// The JSON object is expected to be consumed along the way. A successful parse will
/// typically leave an empty JSON object.
pub trait Parser<T: Sized> {
    fn description() -> String;
    fn from_str(source: &str) -> Result<T, ParseError> {
        match serde_json::from_str(source) {
            Err(err) => Err(ParseError::json(err)),
            Ok(mut json) => Self::parse(Path::new(), &mut json)
        }
    }

    /// Parse a single value from JSON, consuming as much as necessary from JSON.
    fn parse(path: Path, source: &mut JSON) -> Result<T, ParseError>;

    /// Parse a field from JSON, consuming it.
    fn take(path: Path, source: &mut JSON, field_name: &str) -> Result<T, ParseError> {
        match Self::take_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }

    /// Parse a field from JSON, consuming it.
    fn take_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<T, ParseError>> {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(mut v) = obj.remove(field_name) {
                Some(Self::parse(path, &mut v))
            } else {
                None
            }
        } else {
            Some(Err(ParseError::type_error(field_name, &path, "object")))
        }
    }

    /// Parse a field containing an array from JSON, consuming the field.
    fn take_vec_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<Vec<T>, ParseError>>
    {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(ref mut json) = obj.remove(field_name) {
                if let JSON::Array(ref mut vec) = *json {
                    let mut result = Vec::with_capacity(vec.len());
                    for (json, i) in vec.iter_mut().zip(0..) {
                        match path.push(&format!("[{}]", i),
                            |path| Self::parse(path, json)
                        ) {
                            Err(error) => return Some(Err(error)),
                            Ok(parsed) => result.push(parsed)
                        }
                    }
                    Some(Ok(result))
                } else {
                    Some(Err(ParseError::type_error(field_name, &path, "array")))
                }
            } else {
                None
            }
        } else {
            Some(Err(ParseError::missing_field(field_name, &path)))
        }
    }

    fn take_vec(path: Path, source: &mut JSON, field_name: &str) -> Result<Vec<T>, ParseError> {
        match Self::take_vec_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }
}

impl Parser<f64> for f64 {
    fn description() -> String {
        "Number".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::I64(val) => Ok(val as f64),
            JSON::F64(val) => Ok(val),
            JSON::U64(val) => Ok(val as f64),
            _ => Err(ParseError::type_error("as float", &path, "number"))
        }
    }
}

impl Parser<bool> for bool {
    fn description() -> String {
        "bool".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Bool(ref b) => Ok(*b),
            JSON::U64(0) | JSON::I64(0) => Ok(false),
            JSON::U64(1) | JSON::I64(1) => Ok(true),
            JSON::String(ref str) if str == "true" => Ok(true),
            JSON::String(ref str) if str == "false" => Ok(false),
            _ => Err(ParseError::type_error("as bool", &path, "boolean"))
        }
    }
}

impl Parser<u8> for u8 {
    fn description() -> String {
        "byte".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_u64() {
            None => Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) if *val > u8::max_value() as u64 =>
                Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) => Ok(*val as u8)
        }
    }
}

impl<T> Parser<Vec<T>> for Vec<T> where T: Parser<T> {
    fn description() -> String {
        format!("Array<{}>", T::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        // Otherwise, parse as an actual array.
        match *source {
            JSON::Array(ref mut array) => {
                let mut result = Vec::with_capacity(array.len());
                for (source, i) in array.iter_mut ().zip(0..) {
                    let value = try!(path.push_index(i, |path| T::parse(path, source)));
                    result.push(value)
                }
                Ok(result)
            }
            JSON::Null => {
                // Accept `null` as an empty array.
                Ok(vec![])
            }
            _ => {
                // Attempt to promote the value to an array.
                let single = try!(path.push_str("", |path| T::parse(path, source)));
                Ok(vec![single])
            }
        }
    }
}

/*
impl<T, U> Parser<(T, U)> for (T, U) where T: Parser<T>, U: Parser<U> {
    fn description() -> String {
        format!("({}, {})", T::description(), U::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Array(ref mut array) if array.len() == 2 => {
                let mut right = array.pop().unwrap(); // We just checked that length == 2
                let mut left = array.pop().unwrap(); // We just checked that length == 2
                let left_parsed = try!(path.push(&T::description() as &str, |path| {T::parse(path, &mut left)}));
                let right_parsed = try!(path.push(&U::description() as &str, |path| {U::parse(path, &mut right)}));
                Ok((left_parsed, right_parsed))
            }
            _ => Err(ParseError::type_error("pair of values", &path, "array"))
        }
    }
}
*/
impl Parser<String> for String {
    fn description() -> String {
        "String".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_string() {
            Some(str) => Ok(str.to_owned()),
            None => Err(ParseError::type_error("string", &path, "string"))
        }
    }
}

impl<T> Parser<Arc<T>> for Arc<T> where T: Parser<T> {
    fn description() -> String {
        T::description()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        Ok(Arc::new(try!(T::parse(path, source))))
    }
}

pub trait ToJSON {
    fn to_json(&self) -> JSON;
}

impl ToJSON for String {
    fn to_json(&self) -> JSON {
        JSON::String(self.clone())
    }
}

impl ToJSON for bool {
    fn to_json(&self) -> JSON {
        JSON::Bool(*self)
    }
}

impl ToJSON for f64 {
    fn to_json(&self) -> JSON {
        JSON::F64(*self)
    }
}

impl ToJSON for usize {
    fn to_json(&self) -> JSON {
        JSON::U64(*self as u64)
    }
}

impl ToJSON for JSON {
    fn to_json(&self) -> JSON {
        self.clone()
    }
}

impl<T> ToJSON for HashSet<T> where T: ToJSON + Eq + Hash {
    fn to_json(&self) -> JSON {
        JSON::Array((*self).iter().map(T::to_json).collect())
    }
}

impl<T> ToJSON for HashMap<String, T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, v)| (k.clone(), T::to_json(v))).collect())
    }
}

impl<T> ToJSON for Vec<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Array(self.iter().map(|x| x.to_json()).collect())
    }
}

impl<'a, T> ToJSON for Vec<(&'a str, T)> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|&(ref k, ref v)| {
            ((*k).to_owned(), v.to_json())
        }).collect())
    }
}

impl <'a> ToJSON for &'a str {
    fn to_json(&self) -> JSON {
        JSON::String((*self).to_owned())
    }
}

impl<'a, T> ToJSON for &'a T where T: ToJSON {
    fn to_json(&self) -> JSON {
        (**self).to_json()
    }
}

impl<K, T, V> ToJSON for HashMap<Id<K>, Result<T, V>> where T: ToJSON, V: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, result)| {
            let k = k.to_string();
            let result = match *result {
                Ok(ref ok) => ok.to_json(),
                Err(ref err) => vec![("Error", err)].to_json()
            };
            (k, result)
        }).collect())
    }
}

impl<T> ToJSON for Option<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        match *self {
            None => JSON::Null,
            Some(ref result) => result.to_json()
        }
    }
}

impl ToJSON for () {
    fn to_json(&self) -> JSON {
        JSON::Null
    }
}

/*
impl<T> Parser<T> for T where T: Deserialize {
    fn parse(_: Path, source: &mut JSON) -> Result<T, ParseError> {
        use serde_json;
        serde_json::from_value(source.clone()).map_err(ParseError::json)
    }
}
*/
<commit_msg>[fxbox/taxonomy] Merge pull request #95 from Yoric/master<commit_after>//! Utilities for defining a JSON parser.

use util::Id;

use std::cell::RefCell;
use std::collections::{ HashMap, HashSet };
use std::error::Error as StdError;
use std::fmt::{ Display, Debug, Error as FmtError, Formatter };
use std::hash::Hash;
use std::rc::Rc;
use std::sync::Arc;

use serde_json::error;
use serde::ser::{ Serialize, Serializer };
use serde_json;
pub use serde_json::value::Value as JSON;
use serde::de::Error;

/// Utility function: Make sure that we have consumed all the fields of an object.
pub fn check_fields(path: Path, json: &JSON) -> Result<(), ParseError> {
    if let JSON::Object(ref obj) = *json {
        if obj.is_empty() {
            Ok(())
        } else {
            Err(ParseError::unknown_fields(obj.keys().cloned().collect(), &path))
        }
    } else {
        Ok(())
    }
}

/// A path in the JSON tree. Used for displaying error messages.
#[derive(Clone, Debug)]
pub struct Path {
    buf: Rc<RefCell<String>>,
    len: usize,
}
impl Path {
    /// Create an empty Path.
    pub fn new() -> Self {
        Path {
            buf: Rc::new(RefCell::new(String::new())),
            len: 0,
        }
    }

    /// Push a suffix after a path.
    pub fn push_str<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        let buf = self.buf.clone();
        let len;
        {
            let mut str = buf.borrow_mut();
            str.push_str(suffix);
            len = str.len();
        }
        let path = Path {
            buf: buf,
            len: len,
        };
        let result = cb(path);
        {
            let mut str = self.buf.borrow_mut();
            str.truncate(self.len)
        }
        result
    }

    pub fn push_index<F, T>(&self, index: usize, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!("[{}]", index), cb)
    }

    pub fn push<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!(".{}", suffix), cb)
    }

    pub fn to_string(&self) -> String {
        let mut str = self.buf.borrow().clone();
        str.truncate(self.len);
        str
    }
}

/// An error during parsing.
#[derive(Debug)]
pub enum ParseError {
    JSON(JSONError),
    MissingField {
        name: String,
        at: String,
    },
    UnknownFields {
        names: Vec<String>,
        at: String,
    },
    TypeError {
        name: String,
        at: String,
        expected: String,
    },
    EmptyObject {
        at: String
    },
    UnknownConstant {
        at: String,
        constant: String,
    }
}

impl Display for ParseError {
    fn fmt(&self, formatter: &mut Formatter) -> Result<(), FmtError> {
        (self as &Debug).fmt(formatter)
    }
}
impl StdError for ParseError {
    fn description(&self) -> &str {
        "Error while parsing to JSON"
    }
}

impl ParseError {
    pub fn missing_field(name: &str, at: &Path) -> Self {
        ParseError::MissingField {
            name: name.to_owned(),
            at: at.to_string(),
        }
    }
    pub fn type_error(name: &str, at: &Path, expected: &str) -> Self {
        ParseError::TypeError {
            name: name.to_owned(),
            at: at.to_string(),
            expected: expected.to_owned()
        }
    }
    pub fn unknown_fields(names: Vec<String>, at: &Path) -> Self {
        ParseError::UnknownFields {
            names: names,
            at: at.to_string()
        }
    }
    pub fn unknown_constant(value: &str, at: &Path) -> Self {
        ParseError::UnknownConstant {
            constant: value.to_owned(),
            at: at.to_string()
        }
    }
    pub fn empty_object(at: &Path) -> Self {
        ParseError::EmptyObject {
            at: at.to_string(),
        }
    }
    pub fn json(error: error::Error) -> Self {
        ParseError::JSON(JSONError(error))
    }
}

#[derive(Debug)]
pub struct JSONError(error::Error);

impl Serialize for JSONError {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{:?}", self))
    }
}

/// An object that knows how to parse values from JSON into type T.
///
/// The JSON object is expected to be consumed along the way. A successful parse will
/// typically leave an empty JSON object.
pub trait Parser<T: Sized> {
    fn description() -> String;
    fn from_str(source: &str) -> Result<T, ParseError> {
        Self::from_str_at(Path::new(), source)
    }
    fn from_str_at(path: Path, source: &str) -> Result<T, ParseError> {
        match serde_json::from_str(source) {
            Err(err) => Err(ParseError::json(err)),
            Ok(mut json) => Self::parse(path, &mut json)
        }
    }

    /// Parse a single value from JSON, consuming as much as necessary from JSON.
    fn parse(path: Path, source: &mut JSON) -> Result<T, ParseError>;

    /// Parse a field from JSON, consuming it.
    fn take(path: Path, source: &mut JSON, field_name: &str) -> Result<T, ParseError> {
        match Self::take_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }

    /// Parse a field from JSON, consuming it.
    fn take_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<T, ParseError>> {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(mut v) = obj.remove(field_name) {
                Some(Self::parse(path, &mut v))
            } else {
                None
            }
        } else {
            Some(Err(ParseError::type_error(field_name, &path, "object")))
        }
    }

    /// Parse a field containing an array from JSON, consuming the field.
    fn take_vec_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<Vec<T>, ParseError>>
    {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(ref mut json) = obj.remove(field_name) {
                if let JSON::Array(ref mut vec) = *json {
                    let mut result = Vec::with_capacity(vec.len());
                    for (json, i) in vec.iter_mut().zip(0..) {
                        match path.push(&format!("[{}]", i),
                            |path| Self::parse(path, json)
                        ) {
                            Err(error) => return Some(Err(error)),
                            Ok(parsed) => result.push(parsed)
                        }
                    }
                    Some(Ok(result))
                } else {
                    Some(Err(ParseError::type_error(field_name, &path, "array")))
                }
            } else {
                None
            }
        } else {
            Some(Err(ParseError::missing_field(field_name, &path)))
        }
    }

    fn take_vec(path: Path, source: &mut JSON, field_name: &str) -> Result<Vec<T>, ParseError> {
        match Self::take_vec_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }
}

impl Parser<f64> for f64 {
    fn description() -> String {
        "Number".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::I64(val) => Ok(val as f64),
            JSON::F64(val) => Ok(val),
            JSON::U64(val) => Ok(val as f64),
            _ => Err(ParseError::type_error("as float", &path, "number"))
        }
    }
}

impl Parser<bool> for bool {
    fn description() -> String {
        "bool".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Bool(ref b) => Ok(*b),
            JSON::U64(0) | JSON::I64(0) => Ok(false),
            JSON::U64(1) | JSON::I64(1) => Ok(true),
            JSON::String(ref str) if str == "true" => Ok(true),
            JSON::String(ref str) if str == "false" => Ok(false),
            _ => Err(ParseError::type_error("as bool", &path, "boolean"))
        }
    }
}

impl Parser<u8> for u8 {
    fn description() -> String {
        "byte".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_u64() {
            None => Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) if *val > u8::max_value() as u64 =>
                Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) => Ok(*val as u8)
        }
    }
}

impl<T> Parser<Vec<T>> for Vec<T> where T: Parser<T> {
    fn description() -> String {
        format!("Array<{}>", T::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        // Otherwise, parse as an actual array.
        match *source {
            JSON::Array(ref mut array) => {
                let mut result = Vec::with_capacity(array.len());
                for (source, i) in array.iter_mut ().zip(0..) {
                    let value = try!(path.push_index(i, |path| T::parse(path, source)));
                    result.push(value)
                }
                Ok(result)
            }
            JSON::Null => {
                // Accept `null` as an empty array.
                Ok(vec![])
            }
            _ => {
                // Attempt to promote the value to an array.
                let single = try!(path.push_str("", |path| T::parse(path, source)));
                Ok(vec![single])
            }
        }
    }
}

/*
impl<T, U> Parser<(T, U)> for (T, U) where T: Parser<T>, U: Parser<U> {
    fn description() -> String {
        format!("({}, {})", T::description(), U::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Array(ref mut array) if array.len() == 2 => {
                let mut right = array.pop().unwrap(); // We just checked that length == 2
                let mut left = array.pop().unwrap(); // We just checked that length == 2
                let left_parsed = try!(path.push(&T::description() as &str, |path| {T::parse(path, &mut left)}));
                let right_parsed = try!(path.push(&U::description() as &str, |path| {U::parse(path, &mut right)}));
                Ok((left_parsed, right_parsed))
            }
            _ => Err(ParseError::type_error("pair of values", &path, "array"))
        }
    }
}
*/
impl Parser<String> for String {
    fn description() -> String {
        "String".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_string() {
            Some(str) => Ok(str.to_owned()),
            None => Err(ParseError::type_error("string", &path, "string"))
        }
    }
}

impl<T> Parser<Arc<T>> for Arc<T> where T: Parser<T> {
    fn description() -> String {
        T::description()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        Ok(Arc::new(try!(T::parse(path, source))))
    }
}

pub trait ToJSON {
    fn to_json(&self) -> JSON;
}

impl ToJSON for String {
    fn to_json(&self) -> JSON {
        JSON::String(self.clone())
    }
}

impl ToJSON for bool {
    fn to_json(&self) -> JSON {
        JSON::Bool(*self)
    }
}

impl ToJSON for f64 {
    fn to_json(&self) -> JSON {
        JSON::F64(*self)
    }
}

impl ToJSON for usize {
    fn to_json(&self) -> JSON {
        JSON::U64(*self as u64)
    }
}

impl ToJSON for JSON {
    fn to_json(&self) -> JSON {
        self.clone()
    }
}

impl<T> ToJSON for HashSet<T> where T: ToJSON + Eq + Hash {
    fn to_json(&self) -> JSON {
        JSON::Array((*self).iter().map(T::to_json).collect())
    }
}

impl<T> ToJSON for HashMap<String, T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, v)| (k.clone(), T::to_json(v))).collect())
    }
}

impl<T> ToJSON for Vec<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Array(self.iter().map(|x| x.to_json()).collect())
    }
}

impl<'a, T> ToJSON for Vec<(&'a str, T)> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|&(ref k, ref v)| {
            ((*k).to_owned(), v.to_json())
        }).collect())
    }
}

impl <'a> ToJSON for &'a str {
    fn to_json(&self) -> JSON {
        JSON::String((*self).to_owned())
    }
}

impl<'a, T> ToJSON for &'a T where T: ToJSON {
    fn to_json(&self) -> JSON {
        (**self).to_json()
    }
}

impl<K, T, V> ToJSON for HashMap<Id<K>, Result<T, V>> where T: ToJSON, V: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, result)| {
            let k = k.to_string();
            let result = match *result {
                Ok(ref ok) => ok.to_json(),
                Err(ref err) => vec![("Error", err)].to_json()
            };
            (k, result)
        }).collect())
    }
}

impl<T> ToJSON for Option<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        match *self {
            None => JSON::Null,
            Some(ref result) => result.to_json()
        }
    }
}

impl ToJSON for () {
    fn to_json(&self) -> JSON {
        JSON::Null
    }
}

/*
impl<T> Parser<T> for T where T: Deserialize {
    fn parse(_: Path, source: &mut JSON) -> Result<T, ParseError> {
        use serde_json;
        serde_json::from_value(source.clone()).map_err(ParseError::json)
    }
}
*/
<|endoftext|>


<commit_before>use std::sync::Arc;
use std::path::Path;
use std::collections::HashSet;

use rustc_hash::FxHashMap;

use ra_db::{
    CrateGraph, FileId, SourceRoot, SourceRootId, SourceDatabase, salsa,
};
use ra_hir::{db, HirInterner};
use ra_project_model::ProjectWorkspace;
use ra_vfs::{Vfs, VfsChange};

type Result<T> = std::result::Result<T, failure::Error>;

#[salsa::database(
    ra_db::SourceDatabaseStorage,
    db::HirDatabaseStorage,
    db::PersistentHirDatabaseStorage
)]
#[derive(Debug)]
pub struct BatchDatabase {
    runtime: salsa::Runtime<BatchDatabase>,
    interner: Arc<HirInterner>,
}

impl salsa::Database for BatchDatabase {
    fn salsa_runtime(&self) -> &salsa::Runtime<BatchDatabase> {
        &self.runtime
    }
}

impl AsRef<HirInterner> for BatchDatabase {
    fn as_ref(&self) -> &HirInterner {
        &self.interner
    }
}

fn vfs_file_to_id(f: ra_vfs::VfsFile) -> FileId {
    FileId(f.0.into())
}
fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
    SourceRootId(r.0.into())
}

impl BatchDatabase {
    pub fn load(crate_graph: CrateGraph, vfs: &mut Vfs) -> BatchDatabase {
        let mut db =
            BatchDatabase { runtime: salsa::Runtime::default(), interner: Default::default() };
        db.set_crate_graph(Arc::new(crate_graph));

        // wait until Vfs has loaded all roots
        let receiver = vfs.task_receiver().clone();
        let mut roots_loaded = HashSet::new();
        for task in receiver {
            vfs.handle_task(task);
            let mut done = false;
            for change in vfs.commit_changes() {
                match change {
                    VfsChange::AddRoot { root, files } => {
                        let source_root_id = vfs_root_to_id(root);
                        log::debug!(
                            "loaded source root {:?} with path {:?}",
                            source_root_id,
                            vfs.root2path(root)
                        );
                        let mut file_map = FxHashMap::default();
                        for (vfs_file, path, text) in files {
                            let file_id = vfs_file_to_id(vfs_file);
                            db.set_file_text(file_id, text);
                            db.set_file_relative_path(file_id, path.clone());
                            db.set_file_source_root(file_id, source_root_id);
                            file_map.insert(path, file_id);
                        }
                        let source_root = SourceRoot { files: file_map };
                        db.set_source_root(source_root_id, Arc::new(source_root));
                        roots_loaded.insert(source_root_id);
                        if roots_loaded.len() == vfs.num_roots() {
                            done = true;
                        }
                    }
                    VfsChange::AddFile { .. }
                    | VfsChange::RemoveFile { .. }
                    | VfsChange::ChangeFile { .. } => {
                        // We just need the first scan, so just ignore these
                    }
                }
            }
            if done {
                break;
            }
        }

        db
    }

    pub fn load_cargo(root: impl AsRef<Path>) -> Result<(BatchDatabase, Vec<SourceRootId>)> {
        let root = root.as_ref().canonicalize()?;
        let ws = ProjectWorkspace::discover(root.as_ref())?;
        let mut roots = Vec::new();
        roots.push(root.clone());
        for pkg in ws.cargo.packages() {
            roots.push(pkg.root(&ws.cargo).to_path_buf());
        }
        for krate in ws.sysroot.crates() {
            roots.push(krate.root_dir(&ws.sysroot).to_path_buf())
        }
        let (mut vfs, roots) = Vfs::new(roots);
        let mut load = |path: &Path| {
            let vfs_file = vfs.load(path);
            log::debug!("vfs file {:?} -> {:?}", path, vfs_file);
            vfs_file.map(vfs_file_to_id)
        };
        let crate_graph = ws.to_crate_graph(&mut load);
        log::debug!("crate graph: {:?}", crate_graph);

        let local_roots = roots
            .into_iter()
            .filter(|r| vfs.root2path(*r).starts_with(&root))
            .map(vfs_root_to_id)
            .collect();

        let db = BatchDatabase::load(crate_graph, &mut vfs);
        let _ = vfs.shutdown();
        Ok((db, local_roots))
    }
}

#[cfg(test)]
mod tests {
    use ra_hir::Crate;
    use super::*;

    #[test]
    fn test_loading_rust_analyzer() {
        let mut path = std::env::current_exe().unwrap();
        while !path.join("Cargo.toml").is_file() {
            path = path.parent().unwrap().to_owned();
        }
        let (db, roots) = BatchDatabase::load_cargo(path).unwrap();
        let mut num_crates = 0;
        for root in roots {
            for _krate in Crate::source_root_crates(&db, root) {
                num_crates += 1;
            }
        }

        // RA has quite a few crates, but the exact count doesn't matter
        assert!(num_crates > 20);
    }
}
<commit_msg>Remove call to canonicalize in BatchDatabase::load_cargo<commit_after>use std::sync::Arc;
use std::path::Path;
use std::collections::HashSet;

use rustc_hash::FxHashMap;

use ra_db::{
    CrateGraph, FileId, SourceRoot, SourceRootId, SourceDatabase, salsa,
};
use ra_hir::{db, HirInterner};
use ra_project_model::ProjectWorkspace;
use ra_vfs::{Vfs, VfsChange};

type Result<T> = std::result::Result<T, failure::Error>;

#[salsa::database(
    ra_db::SourceDatabaseStorage,
    db::HirDatabaseStorage,
    db::PersistentHirDatabaseStorage
)]
#[derive(Debug)]
pub struct BatchDatabase {
    runtime: salsa::Runtime<BatchDatabase>,
    interner: Arc<HirInterner>,
}

impl salsa::Database for BatchDatabase {
    fn salsa_runtime(&self) -> &salsa::Runtime<BatchDatabase> {
        &self.runtime
    }
}

impl AsRef<HirInterner> for BatchDatabase {
    fn as_ref(&self) -> &HirInterner {
        &self.interner
    }
}

fn vfs_file_to_id(f: ra_vfs::VfsFile) -> FileId {
    FileId(f.0.into())
}
fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
    SourceRootId(r.0.into())
}

impl BatchDatabase {
    pub fn load(crate_graph: CrateGraph, vfs: &mut Vfs) -> BatchDatabase {
        let mut db =
            BatchDatabase { runtime: salsa::Runtime::default(), interner: Default::default() };
        db.set_crate_graph(Arc::new(crate_graph));

        // wait until Vfs has loaded all roots
        let receiver = vfs.task_receiver().clone();
        let mut roots_loaded = HashSet::new();
        for task in receiver {
            vfs.handle_task(task);
            let mut done = false;
            for change in vfs.commit_changes() {
                match change {
                    VfsChange::AddRoot { root, files } => {
                        let source_root_id = vfs_root_to_id(root);
                        log::debug!(
                            "loaded source root {:?} with path {:?}",
                            source_root_id,
                            vfs.root2path(root)
                        );
                        let mut file_map = FxHashMap::default();
                        for (vfs_file, path, text) in files {
                            let file_id = vfs_file_to_id(vfs_file);
                            db.set_file_text(file_id, text);
                            db.set_file_relative_path(file_id, path.clone());
                            db.set_file_source_root(file_id, source_root_id);
                            file_map.insert(path, file_id);
                        }
                        let source_root = SourceRoot { files: file_map };
                        db.set_source_root(source_root_id, Arc::new(source_root));
                        roots_loaded.insert(source_root_id);
                        if roots_loaded.len() == vfs.num_roots() {
                            done = true;
                        }
                    }
                    VfsChange::AddFile { .. }
                    | VfsChange::RemoveFile { .. }
                    | VfsChange::ChangeFile { .. } => {
                        // We just need the first scan, so just ignore these
                    }
                }
            }
            if done {
                break;
            }
        }

        db
    }

    pub fn load_cargo(root: impl AsRef<Path>) -> Result<(BatchDatabase, Vec<SourceRootId>)> {
        let root = std::env::current_dir()?.join(root);
        let ws = ProjectWorkspace::discover(root.as_ref())?;
        let mut roots = Vec::new();
        roots.push(root.clone());
        for pkg in ws.cargo.packages() {
            roots.push(pkg.root(&ws.cargo).to_path_buf());
        }
        for krate in ws.sysroot.crates() {
            roots.push(krate.root_dir(&ws.sysroot).to_path_buf())
        }
        let (mut vfs, roots) = Vfs::new(roots);
        let mut load = |path: &Path| {
            let vfs_file = vfs.load(path);
            log::debug!("vfs file {:?} -> {:?}", path, vfs_file);
            vfs_file.map(vfs_file_to_id)
        };
        let crate_graph = ws.to_crate_graph(&mut load);
        log::debug!("crate graph: {:?}", crate_graph);

        let local_roots = roots
            .into_iter()
            .filter(|r| vfs.root2path(*r).starts_with(&root))
            .map(vfs_root_to_id)
            .collect();

        let db = BatchDatabase::load(crate_graph, &mut vfs);
        let _ = vfs.shutdown();
        Ok((db, local_roots))
    }
}

#[cfg(test)]
mod tests {
    use ra_hir::Crate;
    use super::*;

    #[test]
    fn test_loading_rust_analyzer() {
        let mut path = std::env::current_exe().unwrap();
        while !path.join("Cargo.toml").is_file() {
            path = path.parent().unwrap().to_owned();
        }
        let (db, roots) = BatchDatabase::load_cargo(path).unwrap();
        let mut num_crates = 0;
        for root in roots {
            for _krate in Crate::source_root_crates(&db, root) {
                num_crates += 1;
            }
        }

        // RA has quite a few crates, but the exact count doesn't matter
        assert!(num_crates > 20);
    }
}
<|endoftext|>


<commit_before>mod sourcegen_tests;
mod sourcegen_ast;
mod ast_src;

use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if skip_slow_tests() {
        return;
    }
    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let crates_dir = project_root().join("crates");

    let mut files = ::sourcegen::list_rust_files(&crates_dir);
    files.retain(|path| {
        // Get all files which are not in the crates/syntax/test_data folder
        !path.components().any(|component| component.as_os_str() == "test_data")
    });

    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if f(text).is_ok() {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<commit_msg>minor: better error message<commit_after>mod sourcegen_tests;
mod sourcegen_ast;
mod ast_src;

use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if skip_slow_tests() {
        return;
    }
    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let crates_dir = project_root().join("crates");

    let mut files = ::sourcegen::list_rust_files(&crates_dir);
    files.retain(|path| {
        // Get all files which are not in the crates/syntax/test_data folder
        !path.components().any(|component| component.as_os_str() == "test_data")
    });

    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err[0]))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if f(text).is_ok() {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<|endoftext|>