<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use error::{Error};
use fmt;
use sync::atomic::{AtomicBool, Ordering};
use thread;

pub struct Flag { failed: AtomicBool }

// Note that the Ordering uses to access the `failed` field of `Flag` below is
// always `Relaxed`, and that's because this isn't actually protecting any data,
// it's just a flag whether we've panicked or not.
//
// The actual location that this matters is when a mutex is **locked** which is
// where we have external synchronization ensuring that we see memory
// reads/writes to this flag.
//
// As a result, if it matters, we should see the correct value for `failed` in
// all cases.

impl Flag {
    pub const fn new() -> Flag {
        Flag { failed: AtomicBool::new(false) }
    }

    #[inline]
    pub fn borrow(&self) -> LockResult<Guard> {
        let ret = Guard { panicking: thread::panicking() };
        if self.get() {
            Err(PoisonError::new(ret))
        } else {
            Ok(ret)
        }
    }

    #[inline]
    pub fn done(&self, guard: &Guard) {
        if !guard.panicking && thread::panicking() {
            self.failed.store(true, Ordering::Relaxed);
        }
    }

    #[inline]
    pub fn get(&self) -> bool {
        self.failed.load(Ordering::Relaxed)
    }
}

pub struct Guard {
    panicking: bool,
}

/// A type of error which can be returned whenever a lock is acquired.
///
/// Both Mutexes and RwLocks are poisoned whenever a thread fails while the lock
/// is held. The precise semantics for when a lock is poisoned is documented on
/// each lock, but once a lock is poisoned then all future acquisitions will
/// return this error.
#[stable(feature = "rust1", since = "1.0.0")]
pub struct PoisonError<T> {
    guard: T,
}

/// An enumeration of possible errors which can occur while calling the
/// `try_lock` method.
#[stable(feature = "rust1", since = "1.0.0")]
pub enum TryLockError<T> {
    /// The lock could not be acquired because another thread failed while holding
    /// the lock.
    #[stable(feature = "rust1", since = "1.0.0")]
    Poisoned(#[stable(feature = "rust1", since = "1.0.0")] PoisonError<T>),
    /// The lock could not be acquired at this time because the operation would
    /// otherwise block.
    #[stable(feature = "rust1", since = "1.0.0")]
    WouldBlock,
}

/// A type alias for the result of a lock method which can be poisoned.
///
/// The `Ok` variant of this result indicates that the primitive was not
/// poisoned, and the `Guard` is contained within. The `Err` variant indicates
/// that the primitive was poisoned. Note that the `Err` variant *also* carries
/// the associated guard, and it can be acquired through the `into_inner`
/// method.
#[stable(feature = "rust1", since = "1.0.0")]
pub type LockResult<Guard> = Result<Guard, PoisonError<Guard>>;

/// A type alias for the result of a nonblocking locking method.
///
/// For more information, see `LockResult`. A `TryLockResult` doesn't
/// necessarily hold the associated guard in the `Err` type as the lock may not
/// have been acquired for other reasons.
#[stable(feature = "rust1", since = "1.0.0")]
pub type TryLockResult<Guard> = Result<Guard, TryLockError<Guard>>;

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Debug for PoisonError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        "PoisonError { inner: .. }".fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Display for PoisonError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        "poisoned lock: another task failed inside".fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> Error for PoisonError<T> {
    fn description(&self) -> &str {
        "poisoned lock: another task failed inside"
    }
}

impl<T> PoisonError<T> {
    /// Creates a `PoisonError`.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn new(guard: T) -> PoisonError<T> {
        PoisonError { guard: guard }
    }

    /// Consumes this error indicating that a lock is poisoned, returning the
    /// underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn into_inner(self) -> T { self.guard }

    /// Reaches into this error indicating that a lock is poisoned, returning a
    /// reference to the underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn get_ref(&self) -> &T { &self.guard }

    /// Reaches into this error indicating that a lock is poisoned, returning a
    /// mutable reference to the underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn get_mut(&mut self) -> &mut T { &mut self.guard }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> From<PoisonError<T>> for TryLockError<T> {
    fn from(err: PoisonError<T>) -> TryLockError<T> {
        TryLockError::Poisoned(err)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Debug for TryLockError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TryLockError::Poisoned(..) => "Poisoned(..)".fmt(f),
            TryLockError::WouldBlock => "WouldBlock".fmt(f)
        }
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Display for TryLockError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TryLockError::Poisoned(..) => "poisoned lock: another task failed inside",
            TryLockError::WouldBlock => "try_lock failed because the operation would block"
        }.fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> Error for TryLockError<T> {
    fn description(&self) -> &str {
        match *self {
            TryLockError::Poisoned(ref p) => p.description(),
            TryLockError::WouldBlock => "try_lock failed because the operation would block"
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            TryLockError::Poisoned(ref p) => Some(p),
            _ => None
        }
    }
}

pub fn map_result<T, U, F>(result: LockResult<T>, f: F)
                           -> LockResult<U>
                           where F: FnOnce(T) -> U {
    match result {
        Ok(t) => Ok(f(t)),
        Err(PoisonError { guard }) => Err(PoisonError::new(f(guard)))
    }
}
<commit_msg>Add missing url in sync structs<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use error::{Error};
use fmt;
use sync::atomic::{AtomicBool, Ordering};
use thread;

pub struct Flag { failed: AtomicBool }

// Note that the Ordering uses to access the `failed` field of `Flag` below is
// always `Relaxed`, and that's because this isn't actually protecting any data,
// it's just a flag whether we've panicked or not.
//
// The actual location that this matters is when a mutex is **locked** which is
// where we have external synchronization ensuring that we see memory
// reads/writes to this flag.
//
// As a result, if it matters, we should see the correct value for `failed` in
// all cases.

impl Flag {
    pub const fn new() -> Flag {
        Flag { failed: AtomicBool::new(false) }
    }

    #[inline]
    pub fn borrow(&self) -> LockResult<Guard> {
        let ret = Guard { panicking: thread::panicking() };
        if self.get() {
            Err(PoisonError::new(ret))
        } else {
            Ok(ret)
        }
    }

    #[inline]
    pub fn done(&self, guard: &Guard) {
        if !guard.panicking && thread::panicking() {
            self.failed.store(true, Ordering::Relaxed);
        }
    }

    #[inline]
    pub fn get(&self) -> bool {
        self.failed.load(Ordering::Relaxed)
    }
}

pub struct Guard {
    panicking: bool,
}

/// A type of error which can be returned whenever a lock is acquired.
///
/// Both [`Mutex`]es and [`RwLock`]s are poisoned whenever a thread fails while the lock
/// is held. The precise semantics for when a lock is poisoned is documented on
/// each lock, but once a lock is poisoned then all future acquisitions will
/// return this error.
///
/// [`Mutex`]: ../../std/sync/struct.Mutex.html
/// [`RwLock`]: ../../std/sync/struct.RwLock.html
#[stable(feature = "rust1", since = "1.0.0")]
pub struct PoisonError<T> {
    guard: T,
}

/// An enumeration of possible errors which can occur while calling the
/// `try_lock` method.
#[stable(feature = "rust1", since = "1.0.0")]
pub enum TryLockError<T> {
    /// The lock could not be acquired because another thread failed while holding
    /// the lock.
    #[stable(feature = "rust1", since = "1.0.0")]
    Poisoned(#[stable(feature = "rust1", since = "1.0.0")] PoisonError<T>),
    /// The lock could not be acquired at this time because the operation would
    /// otherwise block.
    #[stable(feature = "rust1", since = "1.0.0")]
    WouldBlock,
}

/// A type alias for the result of a lock method which can be poisoned.
///
/// The [`Ok`] variant of this result indicates that the primitive was not
/// poisoned, and the `Guard` is contained within. The [`Err`] variant indicates
/// that the primitive was poisoned. Note that the [`Err`] variant *also* carries
/// the associated guard, and it can be acquired through the [`into_inner`]
/// method.
///
/// [`Ok`]: ../../std/result/enum.Result.html#variant.Ok
/// [`Err`]: ../../std/result/enum.Result.html#variant.Err
/// [`into_inner`]: ../../std/sync/struct.Mutex.html#method.into_inner
#[stable(feature = "rust1", since = "1.0.0")]
pub type LockResult<Guard> = Result<Guard, PoisonError<Guard>>;

/// A type alias for the result of a nonblocking locking method.
///
/// For more information, see [`LockResult`]. A `TryLockResult` doesn't
/// necessarily hold the associated guard in the [`Err`] type as the lock may not
/// have been acquired for other reasons.
///
/// [`LockResult`]: ../../std/sync/type.LockResult.html
/// [`Err`]: ../../std/result/enum.Result.html#variant.Err
#[stable(feature = "rust1", since = "1.0.0")]
pub type TryLockResult<Guard> = Result<Guard, TryLockError<Guard>>;

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Debug for PoisonError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        "PoisonError { inner: .. }".fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Display for PoisonError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        "poisoned lock: another task failed inside".fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> Error for PoisonError<T> {
    fn description(&self) -> &str {
        "poisoned lock: another task failed inside"
    }
}

impl<T> PoisonError<T> {
    /// Creates a `PoisonError`.
    ///
    /// This is generally created by methods like [`Mutex::lock`] or [`RwLock::read`].
    ///
    /// [`Mutex::lock`]: ../../std/sync/struct.Mutex.html#method.lock
    /// [`RwLock::read`]: ../../std/sync/struct.RwLock.html#method.read
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn new(guard: T) -> PoisonError<T> {
        PoisonError { guard: guard }
    }

    /// Consumes this error indicating that a lock is poisoned, returning the
    /// underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn into_inner(self) -> T { self.guard }

    /// Reaches into this error indicating that a lock is poisoned, returning a
    /// reference to the underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn get_ref(&self) -> &T { &self.guard }

    /// Reaches into this error indicating that a lock is poisoned, returning a
    /// mutable reference to the underlying guard to allow access regardless.
    #[stable(feature = "sync_poison", since = "1.2.0")]
    pub fn get_mut(&mut self) -> &mut T { &mut self.guard }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> From<PoisonError<T>> for TryLockError<T> {
    fn from(err: PoisonError<T>) -> TryLockError<T> {
        TryLockError::Poisoned(err)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Debug for TryLockError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TryLockError::Poisoned(..) => "Poisoned(..)".fmt(f),
            TryLockError::WouldBlock => "WouldBlock".fmt(f)
        }
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> fmt::Display for TryLockError<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            TryLockError::Poisoned(..) => "poisoned lock: another task failed inside",
            TryLockError::WouldBlock => "try_lock failed because the operation would block"
        }.fmt(f)
    }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl<T> Error for TryLockError<T> {
    fn description(&self) -> &str {
        match *self {
            TryLockError::Poisoned(ref p) => p.description(),
            TryLockError::WouldBlock => "try_lock failed because the operation would block"
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            TryLockError::Poisoned(ref p) => Some(p),
            _ => None
        }
    }
}

pub fn map_result<T, U, F>(result: LockResult<T>, f: F)
                           -> LockResult<U>
                           where F: FnOnce(T) -> U {
    match result {
        Ok(t) => Ok(f(t)),
        Err(PoisonError { guard }) => Err(PoisonError::new(f(guard)))
    }
}
<|endoftext|>


<commit_before>use crate::utils::deserialize::deserialize_duration;
use elasticsearch::Elasticsearch;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::Duration;
use url::Url;

pub mod configuration;
pub mod explain;
pub(super) mod internal;
pub mod list;
pub mod models;
pub mod query;
pub mod remote;
pub mod status;
pub mod storage;

/// A structure wrapping around the elasticsearch's client.
#[derive(Clone, Debug)]
pub struct ElasticsearchStorage {
    /// Elasticsearch client
    pub(crate) client: Elasticsearch,
    /// Client configuration
    pub config: ElasticsearchStorageConfig,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ElasticsearchStorageConfig {
    pub url: Url,
    #[serde(deserialize_with = "deserialize_duration")]
    pub timeout: Duration,
    pub version_req: String,
    pub scroll_chunk_size: u64,
    pub scroll_pit_alive: String,
    pub insertion_concurrent_requests: usize,
    pub insertion_chunk_size: usize,
}

impl Default for ElasticsearchStorageConfig {
    /// We retrieve the elasticsearch configuration from ./config/elasticsearch/default.
    fn default() -> Self {
        let config = common::config::config_from(
            &PathBuf::from("config"),
            &["elasticsearch"],
            None,
            None,
            None,
        );

        config
            .expect("cannot build the configuration for testing from config")
            .get("elasticsearch")
            .expect("expected elasticsearch section in configuration from config")
    }
}

impl ElasticsearchStorageConfig {
    pub fn default_testing() -> Self {
        let config_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../config");

        let config = common::config::config_from(
            config_dir.as_path(),
            &["elasticsearch"],
            "testing",
            "MIMIR_TEST",
            None,
        );

        config
            .unwrap_or_else(|_| {
                panic!(
                    "cannot build the configuration for testing from {}",
                    config_dir.display(),
                )
            })
            .get("elasticsearch")
            .unwrap_or_else(|_| {
                panic!(
                    "expected elasticsearch section in configuration from {}",
                    config_dir.display(),
                )
            })
    }
}

#[cfg(test)]
pub mod tests {

    use config::Config;
    use serde::{Deserialize, Serialize};
    use serial_test::serial;

    use super::*;

    use crate::domain::ports::secondary::remote::Remote;
    use crate::domain::ports::secondary::storage::Storage;
    use crate::utils::docker;
    use common::document::{ContainerDocument, Document};

    // In this test we present an invalid configuration (its actually empty) to the
    // create_container function, and expect the error message to be meaningful
    #[tokio::test]
    #[serial]
    async fn should_return_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");

        let config = Config::builder().build().expect("build empty config");

        let res = client.create_container(config).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("Invalid Elasticsearch Index Configuration"));
    }

    #[tokio::test]
    #[serial]
    async fn should_connect_to_elasticsearch() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let _client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
    }

    #[tokio::test]
    #[serial]
    async fn should_create_index_with_valid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");
        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "index": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res.is_ok());
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_report_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "indx": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Container Creation Error"));
    }

    #[derive(Deserialize, Serialize)]
    struct TestObj {
        value: String,
    }

    impl Document for TestObj {
        fn id(&self) -> String {
            self.value.clone()
        }
    }

    impl ContainerDocument for TestObj {
        fn static_doc_type() -> &'static str {
            "test-obj"
        }

        fn default_es_container_config() -> config::Config {
            config::Config::builder()
                .set_default("container.name", Self::static_doc_type())
                .unwrap()
                .set_default("container.dataset", "default")
                .unwrap()
                .set_default("elasticsearch.parameters.timeout", "10s")
                .unwrap()
                .set_default("elasticsearch.parameters.wait_for_active_shards", "1")
                .unwrap()
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "settings": { "index": { "number_of_shards": 1, "number_of_replicas": 1 } } } }"#,
                    config::FileFormat::Json,
                ))
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "mappings": { "properties": { "value": { "type": "text" } } } } }"#,
                    config::FileFormat::Json,
                ))
                .build()
                .expect("invalid container configuration for TestObj")
        }
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_insert_multiple_documents() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");

        client
            .create_container(TestObj::default_es_container_config())
            .await
            .expect("container creation");

        let documents = vec![
            TestObj {
                value: String::from("obj1"),
            },
            TestObj {
                value: String::from("obj2"),
            },
            TestObj {
                value: String::from("obj3"),
            },
            TestObj {
                value: String::from("obj4"),
            },
            TestObj {
                value: String::from("obj5"),
            },
            TestObj {
                value: String::from("obj6"),
            },
        ];
        let documents = futures::stream::iter(documents);

        let res = client
            .insert_documents(
                String::from("root_obj_dataset_test-index-bulk-insert"),
                documents,
            )
            .await;

        assert_eq!(res.expect("insertion stats").created, 6);
    }

    #[tokio::test]
    #[serial]
    async fn should_detect_invalid_elasticsearch_version() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig {
                version_req: ">=9.99.99".to_string(),
                ..ElasticsearchStorageConfig::default_testing()
            })
            .await;

        assert!(client
            .unwrap_err()
            .to_string()
            .contains("Elasticsearch Invalid version"));
    }
}
<commit_msg>fix tests<commit_after>use crate::utils::deserialize::deserialize_duration;
use elasticsearch::Elasticsearch;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::Duration;
use url::Url;

pub mod configuration;
pub mod explain;
pub(super) mod internal;
pub mod list;
pub mod models;
pub mod query;
pub mod remote;
pub mod status;
pub mod storage;

/// A structure wrapping around the elasticsearch's client.
#[derive(Clone, Debug)]
pub struct ElasticsearchStorage {
    /// Elasticsearch client
    pub(crate) client: Elasticsearch,
    /// Client configuration
    pub config: ElasticsearchStorageConfig,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ElasticsearchStorageConfig {
    pub url: Url,
    #[serde(deserialize_with = "deserialize_duration")]
    pub timeout: Duration,
    pub version_req: String,
    pub scroll_chunk_size: u64,
    pub scroll_pit_alive: String,
    pub insertion_concurrent_requests: usize,
    pub insertion_chunk_size: usize,
}

impl Default for ElasticsearchStorageConfig {
    /// We retrieve the elasticsearch configuration from ./config/elasticsearch/default.
    fn default() -> Self {
        let config = common::config::config_from(
            &PathBuf::from("config"),
            &["elasticsearch"],
            None,
            None,
            None,
        );

        config
            .expect("cannot build the configuration for testing from config")
            .get("elasticsearch")
            .expect("expected elasticsearch section in configuration from config")
    }
}

impl ElasticsearchStorageConfig {
    pub fn default_testing() -> Self {
        let config_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../config");

        let config = common::config::config_from(
            config_dir.as_path(),
            &["elasticsearch"],
            "testing",
            "MIMIR_TEST",
            None,
        );

        config
            .unwrap_or_else(|_| {
                panic!(
                    "cannot build the configuration for testing from {}",
                    config_dir.display(),
                )
            })
            .get("elasticsearch")
            .unwrap_or_else(|_| {
                panic!(
                    "expected elasticsearch section in configuration from {}",
                    config_dir.display(),
                )
            })
    }
}

#[cfg(test)]
pub mod tests {

    use config::Config;
    use serde::{Deserialize, Serialize};
    use serial_test::serial;

    use super::*;

    use crate::domain::ports::secondary::remote::Remote;
    use crate::domain::ports::secondary::storage::Storage;
    use crate::utils::docker;
    use common::document::{ContainerDocument, Document};

    // In this test we present an invalid configuration (its actually empty) to the
    // create_container function, and expect the error message to be meaningful
    #[tokio::test]
    #[serial]
    async fn should_return_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");

        let config = Config::builder().build().expect("build empty config");

        let res = client.create_container(config).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("Invalid Elasticsearch Index Configuration"));
    }

    #[tokio::test]
    #[serial]
    async fn should_connect_to_elasticsearch() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let _client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
    }

    #[tokio::test]
    #[serial]
    async fn should_create_index_with_valid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");
        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "force_merge": true,
                            "max_number_segments": 1,
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "index": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res.is_ok());
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_report_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "indx": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Container Creation Error"));
    }

    #[derive(Deserialize, Serialize)]
    struct TestObj {
        value: String,
    }

    impl Document for TestObj {
        fn id(&self) -> String {
            self.value.clone()
        }
    }

    impl ContainerDocument for TestObj {
        fn static_doc_type() -> &'static str {
            "test-obj"
        }

        fn default_es_container_config() -> config::Config {
            config::Config::builder()
                .set_default("container.name", Self::static_doc_type())
                .unwrap()
                .set_default("container.dataset", "default")
                .unwrap()
                .set_default("elasticsearch.parameters.timeout", "10s")
                .unwrap()
                .set_default("elasticsearch.parameters.force_merge", true)
                .unwrap()
                .set_default("elasticsearch.parameters.max_number_segments", 1)
                .unwrap()
                .set_default("elasticsearch.parameters.wait_for_active_shards", "1")
                .unwrap()
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "settings": { "index": { "number_of_shards": 1, "number_of_replicas": 1 } } } }"#,
                    config::FileFormat::Json,
                ))
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "mappings": { "properties": { "value": { "type": "text" } } } } }"#,
                    config::FileFormat::Json,
                ))
                .build()
                .expect("invalid container configuration for TestObj")
        }
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_insert_multiple_documents() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig::default_testing())
            .await
            .expect("Elasticsearch Connection Established");

        client
            .create_container(TestObj::default_es_container_config())
            .await
            .expect("container creation");

        let documents = vec![
            TestObj {
                value: String::from("obj1"),
            },
            TestObj {
                value: String::from("obj2"),
            },
            TestObj {
                value: String::from("obj3"),
            },
            TestObj {
                value: String::from("obj4"),
            },
            TestObj {
                value: String::from("obj5"),
            },
            TestObj {
                value: String::from("obj6"),
            },
        ];
        let documents = futures::stream::iter(documents);

        let res = client
            .insert_documents(
                String::from("root_obj_dataset_test-index-bulk-insert"),
                documents,
            )
            .await;

        assert_eq!(res.expect("insertion stats").created, 6);
    }

    #[tokio::test]
    #[serial]
    async fn should_detect_invalid_elasticsearch_version() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let client = remote::connection_test_pool()
            .conn(ElasticsearchStorageConfig {
                version_req: ">=9.99.99".to_string(),
                ..ElasticsearchStorageConfig::default_testing()
            })
            .await;

        assert!(client
            .unwrap_err()
            .to_string()
            .contains("Elasticsearch Invalid version"));
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::build_project::{build_project, commit_project, BuildProjectFailure};
use crate::config::Config;
use crate::errors::{Error, Result};
use crate::file_source::FileSource;
use crate::graphql_asts::GraphQLAsts;
use crate::red_to_green::RedToGreen;
use crate::{
    compiler_state::{ArtifactMapKind, CompilerState},
    file_source::FileSourceSubscriptionNextChange,
};
use common::{PerfLogEvent, PerfLogger};
use futures::future::join_all;
use log::{debug, info};
use rayon::prelude::*;
use std::sync::Arc;
use tokio::{
    sync::Notify,
    task::{self, JoinHandle},
};

pub struct Compiler<TPerfLogger>
where
    TPerfLogger: PerfLogger + 'static,
{
    config: Arc<Config>,
    perf_logger: Arc<TPerfLogger>,
}

impl<TPerfLogger: PerfLogger> Compiler<TPerfLogger> {
    pub fn new(config: Arc<Config>, perf_logger: Arc<TPerfLogger>) -> Self {
        Self {
            config,
            perf_logger,
        }
    }

    pub async fn compile(&self) -> Result<CompilerState> {
        let setup_event = self.perf_logger.create_event("compiler_setup");
        let file_source = FileSource::connect(&self.config, &setup_event).await?;
        let mut compiler_state = file_source
            .query(&setup_event, self.perf_logger.as_ref())
            .await?;

        self.build_projects(&mut compiler_state, &setup_event)
            .await?;

        self.perf_logger.complete_event(setup_event);
        Ok(compiler_state)
    }

    pub async fn watch(&self) -> Result<()> {
        loop {
            let setup_event = self.perf_logger.create_event("compiler_setup");
            let file_source = FileSource::connect(&self.config, &setup_event).await?;

            let (mut compiler_state, mut subscription) = file_source
                .subscribe(&setup_event, self.perf_logger.as_ref())
                .await?;

            let pending_file_source_changes = compiler_state.pending_file_source_changes.clone();
            let source_control_update_status =
                Arc::clone(&compiler_state.source_control_update_status);

            let notify_sender = Arc::new(Notify::new());
            let notify_receiver = notify_sender.clone();

            // First, set up watchman subscription
            let subscription_handle = task::spawn(async move {
                loop {
                    let next_change = subscription.next_change().await;
                    match next_change {
                        Ok(FileSourceSubscriptionNextChange::Result(file_source_changes)) => {
                            pending_file_source_changes
                                .write()
                                .unwrap()
                                .push(file_source_changes);
                            notify_sender.notify_one();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdateEnter) => {
                            info!("hg.update started...");
                            source_control_update_status.mark_as_started();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdateLeave) => {
                            info!("hg.update completed.");
                            source_control_update_status.set_to_default();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdate) => {
                            info!("hg.update completed. Detected new base revision...");
                            source_control_update_status.mark_as_completed();
                            notify_sender.notify_one();
                            break;
                        }
                        Ok(FileSourceSubscriptionNextChange::None) => {}
                        Err(err) => {
                            panic!("Watchman subscription error: {}", err);
                        }
                    }
                }
            });

            let mut red_to_green = RedToGreen::new();
            if self
                .build_projects(&mut compiler_state, &setup_event)
                .await
                .is_err()
            {
                // build_projects should have logged already
                red_to_green.log_error()
            } else {
                info!("Compilation completed.");
            }
            self.perf_logger.complete_event(setup_event);

            info!("Waiting for changes...");

            self.incremental_build_loop(compiler_state, notify_receiver, &subscription_handle)
                .await?
        }
    }

    async fn incremental_build_loop(
        &self,
        mut compiler_state: CompilerState,
        notify_receiver: Arc<Notify>,
        subscription_handle: &JoinHandle<()>,
    ) -> Result<()> {
        let mut red_to_green = RedToGreen::new();

        loop {
            notify_receiver.notified().await;

            if compiler_state.source_control_update_status.is_started() {
                continue;
            }

            if compiler_state.source_control_update_status.is_completed() {
                subscription_handle.abort();
                return Ok(());
            }

            // Single change to file sometimes produces 2 watchman change events for the same file
            // wait for 50ms in case there is a subsequent request
            tokio::time::sleep(std::time::Duration::from_millis(50)).await;

            if compiler_state.has_pending_file_source_changes() {
                let incremental_build_event =
                    self.perf_logger.create_event("incremental_build_event");
                let incremental_build_time =
                    incremental_build_event.start("incremental_build_time");

                let had_new_changes = compiler_state.merge_file_source_changes(
                    &self.config,
                    self.perf_logger.as_ref(),
                    false,
                )?;

                if had_new_changes {
                    info!("Change detected, start compiling...");
                    if self
                        .build_projects(&mut compiler_state, &incremental_build_event)
                        .await
                        .is_err()
                    {
                        red_to_green.log_error()
                    } else {
                        info!("Compilation completed.");
                        red_to_green.clear_error_and_log(self.perf_logger.as_ref());
                    }
                    incremental_build_event.stop(incremental_build_time);
                    info!("Waiting for changes...");
                } else {
                    debug!("No new changes detected.");
                    incremental_build_event.stop(incremental_build_time);
                }
                self.perf_logger.complete_event(incremental_build_event);
                // We probably don't want the messages queue to grow indefinitely
                // and we need to flush then, as the check/build is completed
                self.perf_logger.flush();
            }
        }
    }

    async fn build_projects(
        &self,
        compiler_state: &mut CompilerState,
        setup_event: &impl PerfLogEvent,
    ) -> Result<()> {
        self.config.status_reporter.build_starts();
        let build_projects_time = setup_event.start("build_projects_time");
        let result = build_projects(
            Arc::clone(&self.config),
            Arc::clone(&self.perf_logger),
            setup_event,
            compiler_state,
        )
        .await;
        setup_event.stop(build_projects_time);
        let result = setup_event.time("post_build_projects_time", || match result {
            Ok(()) => {
                compiler_state.complete_compilation();
                self.config.artifact_writer.finalize()?;
                if let Some(post_artifacts_write) = &self.config.post_artifacts_write {
                    if let Err(error) = post_artifacts_write(&self.config) {
                        let error = Error::PostArtifactsError { error };
                        Err(error)
                    } else {
                        Ok(())
                    }
                } else {
                    Ok(())
                }
            }
            Err(error) => Err(error),
        });
        self.config.status_reporter.build_finishes(&result);
        result
    }
}

async fn build_projects<TPerfLogger: PerfLogger + 'static>(
    config: Arc<Config>,
    perf_logger: Arc<TPerfLogger>,
    setup_event: &impl PerfLogEvent,
    compiler_state: &mut CompilerState,
) -> Result<()> {
    let mut graphql_asts = setup_event.time("parse_sources_time", || {
        GraphQLAsts::from_graphql_sources_map(
            &compiler_state.graphql_sources,
            &compiler_state.get_dirty_definitions(&config),
        )
    })?;

    if compiler_state.should_cancel_current_build() {
        debug!("Build is cancelled: new file changes are pending.");
        return Err(Error::Cancelled);
    }

    let build_results: Vec<_> = config
        .par_enabled_projects()
        .filter(|project_config| {
            if let Some(base) = project_config.base {
                if compiler_state.project_has_pending_changes(base) {
                    return true;
                }
            }
            compiler_state.project_has_pending_changes(project_config.name)
        })
        .map(|project_config| {
            build_project(
                &config,
                project_config,
                compiler_state,
                &graphql_asts,
                Arc::clone(&perf_logger),
            )
        })
        .collect();
    let mut results = Vec::new();
    let mut errors = Vec::new();
    for result in build_results {
        match result {
            Ok(result) => results.push(result),
            Err(error) => match error {
                BuildProjectFailure::Error(error) => errors.push(error),
                BuildProjectFailure::Cancelled => {
                    return Err(Error::Cancelled);
                }
            },
        }
    }

    let mut build_cancelled_during_commit = false;
    if errors.is_empty() {
        if compiler_state.should_cancel_current_build() {
            debug!("Build is cancelled: updates in source code/or new file changes are pending.");
            return Err(Error::Cancelled);
        }

        let mut handles: Vec<JoinHandle<std::result::Result<_, BuildProjectFailure>>> = Vec::new();
        for (project_name, schema, programs, artifacts) in results {
            let config = Arc::clone(&config);
            let perf_logger = Arc::clone(&perf_logger);
            let artifact_map = compiler_state
                .artifacts
                .get(&project_name)
                .cloned()
                .unwrap_or_else(|| Arc::new(ArtifactMapKind::Unconnected(Default::default())));
            let removed_definition_names = graphql_asts
                .remove(&project_name)
                .expect("Expect GraphQLAsts to exist.")
                .removed_definition_names;
            let dirty_artifact_paths = compiler_state
                .dirty_artifact_paths
                .get(&project_name)
                .cloned()
                .unwrap_or_default();

            let source_control_update_status =
                Arc::clone(&compiler_state.source_control_update_status);
            handles.push(task::spawn(async move {
                let project_config = &config.projects[&project_name];
                Ok((
                    project_name,
                    commit_project(
                        &config,
                        project_config,
                        perf_logger,
                        &schema,
                        programs,
                        artifacts,
                        artifact_map,
                        removed_definition_names,
                        dirty_artifact_paths,
                        source_control_update_status,
                    )
                    .await?,
                    schema,
                ))
            }));
        }

        for commit_result in join_all(handles).await {
            let commit_result: std::result::Result<std::result::Result<_, _>, _> = commit_result;
            let inner_result = commit_result.map_err(|e| Error::JoinError {
                error: e.to_string(),
            })?;
            match inner_result {
                Ok((project_name, next_artifact_map, schema)) => {
                    let next_artifact_map = Arc::new(ArtifactMapKind::Mapping(next_artifact_map));
                    compiler_state
                        .artifacts
                        .insert(project_name, next_artifact_map);
                    compiler_state.schema_cache.insert(project_name, schema);
                }
                Err(BuildProjectFailure::Error(error)) => {
                    errors.push(error);
                }
                Err(BuildProjectFailure::Cancelled) => {
                    build_cancelled_during_commit = true;
                }
            }
        }
    }

    if errors.is_empty() {
        match build_cancelled_during_commit {
            true => Err(Error::Cancelled),
            false => Ok(()),
        }
    } else {
        Err(Error::BuildProjectsErrors { errors })
    }
}
<commit_msg>Flush perf logs in more scenarios<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::build_project::{build_project, commit_project, BuildProjectFailure};
use crate::config::Config;
use crate::errors::{Error, Result};
use crate::file_source::FileSource;
use crate::graphql_asts::GraphQLAsts;
use crate::red_to_green::RedToGreen;
use crate::{
    compiler_state::{ArtifactMapKind, CompilerState},
    file_source::FileSourceSubscriptionNextChange,
};
use common::{PerfLogEvent, PerfLogger};
use futures::future::join_all;
use log::{debug, info};
use rayon::prelude::*;
use std::sync::Arc;
use tokio::{
    sync::Notify,
    task::{self, JoinHandle},
};

pub struct Compiler<TPerfLogger>
where
    TPerfLogger: PerfLogger + 'static,
{
    config: Arc<Config>,
    perf_logger: Arc<TPerfLogger>,
}

impl<TPerfLogger: PerfLogger> Compiler<TPerfLogger> {
    pub fn new(config: Arc<Config>, perf_logger: Arc<TPerfLogger>) -> Self {
        Self {
            config,
            perf_logger,
        }
    }

    pub async fn compile(&self) -> Result<CompilerState> {
        let setup_event = self.perf_logger.create_event("compiler_setup");
        let file_source = FileSource::connect(&self.config, &setup_event).await?;
        let mut compiler_state = file_source
            .query(&setup_event, self.perf_logger.as_ref())
            .await?;

        self.build_projects(&mut compiler_state, &setup_event)
            .await?;

        self.perf_logger.complete_event(setup_event);
        Ok(compiler_state)
    }

    pub async fn watch(&self) -> Result<()> {
        loop {
            let setup_event = self.perf_logger.create_event("compiler_setup");
            let file_source = FileSource::connect(&self.config, &setup_event).await?;

            let (mut compiler_state, mut subscription) = file_source
                .subscribe(&setup_event, self.perf_logger.as_ref())
                .await?;

            let pending_file_source_changes = compiler_state.pending_file_source_changes.clone();
            let source_control_update_status =
                Arc::clone(&compiler_state.source_control_update_status);

            let notify_sender = Arc::new(Notify::new());
            let notify_receiver = notify_sender.clone();

            // First, set up watchman subscription
            let subscription_handle = task::spawn(async move {
                loop {
                    let next_change = subscription.next_change().await;
                    match next_change {
                        Ok(FileSourceSubscriptionNextChange::Result(file_source_changes)) => {
                            pending_file_source_changes
                                .write()
                                .unwrap()
                                .push(file_source_changes);
                            notify_sender.notify_one();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdateEnter) => {
                            info!("hg.update started...");
                            source_control_update_status.mark_as_started();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdateLeave) => {
                            info!("hg.update completed.");
                            source_control_update_status.set_to_default();
                        }
                        Ok(FileSourceSubscriptionNextChange::SourceControlUpdate) => {
                            info!("hg.update completed. Detected new base revision...");
                            source_control_update_status.mark_as_completed();
                            notify_sender.notify_one();
                            break;
                        }
                        Ok(FileSourceSubscriptionNextChange::None) => {}
                        Err(err) => {
                            panic!("Watchman subscription error: {}", err);
                        }
                    }
                }
            });

            let mut red_to_green = RedToGreen::new();
            if self
                .build_projects(&mut compiler_state, &setup_event)
                .await
                .is_err()
            {
                // build_projects should have logged already
                red_to_green.log_error()
            } else {
                info!("Compilation completed.");
            }
            self.perf_logger.complete_event(setup_event);
            self.perf_logger.flush();
            info!("Waiting for changes...");

            self.incremental_build_loop(compiler_state, notify_receiver, &subscription_handle)
                .await?
        }
    }

    async fn incremental_build_loop(
        &self,
        mut compiler_state: CompilerState,
        notify_receiver: Arc<Notify>,
        subscription_handle: &JoinHandle<()>,
    ) -> Result<()> {
        let mut red_to_green = RedToGreen::new();

        loop {
            notify_receiver.notified().await;

            if compiler_state.source_control_update_status.is_started() {
                continue;
            }

            if compiler_state.source_control_update_status.is_completed() {
                subscription_handle.abort();
                return Ok(());
            }

            // Single change to file sometimes produces 2 watchman change events for the same file
            // wait for 50ms in case there is a subsequent request
            tokio::time::sleep(std::time::Duration::from_millis(50)).await;

            if compiler_state.has_pending_file_source_changes() {
                let incremental_build_event =
                    self.perf_logger.create_event("incremental_build_event");
                let incremental_build_time =
                    incremental_build_event.start("incremental_build_time");

                let had_new_changes = compiler_state.merge_file_source_changes(
                    &self.config,
                    self.perf_logger.as_ref(),
                    false,
                )?;

                if had_new_changes {
                    info!("Change detected, start compiling...");
                    if self
                        .build_projects(&mut compiler_state, &incremental_build_event)
                        .await
                        .is_err()
                    {
                        red_to_green.log_error()
                    } else {
                        info!("Compilation completed.");
                        red_to_green.clear_error_and_log(self.perf_logger.as_ref());
                    }
                    incremental_build_event.stop(incremental_build_time);
                    info!("Waiting for changes...");
                } else {
                    debug!("No new changes detected.");
                    incremental_build_event.stop(incremental_build_time);
                }
                self.perf_logger.complete_event(incremental_build_event);
                // We probably don't want the messages queue to grow indefinitely
                // and we need to flush then, as the check/build is completed
                self.perf_logger.flush();
            }
        }
    }

    async fn build_projects(
        &self,
        compiler_state: &mut CompilerState,
        setup_event: &impl PerfLogEvent,
    ) -> Result<()> {
        self.config.status_reporter.build_starts();
        let build_projects_time = setup_event.start("build_projects_time");
        let result = build_projects(
            Arc::clone(&self.config),
            Arc::clone(&self.perf_logger),
            setup_event,
            compiler_state,
        )
        .await;
        setup_event.stop(build_projects_time);
        let result = setup_event.time("post_build_projects_time", || match result {
            Ok(()) => {
                compiler_state.complete_compilation();
                self.config.artifact_writer.finalize()?;
                if let Some(post_artifacts_write) = &self.config.post_artifacts_write {
                    if let Err(error) = post_artifacts_write(&self.config) {
                        let error = Error::PostArtifactsError { error };
                        Err(error)
                    } else {
                        Ok(())
                    }
                } else {
                    Ok(())
                }
            }
            Err(error) => Err(error),
        });
        self.config.status_reporter.build_finishes(&result);
        result
    }
}

async fn build_projects<TPerfLogger: PerfLogger + 'static>(
    config: Arc<Config>,
    perf_logger: Arc<TPerfLogger>,
    setup_event: &impl PerfLogEvent,
    compiler_state: &mut CompilerState,
) -> Result<()> {
    let mut graphql_asts = setup_event.time("parse_sources_time", || {
        GraphQLAsts::from_graphql_sources_map(
            &compiler_state.graphql_sources,
            &compiler_state.get_dirty_definitions(&config),
        )
    })?;

    if compiler_state.should_cancel_current_build() {
        debug!("Build is cancelled: new file changes are pending.");
        return Err(Error::Cancelled);
    }

    let build_results: Vec<_> = config
        .par_enabled_projects()
        .filter(|project_config| {
            if let Some(base) = project_config.base {
                if compiler_state.project_has_pending_changes(base) {
                    return true;
                }
            }
            compiler_state.project_has_pending_changes(project_config.name)
        })
        .map(|project_config| {
            build_project(
                &config,
                project_config,
                compiler_state,
                &graphql_asts,
                Arc::clone(&perf_logger),
            )
        })
        .collect();
    let mut results = Vec::new();
    let mut errors = Vec::new();
    for result in build_results {
        match result {
            Ok(result) => results.push(result),
            Err(error) => match error {
                BuildProjectFailure::Error(error) => errors.push(error),
                BuildProjectFailure::Cancelled => {
                    return Err(Error::Cancelled);
                }
            },
        }
    }

    let mut build_cancelled_during_commit = false;
    if errors.is_empty() {
        if compiler_state.should_cancel_current_build() {
            debug!("Build is cancelled: updates in source code/or new file changes are pending.");
            return Err(Error::Cancelled);
        }

        let mut handles: Vec<JoinHandle<std::result::Result<_, BuildProjectFailure>>> = Vec::new();
        for (project_name, schema, programs, artifacts) in results {
            let config = Arc::clone(&config);
            let perf_logger = Arc::clone(&perf_logger);
            let artifact_map = compiler_state
                .artifacts
                .get(&project_name)
                .cloned()
                .unwrap_or_else(|| Arc::new(ArtifactMapKind::Unconnected(Default::default())));
            let removed_definition_names = graphql_asts
                .remove(&project_name)
                .expect("Expect GraphQLAsts to exist.")
                .removed_definition_names;
            let dirty_artifact_paths = compiler_state
                .dirty_artifact_paths
                .get(&project_name)
                .cloned()
                .unwrap_or_default();

            let source_control_update_status =
                Arc::clone(&compiler_state.source_control_update_status);
            handles.push(task::spawn(async move {
                let project_config = &config.projects[&project_name];
                Ok((
                    project_name,
                    commit_project(
                        &config,
                        project_config,
                        perf_logger,
                        &schema,
                        programs,
                        artifacts,
                        artifact_map,
                        removed_definition_names,
                        dirty_artifact_paths,
                        source_control_update_status,
                    )
                    .await?,
                    schema,
                ))
            }));
        }

        for commit_result in join_all(handles).await {
            let commit_result: std::result::Result<std::result::Result<_, _>, _> = commit_result;
            let inner_result = commit_result.map_err(|e| Error::JoinError {
                error: e.to_string(),
            })?;
            match inner_result {
                Ok((project_name, next_artifact_map, schema)) => {
                    let next_artifact_map = Arc::new(ArtifactMapKind::Mapping(next_artifact_map));
                    compiler_state
                        .artifacts
                        .insert(project_name, next_artifact_map);
                    compiler_state.schema_cache.insert(project_name, schema);
                }
                Err(BuildProjectFailure::Error(error)) => {
                    errors.push(error);
                }
                Err(BuildProjectFailure::Cancelled) => {
                    build_cancelled_during_commit = true;
                }
            }
        }
    }

    if errors.is_empty() {
        match build_cancelled_during_commit {
            true => Err(Error::Cancelled),
            false => Ok(()),
        }
    } else {
        Err(Error::BuildProjectsErrors { errors })
    }
}
<|endoftext|>


<commit_before>pub use super::*;

use crate::dataflow::generic::{AnalysisDomain, GenKill, GenKillAnalysis};
use rustc::mir::visit::Visitor;
use rustc::mir::*;
use rustc::ty::{ParamEnv, TyCtxt};
use rustc_span::DUMMY_SP;

pub type MaybeMutBorrowedLocals<'mir, 'tcx> = MaybeBorrowedLocals<MutBorrow<'mir, 'tcx>>;

/// A dataflow analysis that tracks whether a pointer or reference could possibly exist that points
/// to a given local.
///
/// The `K` parameter determines what kind of borrows are tracked. By default,
/// `MaybeBorrowedLocals` looks for *any* borrow of a local. If you are only interested in borrows
/// that might allow mutation, use the `MaybeMutBorrowedLocals` type alias instead.
///
/// At present, this is used as a very limited form of alias analysis. For example,
/// `MaybeBorrowedLocals` is used to compute which locals are live during a yield expression for
/// immovable generators. `MaybeMutBorrowedLocals` is used during const checking to prove that a
/// local has not been mutated via indirect assignment (e.g., `*p = 42`), the side-effects of a
/// function call or inline assembly.
pub struct MaybeBorrowedLocals<K = AnyBorrow> {
    kind: K,
    ignore_borrow_on_drop: bool,
}

impl MaybeBorrowedLocals {
    /// A dataflow analysis that records whether a pointer or reference exists that may alias the
    /// given local.
    pub fn all_borrows() -> Self {
        MaybeBorrowedLocals { kind: AnyBorrow, ignore_borrow_on_drop: false }
    }
}

impl MaybeMutBorrowedLocals<'mir, 'tcx> {
    /// A dataflow analysis that records whether a pointer or reference exists that may *mutably*
    /// alias the given local.
    ///
    /// This includes `&mut` and pointers derived from an `&mut`, as well as shared borrows of
    /// types with interior mutability.
    pub fn mut_borrows_only(
        tcx: TyCtxt<'tcx>,
        body: &'mir mir::Body<'tcx>,
        param_env: ParamEnv<'tcx>,
    ) -> Self {
        MaybeBorrowedLocals {
            kind: MutBorrow { body, tcx, param_env },
            ignore_borrow_on_drop: false,
        }
    }
}

impl<K> MaybeBorrowedLocals<K> {
    /// During dataflow analysis, ignore the borrow that may occur when a place is dropped.
    ///
    /// Drop terminators may call custom drop glue (`Drop::drop`), which takes `&mut self` as a
    /// parameter. In the general case, a drop impl could launder that reference into the
    /// surrounding environment through a raw pointer, thus creating a valid `*mut` pointing to the
    /// dropped local. We are not yet willing to declare this particular case UB, so we must treat
    /// all dropped locals as mutably borrowed for now. See discussion on [#61069].
    ///
    /// In some contexts, we know that this borrow will never occur. For example, during
    /// const-eval, custom drop glue cannot be run. Code that calls this should document the
    /// assumptions that justify `Drop` terminators in this way.
    ///
    /// [#61069]: https://github.com/rust-lang/rust/pull/61069
    pub fn unsound_ignore_borrow_on_drop(self) -> Self {
        MaybeBorrowedLocals { ignore_borrow_on_drop: true, ..self }
    }

    fn transfer_function<'a, T>(&'a self, trans: &'a mut T) -> TransferFunction<'a, T, K> {
        TransferFunction {
            kind: &self.kind,
            trans,
            ignore_borrow_on_drop: self.ignore_borrow_on_drop,
        }
    }
}

impl<K> AnalysisDomain<'tcx> for MaybeBorrowedLocals<K>
where
    K: BorrowAnalysisKind<'tcx>,
{
    type Idx = Local;

    const NAME: &'static str = K::ANALYSIS_NAME;

    fn bits_per_block(&self, body: &mir::Body<'tcx>) -> usize {
        body.local_decls().len()
    }

    fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut BitSet<Self::Idx>) {
        // No locals are aliased on function entry
    }
}

impl<K> GenKillAnalysis<'tcx> for MaybeBorrowedLocals<K>
where
    K: BorrowAnalysisKind<'tcx>,
{
    fn statement_effect(
        &self,
        trans: &mut impl GenKill<Self::Idx>,
        statement: &mir::Statement<'tcx>,
        location: Location,
    ) {
        self.transfer_function(trans).visit_statement(statement, location);
    }

    fn terminator_effect(
        &self,
        trans: &mut impl GenKill<Self::Idx>,
        terminator: &mir::Terminator<'tcx>,
        location: Location,
    ) {
        self.transfer_function(trans).visit_terminator(terminator, location);
    }

    fn call_return_effect(
        &self,
        _trans: &mut impl GenKill<Self::Idx>,
        _block: mir::BasicBlock,
        _func: &mir::Operand<'tcx>,
        _args: &[mir::Operand<'tcx>],
        _dest_place: &mir::Place<'tcx>,
    ) {
    }
}

impl<K> BottomValue for MaybeBorrowedLocals<K> {
    // bottom = unborrowed
    const BOTTOM_VALUE: bool = false;
}

/// A `Visitor` that defines the transfer function for `MaybeBorrowedLocals`.
struct TransferFunction<'a, T, K> {
    trans: &'a mut T,
    kind: &'a K,
    ignore_borrow_on_drop: bool,
}

impl<T, K> Visitor<'tcx> for TransferFunction<'a, T, K>
where
    T: GenKill<Local>,
    K: BorrowAnalysisKind<'tcx>,
{
    fn visit_statement(&mut self, stmt: &Statement<'tcx>, location: Location) {
        self.super_statement(stmt, location);

        // When we reach a `StorageDead` statement, we can assume that any pointers to this memory
        // are now invalid.
        if let StatementKind::StorageDead(local) = stmt.kind {
            self.trans.kill(local);
        }
    }

    fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) {
        self.super_rvalue(rvalue, location);

        match rvalue {
            mir::Rvalue::AddressOf(mt, borrowed_place) => {
                if !borrowed_place.is_indirect() && self.kind.in_address_of(*mt, borrowed_place) {
                    self.trans.gen(borrowed_place.local);
                }
            }

            mir::Rvalue::Ref(_, kind, borrowed_place) => {
                if !borrowed_place.is_indirect() && self.kind.in_ref(*kind, borrowed_place) {
                    self.trans.gen(borrowed_place.local);
                }
            }

            mir::Rvalue::Cast(..)
            | mir::Rvalue::Use(..)
            | mir::Rvalue::Repeat(..)
            | mir::Rvalue::Len(..)
            | mir::Rvalue::BinaryOp(..)
            | mir::Rvalue::CheckedBinaryOp(..)
            | mir::Rvalue::NullaryOp(..)
            | mir::Rvalue::UnaryOp(..)
            | mir::Rvalue::Discriminant(..)
            | mir::Rvalue::Aggregate(..) => {}
        }
    }

    fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) {
        self.super_terminator(terminator, location);

        match terminator.kind {
            mir::TerminatorKind::Drop { location: dropped_place, .. }
            | mir::TerminatorKind::DropAndReplace { location: dropped_place, .. } => {
                // See documentation for `unsound_ignore_borrow_on_drop` for an explanation.
                if !self.ignore_borrow_on_drop {
                    self.trans.gen(dropped_place.local);
                }
            }

            TerminatorKind::Abort
            | TerminatorKind::Assert { .. }
            | TerminatorKind::Call { .. }
            | TerminatorKind::FalseEdges { .. }
            | TerminatorKind::FalseUnwind { .. }
            | TerminatorKind::GeneratorDrop
            | TerminatorKind::Goto { .. }
            | TerminatorKind::Resume
            | TerminatorKind::Return
            | TerminatorKind::SwitchInt { .. }
            | TerminatorKind::Unreachable
            | TerminatorKind::Yield { .. } => {}
        }
    }
}

pub struct AnyBorrow;

pub struct MutBorrow<'mir, 'tcx> {
    tcx: TyCtxt<'tcx>,
    body: &'mir Body<'tcx>,
    param_env: ParamEnv<'tcx>,
}

impl MutBorrow<'mir, 'tcx> {
    // `&` and `&raw` only allow mutation if the borrowed place is `!Freeze`.
    //
    // This assumes that it is UB to take the address of a struct field whose type is
    // `Freeze`, then use pointer arithmetic to derive a pointer to a *different* field of
    // that same struct whose type is `!Freeze`. If we decide that this is not UB, we will
    // have to check the type of the borrowed **local** instead of the borrowed **place**
    // below. See [rust-lang/unsafe-code-guidelines#134].
    //
    // [rust-lang/unsafe-code-guidelines#134]: https://github.com/rust-lang/unsafe-code-guidelines/issues/134
    fn shared_borrow_allows_mutation(&self, place: &Place<'tcx>) -> bool {
        !place.ty(self.body, self.tcx).ty.is_freeze(self.tcx, self.param_env, DUMMY_SP)
    }
}

pub trait BorrowAnalysisKind<'tcx> {
    const ANALYSIS_NAME: &'static str;

    fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool;
    fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool;
}

impl BorrowAnalysisKind<'tcx> for AnyBorrow {
    const ANALYSIS_NAME: &'static str = "maybe_borrowed_locals";

    fn in_ref(&self, _: mir::BorrowKind, _: &Place<'_>) -> bool {
        true
    }
    fn in_address_of(&self, _: Mutability, _: &Place<'_>) -> bool {
        true
    }
}

impl BorrowAnalysisKind<'tcx> for MutBorrow<'mir, 'tcx> {
    const ANALYSIS_NAME: &'static str = "maybe_mut_borrowed_locals";

    fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool {
        match kind {
            mir::BorrowKind::Mut { .. } => true,
            mir::BorrowKind::Shared | mir::BorrowKind::Shallow | mir::BorrowKind::Unique => {
                self.shared_borrow_allows_mutation(place)
            }
        }
    }

    fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool {
        match mt {
            Mutability::Mut => true,
            Mutability::Not => self.shared_borrow_allows_mutation(place),
        }
    }
}
<commit_msg>Use doc comment for explanation of `shared_borrow_allows_mutation`<commit_after>pub use super::*;

use crate::dataflow::generic::{AnalysisDomain, GenKill, GenKillAnalysis};
use rustc::mir::visit::Visitor;
use rustc::mir::*;
use rustc::ty::{ParamEnv, TyCtxt};
use rustc_span::DUMMY_SP;

pub type MaybeMutBorrowedLocals<'mir, 'tcx> = MaybeBorrowedLocals<MutBorrow<'mir, 'tcx>>;

/// A dataflow analysis that tracks whether a pointer or reference could possibly exist that points
/// to a given local.
///
/// The `K` parameter determines what kind of borrows are tracked. By default,
/// `MaybeBorrowedLocals` looks for *any* borrow of a local. If you are only interested in borrows
/// that might allow mutation, use the `MaybeMutBorrowedLocals` type alias instead.
///
/// At present, this is used as a very limited form of alias analysis. For example,
/// `MaybeBorrowedLocals` is used to compute which locals are live during a yield expression for
/// immovable generators. `MaybeMutBorrowedLocals` is used during const checking to prove that a
/// local has not been mutated via indirect assignment (e.g., `*p = 42`), the side-effects of a
/// function call or inline assembly.
pub struct MaybeBorrowedLocals<K = AnyBorrow> {
    kind: K,
    ignore_borrow_on_drop: bool,
}

impl MaybeBorrowedLocals {
    /// A dataflow analysis that records whether a pointer or reference exists that may alias the
    /// given local.
    pub fn all_borrows() -> Self {
        MaybeBorrowedLocals { kind: AnyBorrow, ignore_borrow_on_drop: false }
    }
}

impl MaybeMutBorrowedLocals<'mir, 'tcx> {
    /// A dataflow analysis that records whether a pointer or reference exists that may *mutably*
    /// alias the given local.
    ///
    /// This includes `&mut` and pointers derived from an `&mut`, as well as shared borrows of
    /// types with interior mutability.
    pub fn mut_borrows_only(
        tcx: TyCtxt<'tcx>,
        body: &'mir mir::Body<'tcx>,
        param_env: ParamEnv<'tcx>,
    ) -> Self {
        MaybeBorrowedLocals {
            kind: MutBorrow { body, tcx, param_env },
            ignore_borrow_on_drop: false,
        }
    }
}

impl<K> MaybeBorrowedLocals<K> {
    /// During dataflow analysis, ignore the borrow that may occur when a place is dropped.
    ///
    /// Drop terminators may call custom drop glue (`Drop::drop`), which takes `&mut self` as a
    /// parameter. In the general case, a drop impl could launder that reference into the
    /// surrounding environment through a raw pointer, thus creating a valid `*mut` pointing to the
    /// dropped local. We are not yet willing to declare this particular case UB, so we must treat
    /// all dropped locals as mutably borrowed for now. See discussion on [#61069].
    ///
    /// In some contexts, we know that this borrow will never occur. For example, during
    /// const-eval, custom drop glue cannot be run. Code that calls this should document the
    /// assumptions that justify `Drop` terminators in this way.
    ///
    /// [#61069]: https://github.com/rust-lang/rust/pull/61069
    pub fn unsound_ignore_borrow_on_drop(self) -> Self {
        MaybeBorrowedLocals { ignore_borrow_on_drop: true, ..self }
    }

    fn transfer_function<'a, T>(&'a self, trans: &'a mut T) -> TransferFunction<'a, T, K> {
        TransferFunction {
            kind: &self.kind,
            trans,
            ignore_borrow_on_drop: self.ignore_borrow_on_drop,
        }
    }
}

impl<K> AnalysisDomain<'tcx> for MaybeBorrowedLocals<K>
where
    K: BorrowAnalysisKind<'tcx>,
{
    type Idx = Local;

    const NAME: &'static str = K::ANALYSIS_NAME;

    fn bits_per_block(&self, body: &mir::Body<'tcx>) -> usize {
        body.local_decls().len()
    }

    fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut BitSet<Self::Idx>) {
        // No locals are aliased on function entry
    }
}

impl<K> GenKillAnalysis<'tcx> for MaybeBorrowedLocals<K>
where
    K: BorrowAnalysisKind<'tcx>,
{
    fn statement_effect(
        &self,
        trans: &mut impl GenKill<Self::Idx>,
        statement: &mir::Statement<'tcx>,
        location: Location,
    ) {
        self.transfer_function(trans).visit_statement(statement, location);
    }

    fn terminator_effect(
        &self,
        trans: &mut impl GenKill<Self::Idx>,
        terminator: &mir::Terminator<'tcx>,
        location: Location,
    ) {
        self.transfer_function(trans).visit_terminator(terminator, location);
    }

    fn call_return_effect(
        &self,
        _trans: &mut impl GenKill<Self::Idx>,
        _block: mir::BasicBlock,
        _func: &mir::Operand<'tcx>,
        _args: &[mir::Operand<'tcx>],
        _dest_place: &mir::Place<'tcx>,
    ) {
    }
}

impl<K> BottomValue for MaybeBorrowedLocals<K> {
    // bottom = unborrowed
    const BOTTOM_VALUE: bool = false;
}

/// A `Visitor` that defines the transfer function for `MaybeBorrowedLocals`.
struct TransferFunction<'a, T, K> {
    trans: &'a mut T,
    kind: &'a K,
    ignore_borrow_on_drop: bool,
}

impl<T, K> Visitor<'tcx> for TransferFunction<'a, T, K>
where
    T: GenKill<Local>,
    K: BorrowAnalysisKind<'tcx>,
{
    fn visit_statement(&mut self, stmt: &Statement<'tcx>, location: Location) {
        self.super_statement(stmt, location);

        // When we reach a `StorageDead` statement, we can assume that any pointers to this memory
        // are now invalid.
        if let StatementKind::StorageDead(local) = stmt.kind {
            self.trans.kill(local);
        }
    }

    fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) {
        self.super_rvalue(rvalue, location);

        match rvalue {
            mir::Rvalue::AddressOf(mt, borrowed_place) => {
                if !borrowed_place.is_indirect() && self.kind.in_address_of(*mt, borrowed_place) {
                    self.trans.gen(borrowed_place.local);
                }
            }

            mir::Rvalue::Ref(_, kind, borrowed_place) => {
                if !borrowed_place.is_indirect() && self.kind.in_ref(*kind, borrowed_place) {
                    self.trans.gen(borrowed_place.local);
                }
            }

            mir::Rvalue::Cast(..)
            | mir::Rvalue::Use(..)
            | mir::Rvalue::Repeat(..)
            | mir::Rvalue::Len(..)
            | mir::Rvalue::BinaryOp(..)
            | mir::Rvalue::CheckedBinaryOp(..)
            | mir::Rvalue::NullaryOp(..)
            | mir::Rvalue::UnaryOp(..)
            | mir::Rvalue::Discriminant(..)
            | mir::Rvalue::Aggregate(..) => {}
        }
    }

    fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) {
        self.super_terminator(terminator, location);

        match terminator.kind {
            mir::TerminatorKind::Drop { location: dropped_place, .. }
            | mir::TerminatorKind::DropAndReplace { location: dropped_place, .. } => {
                // See documentation for `unsound_ignore_borrow_on_drop` for an explanation.
                if !self.ignore_borrow_on_drop {
                    self.trans.gen(dropped_place.local);
                }
            }

            TerminatorKind::Abort
            | TerminatorKind::Assert { .. }
            | TerminatorKind::Call { .. }
            | TerminatorKind::FalseEdges { .. }
            | TerminatorKind::FalseUnwind { .. }
            | TerminatorKind::GeneratorDrop
            | TerminatorKind::Goto { .. }
            | TerminatorKind::Resume
            | TerminatorKind::Return
            | TerminatorKind::SwitchInt { .. }
            | TerminatorKind::Unreachable
            | TerminatorKind::Yield { .. } => {}
        }
    }
}

pub struct AnyBorrow;

pub struct MutBorrow<'mir, 'tcx> {
    tcx: TyCtxt<'tcx>,
    body: &'mir Body<'tcx>,
    param_env: ParamEnv<'tcx>,
}

impl MutBorrow<'mir, 'tcx> {
    /// `&` and `&raw` only allow mutation if the borrowed place is `!Freeze`.
    ///
    /// This assumes that it is UB to take the address of a struct field whose type is
    /// `Freeze`, then use pointer arithmetic to derive a pointer to a *different* field of
    /// that same struct whose type is `!Freeze`. If we decide that this is not UB, we will
    /// have to check the type of the borrowed **local** instead of the borrowed **place**
    /// below. See [rust-lang/unsafe-code-guidelines#134].
    ///
    /// [rust-lang/unsafe-code-guidelines#134]: https://github.com/rust-lang/unsafe-code-guidelines/issues/134
    fn shared_borrow_allows_mutation(&self, place: &Place<'tcx>) -> bool {
        !place.ty(self.body, self.tcx).ty.is_freeze(self.tcx, self.param_env, DUMMY_SP)
    }
}

pub trait BorrowAnalysisKind<'tcx> {
    const ANALYSIS_NAME: &'static str;

    fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool;
    fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool;
}

impl BorrowAnalysisKind<'tcx> for AnyBorrow {
    const ANALYSIS_NAME: &'static str = "maybe_borrowed_locals";

    fn in_ref(&self, _: mir::BorrowKind, _: &Place<'_>) -> bool {
        true
    }
    fn in_address_of(&self, _: Mutability, _: &Place<'_>) -> bool {
        true
    }
}

impl BorrowAnalysisKind<'tcx> for MutBorrow<'mir, 'tcx> {
    const ANALYSIS_NAME: &'static str = "maybe_mut_borrowed_locals";

    fn in_ref(&self, kind: mir::BorrowKind, place: &Place<'tcx>) -> bool {
        match kind {
            mir::BorrowKind::Mut { .. } => true,
            mir::BorrowKind::Shared | mir::BorrowKind::Shallow | mir::BorrowKind::Unique => {
                self.shared_borrow_allows_mutation(place)
            }
        }
    }

    fn in_address_of(&self, mt: Mutability, place: &Place<'tcx>) -> bool {
        match mt {
            Mutability::Mut => true,
            Mutability::Not => self.shared_borrow_allows_mutation(place),
        }
    }
}
<|endoftext|>


<commit_before>//! Benchmark operations like highlighting or goto definition.

use std::{env, path::Path, str::FromStr, sync::Arc, time::Instant};

use anyhow::{format_err, Result};
use ra_db::{
    salsa::{Database, Durability},
    AbsPathBuf, FileId,
};
use ra_ide::{Analysis, AnalysisChange, AnalysisHost, CompletionConfig, FilePosition, LineCol};

use crate::cli::{load_cargo::load_cargo, Verbosity};

pub enum BenchWhat {
    Highlight { path: AbsPathBuf },
    Complete(Position),
    GotoDef(Position),
}

pub struct Position {
    pub path: AbsPathBuf,
    pub line: u32,
    pub column: u32,
}

impl FromStr for Position {
    type Err = anyhow::Error;
    fn from_str(s: &str) -> Result<Self> {
        let (path_line, column) = rsplit_at_char(s, ':')?;
        let (path, line) = rsplit_at_char(path_line, ':')?;
        let path = env::current_dir().unwrap().join(path);
        let path = AbsPathBuf::assert(path);
        Ok(Position { path, line: line.parse()?, column: column.parse()? })
    }
}

fn rsplit_at_char(s: &str, c: char) -> Result<(&str, &str)> {
    let idx = s.rfind(c).ok_or_else(|| format_err!("no `{}` in {}", c, s))?;
    Ok((&s[..idx], &s[idx + 1..]))
}

pub fn analysis_bench(
    verbosity: Verbosity,
    path: &Path,
    what: BenchWhat,
    load_output_dirs: bool,
    with_proc_macro: bool,
) -> Result<()> {
    ra_prof::init();

    let start = Instant::now();
    eprint!("loading: ");
    let (mut host, vfs) = load_cargo(path, load_output_dirs, with_proc_macro)?;
    eprintln!("{:?}\n", start.elapsed());

    let file_id = {
        let path = match &what {
            BenchWhat::Highlight { path } => path,
            BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => &pos.path,
        };
        let path = path.clone().into();
        vfs.file_id(&path).ok_or_else(|| format_err!("Can't find {}", path))?
    };

    match &what {
        BenchWhat::Highlight { .. } => {
            let res = do_work(&mut host, file_id, |analysis| {
                analysis.diagnostics(file_id).unwrap();
                analysis.highlight_as_html(file_id, false).unwrap()
            });
            if verbosity.is_verbose() {
                println!("\n{}", res);
            }
        }
        BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => {
            let is_completion = matches!(what, BenchWhat::Complete(..));

            let offset = host
                .analysis()
                .file_line_index(file_id)?
                .offset(LineCol { line: pos.line - 1, col_utf16: pos.column });
            let file_position = FilePosition { file_id, offset };

            if is_completion {
                let options = CompletionConfig::default();
                let res = do_work(&mut host, file_id, |analysis| {
                    analysis.completions(&options, file_position)
                });
                if verbosity.is_verbose() {
                    println!("\n{:#?}", res);
                }
            } else {
                let res =
                    do_work(&mut host, file_id, |analysis| analysis.goto_definition(file_position));
                if verbosity.is_verbose() {
                    println!("\n{:#?}", res);
                }
            }
        }
    }
    Ok(())
}

fn do_work<F: Fn(&Analysis) -> T, T>(host: &mut AnalysisHost, file_id: FileId, work: F) -> T {
    {
        let start = Instant::now();
        eprint!("from scratch:   ");
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("no change:      ");
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("trivial change: ");
        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::LOW);
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("comment change: ");
        {
            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
            text.push_str("\n/* Hello world */\n");
            let mut change = AnalysisChange::new();
            change.change_file(file_id, Some(Arc::new(text)));
            host.apply_change(change);
        }
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("const change:   ");
        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::HIGH);
        let res = work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
        res
    }
}
<commit_msg>Add Item change to the set of benches<commit_after>//! Benchmark operations like highlighting or goto definition.

use std::{env, path::Path, str::FromStr, sync::Arc, time::Instant};

use anyhow::{format_err, Result};
use ra_db::{
    salsa::{Database, Durability},
    AbsPathBuf, FileId,
};
use ra_ide::{Analysis, AnalysisChange, AnalysisHost, CompletionConfig, FilePosition, LineCol};

use crate::cli::{load_cargo::load_cargo, Verbosity};

pub enum BenchWhat {
    Highlight { path: AbsPathBuf },
    Complete(Position),
    GotoDef(Position),
}

pub struct Position {
    pub path: AbsPathBuf,
    pub line: u32,
    pub column: u32,
}

impl FromStr for Position {
    type Err = anyhow::Error;
    fn from_str(s: &str) -> Result<Self> {
        let (path_line, column) = rsplit_at_char(s, ':')?;
        let (path, line) = rsplit_at_char(path_line, ':')?;
        let path = env::current_dir().unwrap().join(path);
        let path = AbsPathBuf::assert(path);
        Ok(Position { path, line: line.parse()?, column: column.parse()? })
    }
}

fn rsplit_at_char(s: &str, c: char) -> Result<(&str, &str)> {
    let idx = s.rfind(c).ok_or_else(|| format_err!("no `{}` in {}", c, s))?;
    Ok((&s[..idx], &s[idx + 1..]))
}

pub fn analysis_bench(
    verbosity: Verbosity,
    path: &Path,
    what: BenchWhat,
    load_output_dirs: bool,
    with_proc_macro: bool,
) -> Result<()> {
    ra_prof::init();

    let start = Instant::now();
    eprint!("loading: ");
    let (mut host, vfs) = load_cargo(path, load_output_dirs, with_proc_macro)?;
    eprintln!("{:?}\n", start.elapsed());

    let file_id = {
        let path = match &what {
            BenchWhat::Highlight { path } => path,
            BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => &pos.path,
        };
        let path = path.clone().into();
        vfs.file_id(&path).ok_or_else(|| format_err!("Can't find {}", path))?
    };

    match &what {
        BenchWhat::Highlight { .. } => {
            let res = do_work(&mut host, file_id, |analysis| {
                analysis.diagnostics(file_id).unwrap();
                analysis.highlight_as_html(file_id, false).unwrap()
            });
            if verbosity.is_verbose() {
                println!("\n{}", res);
            }
        }
        BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => {
            let is_completion = matches!(what, BenchWhat::Complete(..));

            let offset = host
                .analysis()
                .file_line_index(file_id)?
                .offset(LineCol { line: pos.line - 1, col_utf16: pos.column });
            let file_position = FilePosition { file_id, offset };

            if is_completion {
                let options = CompletionConfig::default();
                let res = do_work(&mut host, file_id, |analysis| {
                    analysis.completions(&options, file_position)
                });
                if verbosity.is_verbose() {
                    println!("\n{:#?}", res);
                }
            } else {
                let res =
                    do_work(&mut host, file_id, |analysis| analysis.goto_definition(file_position));
                if verbosity.is_verbose() {
                    println!("\n{:#?}", res);
                }
            }
        }
    }
    Ok(())
}

fn do_work<F: Fn(&Analysis) -> T, T>(host: &mut AnalysisHost, file_id: FileId, work: F) -> T {
    {
        let start = Instant::now();
        eprint!("from scratch:   ");
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("no change:      ");
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("trivial change: ");
        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::LOW);
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("comment change: ");
        {
            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
            text.push_str("\n/* Hello world */\n");
            let mut change = AnalysisChange::new();
            change.change_file(file_id, Some(Arc::new(text)));
            host.apply_change(change);
        }
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("item change:    ");
        {
            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
            text.push_str("\npub fn _dummy() {}\n");
            let mut change = AnalysisChange::new();
            change.change_file(file_id, Some(Arc::new(text)));
            host.apply_change(change);
        }
        work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
    }
    {
        let start = Instant::now();
        eprint!("const change:   ");
        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::HIGH);
        let res = work(&host.analysis());
        eprintln!("{:?}", start.elapsed());
        res
    }
}
<|endoftext|>