<commit_before>use std::{
    fs,
    path::{Path, PathBuf},
};

use binary_node::{BinaryInstallation, BinaryTrait, NodeBinary};
use buildpack::{
    eyre::{self, bail, eyre},
    fs_utils::{clear_dir_all, copy_dir_all, symlink_dir, symlink_file},
    libcnb::{
        self,
        build::{BuildContext, BuildResult, BuildResultBuilder},
        data::{build_plan::BuildPlan, layer_content_metadata::LayerTypes, layer_name},
        detect::{DetectContext, DetectResult, DetectResultBuilder},
        generic::{GenericMetadata, GenericPlatform},
        layer::{Layer, LayerResult, LayerResultBuilder},
        Buildpack,
        Error::BuildpackError,
    },
    platform_is_stencila, tracing, BuildpackTrait, SYSTEM_INSTALLED,
};

pub struct NodeBuildpack;

impl BuildpackTrait for NodeBuildpack {
    fn toml() -> &'static str {
        include_str!("../buildpack.toml")
    }
}

const NVMRC: &str = ".nvmrc";
const PACKAGE_JSON: &str = "package.json";
const PACKAGE_LOCK: &str = "package-lock.json";
const TOOL_VERSIONS: &str = ".tool-versions";

impl Buildpack for NodeBuildpack {
    type Platform = GenericPlatform;
    type Metadata = GenericMetadata;
    type Error = eyre::Report;

    fn detect(&self, _context: DetectContext<Self>) -> libcnb::Result<DetectResult, Self::Error> {
        // Read `.tool-versions` for Node.js version
        let tool_versions = Self::tool_versions();

        // Read `.nvmrc` for Node.js version
        let nvmrc = fs::read_to_string(NVMRC)
            .map(|content| content.trim().to_string())
            .ok();

        // Read `package.json` for Node.js version
        let package_json = fs::read_to_string(PACKAGE_JSON)
            .ok()
            .and_then(|json| serde_json::from_str::<serde_json::Value>(&json).ok());

        // Detect `package-lock.json`
        let package_lock = PathBuf::from(PACKAGE_LOCK);

        // Fail early
        if !(tool_versions.contains_key("nodejs")
            || tool_versions.contains_key("node")
            || package_json.is_some()
            || package_lock.exists()
            || nvmrc.is_some()
            || Self::any_exist(&["main.js", "index.js"]))
        {
            return DetectResultBuilder::fail().build();
        }

        let mut requires = Vec::new();
        let mut provides = Vec::new();

        // Resolve Node.js version from `.tool-versions`, `.nvmrc`, `package.json`, or installed `node` version
        let (version, source) = if let Some(version) = tool_versions
            .get("nodejs")
            .or_else(|| tool_versions.get("node"))
        {
            (version.to_string(), TOOL_VERSIONS)
        } else if let Some(versionish) = nvmrc {
            (versionish, NVMRC)
        } else if let Some(semver) = package_json.as_ref().and_then(|package| {
            package
                .pointer("/engines/node")
                .and_then(|semver| semver.as_str().map(|semver| semver.to_string()))
        }) {
            (semver, PACKAGE_JSON)
        } else if let Some(version) = (NodeBinary {}).installed_version(None) {
            (version, SYSTEM_INSTALLED)
        } else {
            ("".to_string(), "")
        };

        // Require and provide Node.js
        let (require, provide) = Self::require_and_provide(
            format!("node {}", version).trim(),
            source,
            format!("Install Node.js {}", version).trim(),
        );
        requires.push(require);
        provides.push(provide);

        // Determine how NPM packages are to be installed
        if package_lock.exists() || package_json.is_some() {
            let (require, provide) = Self::require_and_provide(
                "npm install",
                if package_lock.exists() {
                    PACKAGE_LOCK
                } else {
                    PACKAGE_JSON
                },
                "Install NPM packages",
            );
            requires.push(require);
            provides.push(provide);
        }

        let mut build_plan = BuildPlan::new();
        build_plan.requires = requires;
        build_plan.provides = provides;
        DetectResultBuilder::pass().build_plan(build_plan).build()
    }

    fn build(&self, context: BuildContext<Self>) -> libcnb::Result<BuildResult, Self::Error> {
        for entry in &context.buildpack_plan.entries {
            let (name, args) = Self::split_entry_name(&entry.name);
            match name.as_str() {
                "node" => {
                    context.handle_layer(layer_name!("node"), NodeLayer::new(args))?;
                }
                "npm" => {
                    context.handle_layer(layer_name!("npm"), NpmLayer)?;
                }
                _ => {
                    return Err(BuildpackError(eyre!(
                        "Unhandled buildpack plan entry: {}",
                        name
                    )))
                }
            };
        }

        BuildResultBuilder::new().build()
    }
}

struct NodeLayer {
    /// A string describing the version, of Node.js to install
    ///
    /// This could be a well formed semantic version (e.g 14.0.1),
    /// a semver requirement (e.g. ^14.0), or an alias (e.g. `lts`).
    /// The `create` method aim to convert them all to a semver requirement.
    versionish: String,
}

impl NodeLayer {
    fn new(args: Vec<String>) -> Self {
        NodeLayer {
            // Join args with commas because semver requirement parser expects it to be so
            versionish: args.join(","),
        }
    }
}

impl Layer for NodeLayer {
    type Buildpack = NodeBuildpack;
    type Metadata = GenericMetadata;

    fn types(&self) -> LayerTypes {
        LayerTypes {
            build: true,
            launch: true,
            cache: false,
        }
    }

    fn create(
        &self,
        context: &BuildContext<Self::Buildpack>,
        layer_path: &Path,
    ) -> Result<LayerResult<Self::Metadata>, eyre::Report> {
        // Determine the semver requirement
        let mut requirement = if let Some(version) = self.versionish.strip_prefix('v') {
            version.to_string()
        } else if self.versionish == "lts" {
            // TODO: Determine LTS without doing a fetch, perhaps based on date
            // https://nodejs.org/en/about/releases/
            "^16".to_string()
        } else {
            self.versionish.clone()
        };
        if requirement.is_empty() {
            requirement = "*".to_string();
        }

        // Ensure a version meeting the semver is installed
        let node = NodeBinary {}.require_sync(Some(requirement), true)?;
        let version = node.version()?;

        // Symlink/copy the installation into the layer
        if platform_is_stencila(&context.platform) {
            if node.is_stencila_install() {
                tracing::info!("Linking to Node.js {} installed by Stencila", version);
                clear_dir_all(&layer_path)?;
                let source = node.grandparent()?;
                let dest = layer_path;
                symlink_dir(source.join("bin"), &dest.join("bin"))?;
                symlink_dir(source.join("lib"), &dest.join("lib"))?;
            } else {
                tracing::info!("Linking to Node.js {} installed on system", version);
                clear_dir_all(&layer_path)?;
                let source = node.parent()?;
                let dest = layer_path.join("bin");
                fs::create_dir_all(&dest)?;
                symlink_file(node.path, dest.join(node.name))?;
                symlink_file(source.join("npm"), dest.join("npm"))?;
                symlink_file(source.join("npx"), dest.join("npx"))?;
            }
        } else {
            #[allow(clippy::collapsible_else_if)]
            if node.is_stencila_install() {
                tracing::info!("Using Node.js {} installed by Stencila", version);
                clear_dir_all(&layer_path)?;
                let source = node.grandparent()?;
                let dest = layer_path;
                copy_dir_all(source, &dest)?;
            } else {
                bail!("Only able to build `node` layer if Node has been installed by Stencila")
            }
        }

        LayerResultBuilder::new(GenericMetadata::default()).build()
    }
}

struct NpmLayer;

impl Layer for NpmLayer {
    type Buildpack = NodeBuildpack;
    type Metadata = GenericMetadata;

    fn types(&self) -> LayerTypes {
        LayerTypes {
            build: true,
            launch: false,
            cache: true,
        }
    }

    fn create(
        &self,
        context: &BuildContext<Self::Buildpack>,
        layer_path: &Path,
    ) -> Result<LayerResult<Self::Metadata>, eyre::Report> {
        tracing::info!("Installing packages using NPM");

        // Get `npm` installed in `NodeLayer`
        let mut npm = BinaryInstallation {
            name: "npm".into(),
            path: layer_path
                .canonicalize()?
                .parent()
                .expect("Should have parent")
                .join("node")
                .join("bin")
                .join("npm"),
            ..Default::default()
        };

        // If Stencila is not the platform use the layer as the NPM cache
        if !platform_is_stencila(&context.platform) {
            npm.envs(&[("NPM_CONFIG_CACHE", layer_path.canonicalize()?.as_os_str())]);
        }

        // Do the install
        npm.run_sync(&["install"])?;

        LayerResultBuilder::new(GenericMetadata::default()).build()
    }
}
<commit_msg>fix(Node buildpack): Fix install of packages when using  Pack<commit_after>use std::{
    fs,
    path::{Path, PathBuf},
};

use binary_node::{BinaryInstallation, BinaryTrait, NodeBinary};
use buildpack::{
    eyre::{self, bail, eyre},
    fs_utils::{clear_dir_all, copy_dir_all, symlink_dir, symlink_file},
    libcnb::{
        self,
        build::{BuildContext, BuildResult, BuildResultBuilder},
        data::{build_plan::BuildPlan, layer_content_metadata::LayerTypes, layer_name},
        detect::{DetectContext, DetectResult, DetectResultBuilder},
        generic::{GenericMetadata, GenericPlatform},
        layer::{Layer, LayerResult, LayerResultBuilder},
        Buildpack,
        Error::BuildpackError,
    },
    platform_is_stencila, tracing, BuildpackTrait, SYSTEM_INSTALLED,
};

pub struct NodeBuildpack;

impl BuildpackTrait for NodeBuildpack {
    fn toml() -> &'static str {
        include_str!("../buildpack.toml")
    }
}

const NVMRC: &str = ".nvmrc";
const PACKAGE_JSON: &str = "package.json";
const PACKAGE_LOCK: &str = "package-lock.json";
const TOOL_VERSIONS: &str = ".tool-versions";

impl Buildpack for NodeBuildpack {
    type Platform = GenericPlatform;
    type Metadata = GenericMetadata;
    type Error = eyre::Report;

    fn detect(&self, _context: DetectContext<Self>) -> libcnb::Result<DetectResult, Self::Error> {
        // Read `.tool-versions` for Node.js version
        let tool_versions = Self::tool_versions();

        // Read `.nvmrc` for Node.js version
        let nvmrc = fs::read_to_string(NVMRC)
            .map(|content| content.trim().to_string())
            .ok();

        // Read `package.json` for Node.js version
        let package_json = fs::read_to_string(PACKAGE_JSON)
            .ok()
            .and_then(|json| serde_json::from_str::<serde_json::Value>(&json).ok());

        // Detect `package-lock.json`
        let package_lock = PathBuf::from(PACKAGE_LOCK);

        // Fail early
        if !(tool_versions.contains_key("nodejs")
            || tool_versions.contains_key("node")
            || package_json.is_some()
            || package_lock.exists()
            || nvmrc.is_some()
            || Self::any_exist(&["main.js", "index.js"]))
        {
            return DetectResultBuilder::fail().build();
        }

        let mut requires = Vec::new();
        let mut provides = Vec::new();

        // Resolve Node.js version from `.tool-versions`, `.nvmrc`, `package.json`, or installed `node` version
        let (version, source) = if let Some(version) = tool_versions
            .get("nodejs")
            .or_else(|| tool_versions.get("node"))
        {
            (version.to_string(), TOOL_VERSIONS)
        } else if let Some(versionish) = nvmrc {
            (versionish, NVMRC)
        } else if let Some(semver) = package_json.as_ref().and_then(|package| {
            package
                .pointer("/engines/node")
                .and_then(|semver| semver.as_str().map(|semver| semver.to_string()))
        }) {
            (semver, PACKAGE_JSON)
        } else if let Some(version) = (NodeBinary {}).installed_version(None) {
            (version, SYSTEM_INSTALLED)
        } else {
            ("".to_string(), "")
        };

        // Require and provide Node.js
        let (require, provide) = Self::require_and_provide(
            format!("node {}", version).trim(),
            source,
            format!("Install Node.js {}", version).trim(),
        );
        requires.push(require);
        provides.push(provide);

        // Determine how NPM packages are to be installed
        if package_lock.exists() || package_json.is_some() {
            let (require, provide) = Self::require_and_provide(
                "npm install",
                if package_lock.exists() {
                    PACKAGE_LOCK
                } else {
                    PACKAGE_JSON
                },
                "Install NPM packages",
            );
            requires.push(require);
            provides.push(provide);
        }

        let mut build_plan = BuildPlan::new();
        build_plan.requires = requires;
        build_plan.provides = provides;
        DetectResultBuilder::pass().build_plan(build_plan).build()
    }

    fn build(&self, context: BuildContext<Self>) -> libcnb::Result<BuildResult, Self::Error> {
        for entry in &context.buildpack_plan.entries {
            let (name, args) = Self::split_entry_name(&entry.name);
            match name.as_str() {
                "node" => {
                    context.handle_layer(layer_name!("node"), NodeLayer::new(args))?;
                }
                "npm" => {
                    context.handle_layer(layer_name!("npm"), NpmLayer)?;
                }
                _ => {
                    return Err(BuildpackError(eyre!(
                        "Unhandled buildpack plan entry: {}",
                        name
                    )))
                }
            };
        }

        BuildResultBuilder::new().build()
    }
}

struct NodeLayer {
    /// A string describing the version, of Node.js to install
    ///
    /// This could be a well formed semantic version (e.g 14.0.1),
    /// a semver requirement (e.g. ^14.0), or an alias (e.g. `lts`).
    /// The `create` method aim to convert them all to a semver requirement.
    versionish: String,
}

impl NodeLayer {
    fn new(args: Vec<String>) -> Self {
        NodeLayer {
            // Join args with commas because semver requirement parser expects it to be so
            versionish: args.join(","),
        }
    }
}

impl Layer for NodeLayer {
    type Buildpack = NodeBuildpack;
    type Metadata = GenericMetadata;

    fn types(&self) -> LayerTypes {
        LayerTypes {
            build: true,
            launch: true,
            cache: true,
        }
    }

    fn create(
        &self,
        context: &BuildContext<Self::Buildpack>,
        layer_path: &Path,
    ) -> Result<LayerResult<Self::Metadata>, eyre::Report> {
        // Determine the semver requirement
        let mut requirement = if let Some(version) = self.versionish.strip_prefix('v') {
            version.to_string()
        } else if self.versionish == "lts" {
            // TODO: Determine LTS without doing a fetch, perhaps based on date
            // https://nodejs.org/en/about/releases/
            "^16".to_string()
        } else {
            self.versionish.clone()
        };
        if requirement.is_empty() {
            requirement = "*".to_string();
        }

        // First check if there is already a `node` binary meeting the semver requirement
        // in the expected place
        let installed = NodeBinary {}.installed_at(
            &layer_path.join("bin").join("node"),
            Some(requirement.clone()),
        )?;

        if installed {
            tracing::info!("Node.js {} already installed", requirement);
        } else {
            tracing::info!(
                "No version of Node.js meeting requirement `{}` yet installed",
                requirement
            );

            // Ensure a version meeting the semver requirement is installed on the builder
            let node = NodeBinary {}.require_sync(Some(requirement), true)?;
            let version = node.version()?;

            // Symlink/copy the installation into the layer
            if platform_is_stencila(&context.platform) {
                if node.is_stencila_install() {
                    tracing::info!("Linking to Node.js {} installed by Stencila", version);
                    clear_dir_all(&layer_path)?;
                    let source = node.grandparent()?;

                    symlink_dir(source.join("bin"), &layer_path.join("bin"))?;
                    symlink_dir(source.join("lib"), &layer_path.join("lib"))?;
                } else {
                    tracing::info!("Linking to Node.js {} installed on system", version);
                    clear_dir_all(&layer_path)?;
                    let source = node.parent()?;

                    let bin_path = layer_path.join("bin");
                    fs::create_dir_all(&bin_path)?;
                    symlink_file(node.path, bin_path.join(node.name))?;
                    symlink_file(source.join("npm"), bin_path.join("npm"))?;
                    symlink_file(source.join("npx"), bin_path.join("npx"))?;

                    let lib_path = layer_path.join("lib");
                    fs::create_dir_all(&lib_path)?;
                    symlink_dir(
                        source.join("..").join("lib").join("node_modules"),
                        lib_path.join("node_modules"),
                    )?;
                }
            } else {
                #[allow(clippy::collapsible_else_if)]
                if node.is_stencila_install() {
                    tracing::info!("Using Node.js {} installed by Stencila", version);
                    clear_dir_all(&layer_path)?;
                    let source = node.grandparent()?;

                    copy_dir_all(source, &layer_path)?;
                } else {
                    bail!("Only able to build `node` layer if Node has been installed by Stencila")
                }
            }
        }

        LayerResultBuilder::new(GenericMetadata::default()).build()
    }
}

struct NpmLayer;

impl Layer for NpmLayer {
    type Buildpack = NodeBuildpack;
    type Metadata = GenericMetadata;

    fn types(&self) -> LayerTypes {
        LayerTypes {
            build: true,
            launch: false,
            cache: true,
        }
    }

    fn create(
        &self,
        context: &BuildContext<Self::Buildpack>,
        layer_path: &Path,
    ) -> Result<LayerResult<Self::Metadata>, eyre::Report> {
        tracing::info!("Installing packages using NPM");

        // Use `node` from the previous layer
        let node_layer = layer_path
            .canonicalize()?
            .parent()
            .expect("Should have parent")
            .join("node");
        let mut node = BinaryInstallation {
            name: "node".into(),
            path: node_layer.join("bin").join("node"),
            ..Default::default()
        };

        // Use the `npm-cli.js` script from the previous layer
        // This is done because the `npm` script in `bin` has a
        // `/usr/bin/env node` shebang and so needs the right `node` in $PATH.
        // Even if that is done, there are issues with node `require`
        // module resolution when doing so. But this works...
        let npm = node_layer
            .join("lib")
            .join("node_modules")
            .join("npm")
            .join("bin")
            .join("npm-cli.js")
            .display()
            .to_string();

        // If Stencila is not the platform use the layer as the NPM cache
        if !platform_is_stencila(&context.platform) {
            node.envs(&[("NPM_CONFIG_CACHE", layer_path.canonicalize()?)]);
        }

        // Do the install
        node.run_sync(&[&npm, "install"])?;

        LayerResultBuilder::new(GenericMetadata::default()).build()
    }
}
<|endoftext|>


<commit_before>//! The global data and participant for garbage collection.
//!
//! # Registration
//!
//! In order to track all participants in one place, we need some form of participant
//! registration. When a participant is created, it is registered to a global lock-free
//! singly-linked list of registries; and when a participant is leaving, it is unregistered from the
//! list.
//!
//! # Pinning
//!
//! Every participant contains an integer that tells whether the participant is pinned and if so,
//! what was the global epoch at the time it was pinned. Participants also hold a pin counter that
//! aids in periodic global epoch advancement.
//!
//! When a participant is pinned, a `Guard` is returned as a witness that the participant is pinned.
//! Guards are necessary for performing atomic operations, and for freeing/dropping locations.

use core::cell::{Cell, UnsafeCell};
use core::mem::{self, ManuallyDrop};
use core::num::Wrapping;
use core::ptr;
use core::sync::atomic;
use core::sync::atomic::Ordering;
use alloc::boxed::Box;
use alloc::arc::Arc;

use crossbeam_utils::cache_padded::CachePadded;

use atomic::Owned;
use collector::Handle;
use epoch::{AtomicEpoch, Epoch};
use guard::{unprotected, Guard};
use garbage::{Bag, Garbage};
use sync::list::{List, Entry, IterError, IsElement};
use sync::queue::Queue;

/// The global data for a garbage collector.
pub struct Global {
    /// The intrusive linked list of `Local`s.
    locals: List<Local>,

    /// The global queue of bags of deferred functions.
    queue: Queue<(Epoch, Bag)>,

    /// The global epoch.
    pub(crate) epoch: CachePadded<AtomicEpoch>,
}

impl Global {
    /// Number of bags to destroy.
    const COLLECT_STEPS: usize = 8;

    /// Creates a new global data for garbage collection.
    #[inline]
    pub fn new() -> Self {
        Self {
            locals: List::new(),
            queue: Queue::new(),
            epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())),
        }
    }

    /// Pushes the bag into the global queue and replaces the bag with a new empty bag.
    pub fn push_bag(&self, bag: &mut Bag, guard: &Guard) {
        let epoch = self.epoch.load(Ordering::Relaxed);
        let bag = mem::replace(bag, Bag::new());

        atomic::fence(Ordering::SeqCst);

        self.queue.push((epoch, bag), guard);
    }

    /// Collects several bags from the global queue and executes deferred functions in them.
    ///
    /// Note: This may itself produce garbage and in turn allocate new bags.
    ///
    /// `pin()` rarely calls `collect()`, so we want the compiler to place that call on a cold
    /// path. In other words, we want the compiler to optimize branching for the case when
    /// `collect()` is not called.
    #[cold]
    pub fn collect(&self, guard: &Guard) {
        let global_epoch = self.try_advance(guard);

        let condition = |item: &(Epoch, Bag)| {
            // A pinned participant can witness at most one epoch advancement. Therefore, any bag
            // that is within one epoch of the current one cannot be destroyed yet.
            global_epoch.wrapping_sub(item.0) >= 2
        };

        for _ in 0..Self::COLLECT_STEPS {
            match self.queue.try_pop_if(&condition, guard) {
                None => break,
                Some(bag) => drop(bag),
            }
        }
    }

    /// Attempts to advance the global epoch.
    ///
    /// The global epoch can advance only if all currently pinned participants have been pinned in
    /// the current epoch.
    ///
    /// Returns the current global epoch.
    ///
    /// `try_advance()` is annotated `#[cold]` because it is rarely called.
    #[cold]
    pub fn try_advance(&self, guard: &Guard) -> Epoch {
        let global_epoch = self.epoch.load(Ordering::Relaxed);
        atomic::fence(Ordering::SeqCst);

        // TODO(stjepang): `Local`s are stored in a linked list because linked lists are fairly
        // easy to implement in a lock-free manner. However, traversal can be slow due to cache
        // misses and data dependencies. We should experiment with other data structures as well.
        for local in self.locals.iter(&guard) {
            match local {
                Err(IterError::Stalled) => {
                    // A concurrent thread stalled this iteration. That thread might also try to
                    // advance the epoch, in which case we leave the job to it. Otherwise, the
                    // epoch will not be advanced.
                    return global_epoch;
                }
                Ok(local) => {
                    let local_epoch = if cfg!(feature = "sanitize") {
                        // HACK(stjepang): This is necessary because thread sanitizer doesn't
                        // understand fences.
                        local.epoch.load(Ordering::Acquire)
                    } else {
                        local.epoch.load(Ordering::Relaxed)
                    };

                    // If the participant was pinned in a different epoch, we cannot advance the
                    // global epoch just yet.
                    if local_epoch.is_pinned() && local_epoch.unpinned() != global_epoch {
                        return global_epoch;
                    }
                }
            }
        }
        atomic::fence(Ordering::Acquire);

        // All pinned participants were pinned in the current global epoch.
        // Now let's advance the global epoch...
        //
        // Note that if another thread already advanced it before us, this store will simply
        // overwrite the global epoch with the same value. This is true because `try_advance` was
        // called from a thread that was pinned in `global_epoch`, and the global epoch cannot be
        // advanced two steps ahead of it.
        let new_epoch = global_epoch.successor();
        self.epoch.store(new_epoch, Ordering::Release);
        new_epoch
    }
}

/// Participant for garbage collection.
pub struct Local {
    /// A node in the intrusive linked list of `Local`s.
    entry: Entry,

    /// The local epoch.
    epoch: AtomicEpoch,

    /// A reference to the global data.
    ///
    /// When all guards and handles get dropped, this reference is destroyed.
    global: UnsafeCell<ManuallyDrop<Arc<Global>>>,

    /// The local bag of deferred functions.
    pub(crate) bag: UnsafeCell<Bag>,

    /// The number of guards keeping this participant pinned.
    guard_count: Cell<usize>,

    /// The number of active handles.
    handle_count: Cell<usize>,

    /// Total number of pinnings performed.
    ///
    /// This is just an auxilliary counter that sometimes kicks off collection.
    pin_count: Cell<Wrapping<usize>>,
}

unsafe impl Sync for Local {}

impl Local {
    /// Number of pinnings after which a participant will execute some deferred functions from the
    /// global queue.
    const PINNINGS_BETWEEN_COLLECT: usize = 128;

    /// Registers a new `Local` in the provided `Global`.
    pub fn register(global: Arc<Global>) -> Handle {
        unsafe {
            // Since we dereference no pointers in this block, it is safe to use `unprotected`.

            let local = Owned::new(Local {
                entry: Entry::default(),
                epoch: AtomicEpoch::new(Epoch::starting()),
                global: UnsafeCell::new(ManuallyDrop::new(global.clone())),
                bag: UnsafeCell::new(Bag::new()),
                guard_count: Cell::new(0),
                handle_count: Cell::new(1),
                pin_count: Cell::new(Wrapping(0)),
            }).into_shared(&unprotected());
            global.locals.insert(local, &unprotected());
            Handle { local: local.as_raw() }
        }
    }

    /// Returns a reference to the `Global` in which this `Local` resides.
    #[inline]
    pub fn global(&self) -> &Global {
        unsafe { &*self.global.get() }
    }

    /// Returns `true` if the current participant is pinned.
    #[inline]
    pub fn is_pinned(&self) -> bool {
        self.guard_count.get() > 0
    }

    pub fn defer(&self, mut garbage: Garbage, guard: &Guard) {
        let bag = unsafe { &mut *self.bag.get() };

        while let Err(g) = bag.try_push(garbage) {
            self.global().push_bag(bag, guard);
            garbage = g;
        }
    }

    pub fn flush(&self, guard: &Guard) {
        let bag = unsafe { &mut *self.bag.get() };

        if !bag.is_empty() {
            self.global().push_bag(bag, guard);
        }

        self.global().collect(guard);
    }

    /// Pins the `Local`.
    #[inline]
    pub fn pin(&self) -> Guard {
        let guard = Guard { local: self };

        let guard_count = self.guard_count.get();
        self.guard_count.set(guard_count.checked_add(1).unwrap());

        if guard_count == 0 {
            let global_epoch = self.global().epoch.load(Ordering::Relaxed);
            let new_epoch = global_epoch.pinned();

            // Now we must store `new_epoch` into `self.epoch` and execute a `SeqCst` fence.
            // The fence makes sure that any future loads from `Atomic`s will not happen before
            // this store.
            if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
                // HACK(stjepang): On x86 architectures there are two different ways of executing
                // a `SeqCst` fence.
                //
                // 1. `atomic::fence(SeqCst)`, which compiles into a `mfence` instruction.
                // 2. `_.compare_and_swap(_, _, SeqCst)`, which compiles into a `lock cmpxchg`
                //    instruction.
                //
                // Both instructions have the effect of a full barrier, but benchmarks have shown
                // that the second one makes pinning faster in this particular case.
                let current = Epoch::starting();
                let previous = self.epoch.compare_and_swap(current, new_epoch, Ordering::SeqCst);
                debug_assert_eq!(current, previous, "participant was expected to be unpinned");
            } else {
                self.epoch.store(new_epoch, Ordering::Relaxed);
                atomic::fence(Ordering::SeqCst);
            }

            // Increment the pin counter.
            let count = self.pin_count.get();
            self.pin_count.set(count + Wrapping(1));

            // After every `PINNINGS_BETWEEN_COLLECT` try advancing the epoch and collecting
            // some garbage.
            if count.0 % Self::PINNINGS_BETWEEN_COLLECT == 0 {
                self.global().collect(&guard);
            }
        }

        guard
    }

    /// Unpins the `Local`.
    #[inline]
    pub fn unpin(&self) {
        let guard_count = self.guard_count.get();
        self.guard_count.set(guard_count - 1);

        if guard_count == 1 {
            self.epoch.store(Epoch::starting(), Ordering::Release);

            if self.handle_count.get() == 0 {
                self.finalize();
            }
        }
    }

    /// Unpins and then pins the `Local`.
    #[inline]
    pub fn repin(&self) {
        let guard_count = self.guard_count.get();

        // Update the local epoch only if there's only one guard.
        if guard_count == 1 {
            let epoch = self.epoch.load(Ordering::Relaxed);
            let global_epoch = self.global().epoch.load(Ordering::Relaxed);

            // Update the local epoch only if the global epoch is greater than the local epoch.
            if epoch != global_epoch {
                // We store the new epoch with `Release` because we need to ensure any memory
                // accesses from the previous epoch do not leak into the new one.
                self.epoch.store(global_epoch, Ordering::Release);

                // However, we don't need a following `SeqCst` fence, because it is safe for memory
                // accesses from the new epoch to be executed before updating the local epoch.  At
                // worse, other threads will see the new epoch late and delay GC slightly.
            }
        }
    }

    /// Increments the handle count.
    #[inline]
    pub fn acquire_handle(&self) {
        let handle_count = self.handle_count.get();
        debug_assert!(handle_count >= 1);
        self.handle_count.set(handle_count + 1);
    }

    /// Decrements the handle count.
    #[inline]
    pub fn release_handle(&self) {
        let guard_count = self.guard_count.get();
        let handle_count = self.handle_count.get();
        debug_assert!(handle_count >= 1);
        self.handle_count.set(handle_count - 1);

        if guard_count == 0 && handle_count == 1 {
            self.finalize();
        }
    }

    /// Removes the `Local` from the global linked list.
    #[cold]
    fn finalize(&self) {
        debug_assert_eq!(self.guard_count.get(), 0);
        debug_assert_eq!(self.handle_count.get(), 0);

        unsafe {
            // Take the reference to the `Global` out of this `Local`.
            let global: Arc<Global> = ptr::read(&**self.global.get());

            // Move the local bag into the global queue.
            global.push_bag(&mut *self.bag.get(), &unprotected());

            // Mark this node in the linked list as deleted.
            self.entry.delete(&unprotected());

            // Finally, drop the reference to the global.  Note that this might be the last
            // reference to the `Global`. If so, the global data will be destroyed and all deferred
            // functions in its queue will be executed.
            drop(global);
        }
    }
}

impl IsElement<Local> for Local {
    fn entry_of(local: &Local) -> &Entry {
        let entry_ptr = (local as *const Local as usize + offset_of!(Local, entry)) as *const Entry;
        unsafe { &*entry_ptr }
    }

    unsafe fn element_of(entry: &Entry) -> &Local {
        // offset_of! macro uses unsafe, but it's unnecessary in this context.
        #[allow(unused_unsafe)]
        let local_ptr = (entry as *const Entry as usize - offset_of!(Local, entry)) as *const Local;
        &*local_ptr
    }

    unsafe fn finalize(entry: &Entry) {
        let local = Self::element_of(entry);
        drop(Box::from_raw(local as *const Local as *mut Local));
    }
}
<commit_msg>Fix an ordering bug in Global::push_bag() (#53)<commit_after>//! The global data and participant for garbage collection.
//!
//! # Registration
//!
//! In order to track all participants in one place, we need some form of participant
//! registration. When a participant is created, it is registered to a global lock-free
//! singly-linked list of registries; and when a participant is leaving, it is unregistered from the
//! list.
//!
//! # Pinning
//!
//! Every participant contains an integer that tells whether the participant is pinned and if so,
//! what was the global epoch at the time it was pinned. Participants also hold a pin counter that
//! aids in periodic global epoch advancement.
//!
//! When a participant is pinned, a `Guard` is returned as a witness that the participant is pinned.
//! Guards are necessary for performing atomic operations, and for freeing/dropping locations.

use core::cell::{Cell, UnsafeCell};
use core::mem::{self, ManuallyDrop};
use core::num::Wrapping;
use core::ptr;
use core::sync::atomic;
use core::sync::atomic::Ordering;
use alloc::boxed::Box;
use alloc::arc::Arc;

use crossbeam_utils::cache_padded::CachePadded;

use atomic::Owned;
use collector::Handle;
use epoch::{AtomicEpoch, Epoch};
use guard::{unprotected, Guard};
use garbage::{Bag, Garbage};
use sync::list::{List, Entry, IterError, IsElement};
use sync::queue::Queue;

/// The global data for a garbage collector.
pub struct Global {
    /// The intrusive linked list of `Local`s.
    locals: List<Local>,

    /// The global queue of bags of deferred functions.
    queue: Queue<(Epoch, Bag)>,

    /// The global epoch.
    pub(crate) epoch: CachePadded<AtomicEpoch>,
}

impl Global {
    /// Number of bags to destroy.
    const COLLECT_STEPS: usize = 8;

    /// Creates a new global data for garbage collection.
    #[inline]
    pub fn new() -> Self {
        Self {
            locals: List::new(),
            queue: Queue::new(),
            epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())),
        }
    }

    /// Pushes the bag into the global queue and replaces the bag with a new empty bag.
    pub fn push_bag(&self, bag: &mut Bag, guard: &Guard) {
        let bag = mem::replace(bag, Bag::new());

        atomic::fence(Ordering::SeqCst);

        let epoch = self.epoch.load(Ordering::Relaxed);
        self.queue.push((epoch, bag), guard);
    }

    /// Collects several bags from the global queue and executes deferred functions in them.
    ///
    /// Note: This may itself produce garbage and in turn allocate new bags.
    ///
    /// `pin()` rarely calls `collect()`, so we want the compiler to place that call on a cold
    /// path. In other words, we want the compiler to optimize branching for the case when
    /// `collect()` is not called.
    #[cold]
    pub fn collect(&self, guard: &Guard) {
        let global_epoch = self.try_advance(guard);

        let condition = |item: &(Epoch, Bag)| {
            // A pinned participant can witness at most one epoch advancement. Therefore, any bag
            // that is within one epoch of the current one cannot be destroyed yet.
            global_epoch.wrapping_sub(item.0) >= 2
        };

        for _ in 0..Self::COLLECT_STEPS {
            match self.queue.try_pop_if(&condition, guard) {
                None => break,
                Some(bag) => drop(bag),
            }
        }
    }

    /// Attempts to advance the global epoch.
    ///
    /// The global epoch can advance only if all currently pinned participants have been pinned in
    /// the current epoch.
    ///
    /// Returns the current global epoch.
    ///
    /// `try_advance()` is annotated `#[cold]` because it is rarely called.
    #[cold]
    pub fn try_advance(&self, guard: &Guard) -> Epoch {
        let global_epoch = self.epoch.load(Ordering::Relaxed);
        atomic::fence(Ordering::SeqCst);

        // TODO(stjepang): `Local`s are stored in a linked list because linked lists are fairly
        // easy to implement in a lock-free manner. However, traversal can be slow due to cache
        // misses and data dependencies. We should experiment with other data structures as well.
        for local in self.locals.iter(&guard) {
            match local {
                Err(IterError::Stalled) => {
                    // A concurrent thread stalled this iteration. That thread might also try to
                    // advance the epoch, in which case we leave the job to it. Otherwise, the
                    // epoch will not be advanced.
                    return global_epoch;
                }
                Ok(local) => {
                    let local_epoch = if cfg!(feature = "sanitize") {
                        // HACK(stjepang): This is necessary because thread sanitizer doesn't
                        // understand fences.
                        local.epoch.load(Ordering::Acquire)
                    } else {
                        local.epoch.load(Ordering::Relaxed)
                    };

                    // If the participant was pinned in a different epoch, we cannot advance the
                    // global epoch just yet.
                    if local_epoch.is_pinned() && local_epoch.unpinned() != global_epoch {
                        return global_epoch;
                    }
                }
            }
        }
        atomic::fence(Ordering::Acquire);

        // All pinned participants were pinned in the current global epoch.
        // Now let's advance the global epoch...
        //
        // Note that if another thread already advanced it before us, this store will simply
        // overwrite the global epoch with the same value. This is true because `try_advance` was
        // called from a thread that was pinned in `global_epoch`, and the global epoch cannot be
        // advanced two steps ahead of it.
        let new_epoch = global_epoch.successor();
        self.epoch.store(new_epoch, Ordering::Release);
        new_epoch
    }
}

/// Participant for garbage collection.
pub struct Local {
    /// A node in the intrusive linked list of `Local`s.
    entry: Entry,

    /// The local epoch.
    epoch: AtomicEpoch,

    /// A reference to the global data.
    ///
    /// When all guards and handles get dropped, this reference is destroyed.
    global: UnsafeCell<ManuallyDrop<Arc<Global>>>,

    /// The local bag of deferred functions.
    pub(crate) bag: UnsafeCell<Bag>,

    /// The number of guards keeping this participant pinned.
    guard_count: Cell<usize>,

    /// The number of active handles.
    handle_count: Cell<usize>,

    /// Total number of pinnings performed.
    ///
    /// This is just an auxilliary counter that sometimes kicks off collection.
    pin_count: Cell<Wrapping<usize>>,
}

unsafe impl Sync for Local {}

impl Local {
    /// Number of pinnings after which a participant will execute some deferred functions from the
    /// global queue.
    const PINNINGS_BETWEEN_COLLECT: usize = 128;

    /// Registers a new `Local` in the provided `Global`.
    pub fn register(global: Arc<Global>) -> Handle {
        unsafe {
            // Since we dereference no pointers in this block, it is safe to use `unprotected`.

            let local = Owned::new(Local {
                entry: Entry::default(),
                epoch: AtomicEpoch::new(Epoch::starting()),
                global: UnsafeCell::new(ManuallyDrop::new(global.clone())),
                bag: UnsafeCell::new(Bag::new()),
                guard_count: Cell::new(0),
                handle_count: Cell::new(1),
                pin_count: Cell::new(Wrapping(0)),
            }).into_shared(&unprotected());
            global.locals.insert(local, &unprotected());
            Handle { local: local.as_raw() }
        }
    }

    /// Returns a reference to the `Global` in which this `Local` resides.
    #[inline]
    pub fn global(&self) -> &Global {
        unsafe { &*self.global.get() }
    }

    /// Returns `true` if the current participant is pinned.
    #[inline]
    pub fn is_pinned(&self) -> bool {
        self.guard_count.get() > 0
    }

    pub fn defer(&self, mut garbage: Garbage, guard: &Guard) {
        let bag = unsafe { &mut *self.bag.get() };

        while let Err(g) = bag.try_push(garbage) {
            self.global().push_bag(bag, guard);
            garbage = g;
        }
    }

    pub fn flush(&self, guard: &Guard) {
        let bag = unsafe { &mut *self.bag.get() };

        if !bag.is_empty() {
            self.global().push_bag(bag, guard);
        }

        self.global().collect(guard);
    }

    /// Pins the `Local`.
    #[inline]
    pub fn pin(&self) -> Guard {
        let guard = Guard { local: self };

        let guard_count = self.guard_count.get();
        self.guard_count.set(guard_count.checked_add(1).unwrap());

        if guard_count == 0 {
            let global_epoch = self.global().epoch.load(Ordering::Relaxed);
            let new_epoch = global_epoch.pinned();

            // Now we must store `new_epoch` into `self.epoch` and execute a `SeqCst` fence.
            // The fence makes sure that any future loads from `Atomic`s will not happen before
            // this store.
            if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
                // HACK(stjepang): On x86 architectures there are two different ways of executing
                // a `SeqCst` fence.
                //
                // 1. `atomic::fence(SeqCst)`, which compiles into a `mfence` instruction.
                // 2. `_.compare_and_swap(_, _, SeqCst)`, which compiles into a `lock cmpxchg`
                //    instruction.
                //
                // Both instructions have the effect of a full barrier, but benchmarks have shown
                // that the second one makes pinning faster in this particular case.
                let current = Epoch::starting();
                let previous = self.epoch.compare_and_swap(current, new_epoch, Ordering::SeqCst);
                debug_assert_eq!(current, previous, "participant was expected to be unpinned");
            } else {
                self.epoch.store(new_epoch, Ordering::Relaxed);
                atomic::fence(Ordering::SeqCst);
            }

            // Increment the pin counter.
            let count = self.pin_count.get();
            self.pin_count.set(count + Wrapping(1));

            // After every `PINNINGS_BETWEEN_COLLECT` try advancing the epoch and collecting
            // some garbage.
            if count.0 % Self::PINNINGS_BETWEEN_COLLECT == 0 {
                self.global().collect(&guard);
            }
        }

        guard
    }

    /// Unpins the `Local`.
    #[inline]
    pub fn unpin(&self) {
        let guard_count = self.guard_count.get();
        self.guard_count.set(guard_count - 1);

        if guard_count == 1 {
            self.epoch.store(Epoch::starting(), Ordering::Release);

            if self.handle_count.get() == 0 {
                self.finalize();
            }
        }
    }

    /// Unpins and then pins the `Local`.
    #[inline]
    pub fn repin(&self) {
        let guard_count = self.guard_count.get();

        // Update the local epoch only if there's only one guard.
        if guard_count == 1 {
            let epoch = self.epoch.load(Ordering::Relaxed);
            let global_epoch = self.global().epoch.load(Ordering::Relaxed);

            // Update the local epoch only if the global epoch is greater than the local epoch.
            if epoch != global_epoch {
                // We store the new epoch with `Release` because we need to ensure any memory
                // accesses from the previous epoch do not leak into the new one.
                self.epoch.store(global_epoch, Ordering::Release);

                // However, we don't need a following `SeqCst` fence, because it is safe for memory
                // accesses from the new epoch to be executed before updating the local epoch.  At
                // worse, other threads will see the new epoch late and delay GC slightly.
            }
        }
    }

    /// Increments the handle count.
    #[inline]
    pub fn acquire_handle(&self) {
        let handle_count = self.handle_count.get();
        debug_assert!(handle_count >= 1);
        self.handle_count.set(handle_count + 1);
    }

    /// Decrements the handle count.
    #[inline]
    pub fn release_handle(&self) {
        let guard_count = self.guard_count.get();
        let handle_count = self.handle_count.get();
        debug_assert!(handle_count >= 1);
        self.handle_count.set(handle_count - 1);

        if guard_count == 0 && handle_count == 1 {
            self.finalize();
        }
    }

    /// Removes the `Local` from the global linked list.
    #[cold]
    fn finalize(&self) {
        debug_assert_eq!(self.guard_count.get(), 0);
        debug_assert_eq!(self.handle_count.get(), 0);

        unsafe {
            // Take the reference to the `Global` out of this `Local`.
            let global: Arc<Global> = ptr::read(&**self.global.get());

            // Move the local bag into the global queue.
            global.push_bag(&mut *self.bag.get(), &unprotected());

            // Mark this node in the linked list as deleted.
            self.entry.delete(&unprotected());

            // Finally, drop the reference to the global.  Note that this might be the last
            // reference to the `Global`. If so, the global data will be destroyed and all deferred
            // functions in its queue will be executed.
            drop(global);
        }
    }
}

impl IsElement<Local> for Local {
    fn entry_of(local: &Local) -> &Entry {
        let entry_ptr = (local as *const Local as usize + offset_of!(Local, entry)) as *const Entry;
        unsafe { &*entry_ptr }
    }

    unsafe fn element_of(entry: &Entry) -> &Local {
        // offset_of! macro uses unsafe, but it's unnecessary in this context.
        #[allow(unused_unsafe)]
        let local_ptr = (entry as *const Entry as usize - offset_of!(Local, entry)) as *const Local;
        &*local_ptr
    }

    unsafe fn finalize(entry: &Entry) {
        let local = Self::element_of(entry);
        drop(Box::from_raw(local as *const Local as *mut Local));
    }
}
<|endoftext|>


<commit_before>//! Basic types to build the parsers

use self::IResult::*;
use self::Needed::*;

#[cfg(feature = "core")]
use std::prelude::v1::*;

#[cfg(feature = "verbose-errors")]
use verbose_errors::Err;

#[cfg(not(feature = "verbose-errors"))]
use simple_errors::Err;

/// Contains information on needed data if a parser returned `Incomplete`
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum Needed {
  /// needs more data, but we do not know how much
  Unknown,
  /// contains the required data size
  Size(usize)
}

impl Needed {
  pub fn is_known(&self) -> bool {
    *self != Unknown
  }

  /// Maps a `Needed` to `Needed` by appling a function to a contained `Size` value.
  #[inline]
  pub fn map<F: FnOnce(usize) -> usize>(self, f: F) -> Needed {
    match self {
      Unknown => Unknown,
      Size(n) => Size(f(n)),
    }
  }
}

#[cfg(feature = "verbose-errors")]
/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IResult<I,O,E=u32> {
   /// indicates a correct parsing, the first field containing the rest of the unparsed data, the second field contains the parsed data
  Done(I,O),
  /// contains a Err, an enum that can indicate an error code, a position in the input, and a pointer to another error, making a list of errors in the parsing tree
  Error(Err<I,E>),
  /// Incomplete contains a Needed, an enum than can represent a known quantity of input data, or unknown
  Incomplete(Needed)
}

#[cfg(not(feature = "verbose-errors"))]
/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IResult<I,O,E=u32> {
   /// indicates a correct parsing, the first field containing the rest of the unparsed data, the second field contains the parsed data
  Done(I,O),
  /// contains a Err, an enum that can indicate an error code, a position in the input, and a pointer to another error, making a list of errors in the parsing tree
  Error(Err<E>),
  /// Incomplete contains a Needed, an enum than can represent a known quantity of input data, or unknown
  Incomplete(Needed)
}

#[cfg(feature = "verbose-errors")]
/// This is the same as IResult, but without Done
///
/// This is used as the Error type when converting to std::result::Result
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IError<I,E=u32> {
  Error(Err<I,E>),
  Incomplete(Needed)
}

#[cfg(not(feature = "verbose-errors"))]
/// This is the same as IResult, but without Done
///
/// This is used as the Error type when converting to std::result::Result
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IError<I,E=u32> {
  Error(Err<E>),
  Incomplete(Needed)
}

impl<I,O,E> IResult<I,O,E> {
  pub fn is_done(&self) -> bool {
    match *self {
      Done(_,_) => true,
      _         => false
    }
  }

  pub fn is_err(&self) -> bool {
    match *self {
      Error(_) => true,
      _        => false
    }
  }

  pub fn is_incomplete(&self) -> bool {
    match *self {
      Incomplete(_) => true,
      _             => false
    }
  }

  /// Maps a `IResult<I, O, E>` to `IResult<I, N, E>` by appling a function
  /// to a contained `Done` value, leaving `Error` and `Incomplete` value
  /// untouched.
  #[inline]
  pub fn map<N, F: FnOnce(O) -> N>(self, f: F) -> IResult<I, N, E> {
    match self {
      Done(i, o)    => Done(i, f(o)),
      Error(e)      => Error(e),
      Incomplete(n) => Incomplete(n),
    }
  }

  /// Maps a `IResult<I, O, E>` to `IResult<I, O, E>` by appling a function
  /// to a contained `Incomplete` value, leaving `Done` and `Error` value
  /// untouched.
  #[inline]
  pub fn map_inc<F>(self, f: F) -> IResult<I, O, E>
   where F: FnOnce(Needed) -> Needed {
    match self {
      Error(e)      => Error(e),
      Incomplete(n) => Incomplete(f(n)),
      Done(i, o)    => Done(i, o),
    }
  }

  /// Unwrap the contained `Done(I, O)` value, or panic if the `IResult` is not
  /// `Done`.
  pub fn unwrap(self) -> (I, O) {
    match self {
      Done(i, o)    => (i, o),
      Incomplete(_) => panic!("unwrap() called on an IResult that is Incomplete"),
      Error(_)      => panic!("unwrap() called on an IResult that is Error")
    }
  }

  /// Unwrap the contained `Incomplete(n)` value, or panic if the `IResult` is not
  /// `Incomplete`.
  pub fn unwrap_inc(self) -> Needed {
    match self {
      Incomplete(n) => n,
      Done(_, _)    => panic!("unwrap_inc() called on an IResult that is Done"),
      Error(_)      => panic!("unwrap_inc() called on an IResult that is Error")
    }
  }

  /// Convert the IResult to a std::result::Result
  pub fn to_result(self) -> Result<O, IError<I,E>> {
    match self {
      Done(_, o)    => Ok(o),
      Incomplete(n) => Err(IError::Incomplete(n)),
      Error(e)      => Err(IError::Error(e))
    }
  }
}

pub trait GetInput<I> {
  fn remaining_input(&self) -> Option<I>;
}

pub trait GetOutput<O> {
  fn output(&self) -> Option<O>;
}

impl<'a,I,O,E> GetInput<&'a[I]> for IResult<&'a[I],O,E> {
  fn remaining_input(&self) -> Option<&'a[I]> {
    match *self {
      Done(ref i,_) => Some(*i),
      _             => None
    }
  }
}

impl<O,E> GetInput<()> for IResult<(),O,E> {
  fn remaining_input(&self) -> Option<()> {
    match *self {
      Done((),_) => Some(()),
      _          => None
    }
  }
}

impl<'a,O,E> GetInput<&'a str> for IResult<&'a str,O,E> {
  fn remaining_input(&self) -> Option<&'a str> {
    match *self {
      Done(ref i,_) => Some(*i),
      _          => None
    }
  }
}

impl<'a,I,O,E> GetOutput<&'a[O]> for IResult<I,&'a[O],E> {
  fn output(&self) -> Option<&'a[O]> {
    match *self {
      Done(_, ref o) => Some(*o),
      _              => None
    }
  }
}

impl<I,E> GetOutput<()> for IResult<I,(),E> {
  fn output(&self) -> Option<()> {
    match *self {
      Done(_,()) => Some(()),
      _          => None
    }
  }
}

impl<'a,I,E> GetOutput<&'a str> for IResult<I,&'a str,E> {
  fn output(&self) -> Option<&'a str> {
    match *self {
      Done(_,ref o) => Some(*o),
      _          => None
    }
  }
}

#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::any::Any;
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::{error,fmt};
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::fmt::Debug;
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
impl<P:Debug+Any,E:Debug+Any> error::Error for Err<P,E> {
  fn description(&self) -> &str {
    let kind = match *self {
      Err::Code(ref e) | Err::Node(ref e, _) | Err::Position(ref e, _) | Err::NodePosition(ref e, _, _) => e
    };
    kind.description()
  }
}

#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
impl<P:fmt::Debug,E:fmt::Debug> fmt::Display for Err<P,E> {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    match *self {
      Err::Code(ref e) | Err::Node(ref e, _) => {
        write!(f, "{:?}", e)
      },
      Err::Position(ref e, ref p) | Err::NodePosition(ref e, ref p, _) => {
        write!(f, "{:?}:{:?}", p, e)
      }
    }
  }
}

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_code(
  ($code:expr) => ($crate::Err::Code($code));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_code(
  ($code:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_node(
  ($code:expr, $next:expr) => ($crate::Err::Node($code, ::std::boxed::Box::new($next)));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_node(
  ($code:expr, $next:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_position(
  ($code:expr, $input:expr) => ($crate::Err::Position($code, $input));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_position(
  ($code:expr, $input:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_node_position(
  ($code:expr, $input:expr, $next:expr) => ($crate::Err::NodePosition($code, $input, ::std::boxed::Box::new($next)));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_node_position(
  ($code:expr, $input: expr, $next:expr) => ($code);
);

#[cfg(test)]
mod tests {
  use super::*;
  use util::ErrorKind;

  const REST: [u8; 0] = [];
  const DONE: IResult<&'static [u8], u32> = IResult::Done(&REST, 5);
  const ERROR: IResult<&'static [u8], u32> = IResult::Error(error_code!(ErrorKind::Tag));
  const INCOMPLETE: IResult<&'static [u8], u32> = IResult::Incomplete(Needed::Unknown);

  #[test]
  fn needed_map() {
    let unknown = Needed::Unknown;
    let size = Needed::Size(5);

    assert_eq!(size.map(|x| x * 2), Needed::Size(10));
    assert_eq!(unknown.map(|x| x * 2), Needed::Unknown);
  }

  #[test]
  fn iresult_map() {
    assert_eq!(DONE.map(|x| x * 2), IResult::Done(&b""[..], 10));
    assert_eq!(ERROR.map(|x| x * 2), IResult::Error(error_code!(ErrorKind::Tag)));
    assert_eq!(INCOMPLETE.map(|x| x * 2), IResult::Incomplete(Needed::Unknown));
  }

  #[test]
  fn iresult_map_inc() {
    let inc_unknown: IResult<&[u8], u32> = IResult::Incomplete(Needed::Unknown);
    let inc_size: IResult<&[u8], u32> = IResult::Incomplete(Needed::Size(5));

    assert_eq!(DONE.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Done(&b""[..], 5));
    assert_eq!(ERROR.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Error(error_code!(ErrorKind::Tag)));
    assert_eq!(inc_unknown.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Incomplete(Needed::Unknown));
    assert_eq!(inc_size.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Incomplete(Needed::Size(6)));
  }

  #[test]
  fn iresult_map_err() {
    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
    struct Error(u32);

    let error_kind = error_code!(ErrorKind::Custom(Error(5)));

    assert_eq!(DONE.map_err(|_| error_kind.clone()), IResult::Done(&b""[..], 5));
    assert_eq!(ERROR.map_err(|x| {println!("err: {:?}", x); error_kind.clone()}), IResult::Error(error_kind.clone()));
    assert_eq!(INCOMPLETE.map_err(|x| {println!("err: {:?}", x); error_kind.clone()}), IResult::Incomplete(Needed::Unknown));
  }

  #[test]
  fn iresult_unwrap_on_done() {
    assert_eq!(DONE.unwrap(), (&b""[..], 5));
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_on_err() {
    ERROR.unwrap();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_on_inc() {
    INCOMPLETE.unwrap();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_err_on_done() {
    DONE.unwrap_err();
  }

  #[test]
  fn iresult_unwrap_err_on_err() {
    assert_eq!(ERROR.unwrap_err(), error_code!(ErrorKind::Tag));
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_err_on_inc() {
    INCOMPLETE.unwrap_err();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_inc_on_done() {
    DONE.unwrap_inc();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_inc_on_err() {
    ERROR.unwrap_inc();
  }

  #[test]
  fn iresult_unwrap_inc_on_inc() {
    assert_eq!(INCOMPLETE.unwrap_inc(), Needed::Unknown);
  }

  #[test]
  fn iresult_to_result() {
    assert_eq!(DONE.to_result(), Ok(5));
    assert_eq!(INCOMPLETE.to_result(), Err(IError::Incomplete(Needed::Unknown)));
    assert_eq!(ERROR.to_result(), Err(IError::Error(error_code!(ErrorKind::Tag))));
  }
}
<commit_msg>rename IResult.to_result() to to_full_result()<commit_after>//! Basic types to build the parsers

use self::IResult::*;
use self::Needed::*;

#[cfg(feature = "core")]
use std::prelude::v1::*;

#[cfg(feature = "verbose-errors")]
use verbose_errors::Err;

#[cfg(not(feature = "verbose-errors"))]
use simple_errors::Err;

/// Contains information on needed data if a parser returned `Incomplete`
#[derive(Debug,PartialEq,Eq,Clone,Copy)]
pub enum Needed {
  /// needs more data, but we do not know how much
  Unknown,
  /// contains the required data size
  Size(usize)
}

impl Needed {
  pub fn is_known(&self) -> bool {
    *self != Unknown
  }

  /// Maps a `Needed` to `Needed` by appling a function to a contained `Size` value.
  #[inline]
  pub fn map<F: FnOnce(usize) -> usize>(self, f: F) -> Needed {
    match self {
      Unknown => Unknown,
      Size(n) => Size(f(n)),
    }
  }
}

#[cfg(feature = "verbose-errors")]
/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IResult<I,O,E=u32> {
   /// indicates a correct parsing, the first field containing the rest of the unparsed data, the second field contains the parsed data
  Done(I,O),
  /// contains a Err, an enum that can indicate an error code, a position in the input, and a pointer to another error, making a list of errors in the parsing tree
  Error(Err<I,E>),
  /// Incomplete contains a Needed, an enum than can represent a known quantity of input data, or unknown
  Incomplete(Needed)
}

#[cfg(not(feature = "verbose-errors"))]
/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IResult<I,O,E=u32> {
   /// indicates a correct parsing, the first field containing the rest of the unparsed data, the second field contains the parsed data
  Done(I,O),
  /// contains a Err, an enum that can indicate an error code, a position in the input, and a pointer to another error, making a list of errors in the parsing tree
  Error(Err<E>),
  /// Incomplete contains a Needed, an enum than can represent a known quantity of input data, or unknown
  Incomplete(Needed)
}

#[cfg(feature = "verbose-errors")]
/// This is the same as IResult, but without Done
///
/// This is used as the Error type when converting to std::result::Result
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IError<I,E=u32> {
  Error(Err<I,E>),
  Incomplete(Needed)
}

#[cfg(not(feature = "verbose-errors"))]
/// This is the same as IResult, but without Done
///
/// This is used as the Error type when converting to std::result::Result
#[derive(Debug,PartialEq,Eq,Clone)]
pub enum IError<I,E=u32> {
  Error(Err<E>),
  Incomplete(Needed)
}

impl<I,O,E> IResult<I,O,E> {
  pub fn is_done(&self) -> bool {
    match *self {
      Done(_,_) => true,
      _         => false
    }
  }

  pub fn is_err(&self) -> bool {
    match *self {
      Error(_) => true,
      _        => false
    }
  }

  pub fn is_incomplete(&self) -> bool {
    match *self {
      Incomplete(_) => true,
      _             => false
    }
  }

  /// Maps a `IResult<I, O, E>` to `IResult<I, N, E>` by appling a function
  /// to a contained `Done` value, leaving `Error` and `Incomplete` value
  /// untouched.
  #[inline]
  pub fn map<N, F: FnOnce(O) -> N>(self, f: F) -> IResult<I, N, E> {
    match self {
      Done(i, o)    => Done(i, f(o)),
      Error(e)      => Error(e),
      Incomplete(n) => Incomplete(n),
    }
  }

  /// Maps a `IResult<I, O, E>` to `IResult<I, O, E>` by appling a function
  /// to a contained `Incomplete` value, leaving `Done` and `Error` value
  /// untouched.
  #[inline]
  pub fn map_inc<F>(self, f: F) -> IResult<I, O, E>
   where F: FnOnce(Needed) -> Needed {
    match self {
      Error(e)      => Error(e),
      Incomplete(n) => Incomplete(f(n)),
      Done(i, o)    => Done(i, o),
    }
  }

  /// Unwrap the contained `Done(I, O)` value, or panic if the `IResult` is not
  /// `Done`.
  pub fn unwrap(self) -> (I, O) {
    match self {
      Done(i, o)    => (i, o),
      Incomplete(_) => panic!("unwrap() called on an IResult that is Incomplete"),
      Error(_)      => panic!("unwrap() called on an IResult that is Error")
    }
  }

  /// Unwrap the contained `Incomplete(n)` value, or panic if the `IResult` is not
  /// `Incomplete`.
  pub fn unwrap_inc(self) -> Needed {
    match self {
      Incomplete(n) => n,
      Done(_, _)    => panic!("unwrap_inc() called on an IResult that is Done"),
      Error(_)      => panic!("unwrap_inc() called on an IResult that is Error")
    }
  }

  /// Convert the IResult to a std::result::Result
  pub fn to_full_result(self) -> Result<O, IError<I,E>> {
    match self {
      Done(_, o)    => Ok(o),
      Incomplete(n) => Err(IError::Incomplete(n)),
      Error(e)      => Err(IError::Error(e))
    }
  }
}

pub trait GetInput<I> {
  fn remaining_input(&self) -> Option<I>;
}

pub trait GetOutput<O> {
  fn output(&self) -> Option<O>;
}

impl<'a,I,O,E> GetInput<&'a[I]> for IResult<&'a[I],O,E> {
  fn remaining_input(&self) -> Option<&'a[I]> {
    match *self {
      Done(ref i,_) => Some(*i),
      _             => None
    }
  }
}

impl<O,E> GetInput<()> for IResult<(),O,E> {
  fn remaining_input(&self) -> Option<()> {
    match *self {
      Done((),_) => Some(()),
      _          => None
    }
  }
}

impl<'a,O,E> GetInput<&'a str> for IResult<&'a str,O,E> {
  fn remaining_input(&self) -> Option<&'a str> {
    match *self {
      Done(ref i,_) => Some(*i),
      _          => None
    }
  }
}

impl<'a,I,O,E> GetOutput<&'a[O]> for IResult<I,&'a[O],E> {
  fn output(&self) -> Option<&'a[O]> {
    match *self {
      Done(_, ref o) => Some(*o),
      _              => None
    }
  }
}

impl<I,E> GetOutput<()> for IResult<I,(),E> {
  fn output(&self) -> Option<()> {
    match *self {
      Done(_,()) => Some(()),
      _          => None
    }
  }
}

impl<'a,I,E> GetOutput<&'a str> for IResult<I,&'a str,E> {
  fn output(&self) -> Option<&'a str> {
    match *self {
      Done(_,ref o) => Some(*o),
      _          => None
    }
  }
}

#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::any::Any;
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::{error,fmt};
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
use std::fmt::Debug;
#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
impl<P:Debug+Any,E:Debug+Any> error::Error for Err<P,E> {
  fn description(&self) -> &str {
    let kind = match *self {
      Err::Code(ref e) | Err::Node(ref e, _) | Err::Position(ref e, _) | Err::NodePosition(ref e, _, _) => e
    };
    kind.description()
  }
}

#[cfg(not(feature = "core"))]
#[cfg(feature = "verbose-errors")]
impl<P:fmt::Debug,E:fmt::Debug> fmt::Display for Err<P,E> {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    match *self {
      Err::Code(ref e) | Err::Node(ref e, _) => {
        write!(f, "{:?}", e)
      },
      Err::Position(ref e, ref p) | Err::NodePosition(ref e, ref p, _) => {
        write!(f, "{:?}:{:?}", p, e)
      }
    }
  }
}

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_code(
  ($code:expr) => ($crate::Err::Code($code));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_code(
  ($code:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_node(
  ($code:expr, $next:expr) => ($crate::Err::Node($code, ::std::boxed::Box::new($next)));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_node(
  ($code:expr, $next:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_position(
  ($code:expr, $input:expr) => ($crate::Err::Position($code, $input));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_position(
  ($code:expr, $input:expr) => ($code);
);

#[cfg(feature = "verbose-errors")]
#[macro_export]
macro_rules! error_node_position(
  ($code:expr, $input:expr, $next:expr) => ($crate::Err::NodePosition($code, $input, ::std::boxed::Box::new($next)));
);

#[cfg(not(feature = "verbose-errors"))]
#[macro_export]
macro_rules! error_node_position(
  ($code:expr, $input: expr, $next:expr) => ($code);
);

#[cfg(test)]
mod tests {
  use super::*;
  use util::ErrorKind;

  const REST: [u8; 0] = [];
  const DONE: IResult<&'static [u8], u32> = IResult::Done(&REST, 5);
  const ERROR: IResult<&'static [u8], u32> = IResult::Error(error_code!(ErrorKind::Tag));
  const INCOMPLETE: IResult<&'static [u8], u32> = IResult::Incomplete(Needed::Unknown);

  #[test]
  fn needed_map() {
    let unknown = Needed::Unknown;
    let size = Needed::Size(5);

    assert_eq!(size.map(|x| x * 2), Needed::Size(10));
    assert_eq!(unknown.map(|x| x * 2), Needed::Unknown);
  }

  #[test]
  fn iresult_map() {
    assert_eq!(DONE.map(|x| x * 2), IResult::Done(&b""[..], 10));
    assert_eq!(ERROR.map(|x| x * 2), IResult::Error(error_code!(ErrorKind::Tag)));
    assert_eq!(INCOMPLETE.map(|x| x * 2), IResult::Incomplete(Needed::Unknown));
  }

  #[test]
  fn iresult_map_inc() {
    let inc_unknown: IResult<&[u8], u32> = IResult::Incomplete(Needed::Unknown);
    let inc_size: IResult<&[u8], u32> = IResult::Incomplete(Needed::Size(5));

    assert_eq!(DONE.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Done(&b""[..], 5));
    assert_eq!(ERROR.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Error(error_code!(ErrorKind::Tag)));
    assert_eq!(inc_unknown.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Incomplete(Needed::Unknown));
    assert_eq!(inc_size.map_inc(|n| if let Needed::Size(i) = n {Needed::Size(i+1)} else {n}), IResult::Incomplete(Needed::Size(6)));
  }

  #[test]
  fn iresult_map_err() {
    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
    struct Error(u32);

    let error_kind = error_code!(ErrorKind::Custom(Error(5)));

    assert_eq!(DONE.map_err(|_| error_kind.clone()), IResult::Done(&b""[..], 5));
    assert_eq!(ERROR.map_err(|x| {println!("err: {:?}", x); error_kind.clone()}), IResult::Error(error_kind.clone()));
    assert_eq!(INCOMPLETE.map_err(|x| {println!("err: {:?}", x); error_kind.clone()}), IResult::Incomplete(Needed::Unknown));
  }

  #[test]
  fn iresult_unwrap_on_done() {
    assert_eq!(DONE.unwrap(), (&b""[..], 5));
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_on_err() {
    ERROR.unwrap();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_on_inc() {
    INCOMPLETE.unwrap();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_err_on_done() {
    DONE.unwrap_err();
  }

  #[test]
  fn iresult_unwrap_err_on_err() {
    assert_eq!(ERROR.unwrap_err(), error_code!(ErrorKind::Tag));
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_err_on_inc() {
    INCOMPLETE.unwrap_err();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_inc_on_done() {
    DONE.unwrap_inc();
  }

  #[test]
  #[should_panic]
  fn iresult_unwrap_inc_on_err() {
    ERROR.unwrap_inc();
  }

  #[test]
  fn iresult_unwrap_inc_on_inc() {
    assert_eq!(INCOMPLETE.unwrap_inc(), Needed::Unknown);
  }

  #[test]
  fn iresult_to_full_result() {
    assert_eq!(DONE.to_full_result(), Ok(5));
    assert_eq!(INCOMPLETE.to_full_result(), Err(IError::Incomplete(Needed::Unknown)));
    assert_eq!(ERROR.to_full_result(), Err(IError::Error(error_code!(ErrorKind::Tag))));
  }
}
<|endoftext|>


<commit_before>use std::hash::Hash;
use std::path::Path;
use std::ops::{Deref, DerefMut};
use std::collections::HashMap;

use perlin_core::index::Index;
use perlin_core::index::posting::DocId;

use language::PipelineBucket;

mod filter;
mod hierarchy;

pub use field::filter::Filter;
pub use field::hierarchy::Hierarchy;

pub enum FieldSupplement<T> {
    None,
    Filter(Filter<T>),
    Hierarchy(Hierarchy<T>),
}

pub struct Field<T: Hash + Eq> {
    index: Index<T>,
    pub term_doc_ratio: f32,
    pub supplement: FieldSupplement<T>,
}

impl<T: Hash + Eq + Ord + Clone + 'static> Field<T> {
    pub fn commit(&mut self) {
        self.index.commit();

        if let FieldSupplement::Filter(ref mut filter) = self.supplement {
            filter.commit(&self.index);
        }
    }
}

impl<TTerm> PipelineBucket<TTerm> for Field<TTerm>
    where TTerm: Hash + Eq + Ord
{
    fn put(&mut self, doc_id: DocId, term: TTerm) {
        self.index.index_term(doc_id, term);
    }
}

impl<T: Hash + Eq> DerefMut for Field<T> {
    fn deref_mut(&mut self) -> &mut Index<T> {
        &mut self.index
    }
}

impl<T: Hash + Eq> Deref for Field<T> {
    type Target = Index<T>;
    fn deref(&self) -> &Index<T> {
        &self.index
    }
}

pub struct Fields<T: Hash + Eq> {
    pub fields: HashMap<String, Field<T>>,
}

impl<T: Hash + Eq + Ord> Fields<T> {
    pub fn commit(&mut self) {
        for field in self.fields.values_mut() {
            field.commit();
        }
    }

    pub fn add_field(&mut self,
                     name: String,
                     path: &Path,
                     supplement: FieldSupplement<T>)
                     -> Result<(), ()> {
        use perlin_core::page_manager::{RamPageCache, FsPageManager};
        use perlin_core::index::vocabulary::SharedVocabulary;
        use perlin_core::index::Index;
        if self.fields.contains_key(&name) {
            return Err(());
        } else {
            let page_cache =
                RamPageCache::new(FsPageManager::new(&path.join(format!("{}_page_cache", name))));
            self.fields.insert(name,
                               Field {
                                   index: Index::new(page_cache, SharedVocabulary::new()),
                                   term_doc_ratio: 1.0,
                                   supplement,
                               });
            return Ok(());
        }
    }

    pub fn new() -> Self {
        Fields { fields: HashMap::new() }
    }
}

#[cfg(test)]
mod tests {
    use perlin_core::index::posting::DocId;
    use field::{Fields};

    use rust_stemmers::Algorithm;

    #[derive(PerlinDocument)]
    pub struct FilterTest {
        body: Fields<String>,
    }
    // pub use self::perlin_impl::FilterTestIndex;
    // mod perlin_impl {
    //     use super::*;
    //     use std::path::{Path, PathBuf};
    //     use std::borrow::Cow;
    //     use document_index::Pipeline;
    //     use document_index::QueryPipeline;
    //     use document_index::QueryResultIterator;
    //     use query::Operand;
    //     use query::Query;
    //     use perlin_core::index::posting::{PostingIterator, DocId};
    //     use perlin_core::index::vocabulary::TermId;
    //     pub struct FilterTestIndex {
    //         pub documents: FilterTest,
    //         pub query_pipeline: Option<QueryPipeline<FilterTest>>,
    //         pub doc_counter: DocId,
    //     }
    //     impl FilterTestIndex {
    //         pub fn create(base_path: PathBuf) -> Self {
    //             FilterTestIndex {
    //                 documents: FilterTest::create(&base_path),
    //                 query_pipeline: None,
    //                 doc_counter: DocId::none(),
    //             }
    //         }
    //         pub fn commit(&mut self) {
    //             self.documents.commit();
    //         }
    // pub fn set_query_pipeline(&mut self, pipe:
    // QueryPipeline<FilterTest>) {
    //             self.query_pipeline = Some(pipe);
    //         }
    //         pub fn run_query<'a>(&'a self, query: Query<'a>) -> Operand<'a> {
    //             if let Some(ref query_pipe) = self.query_pipeline {
    //                 query_pipe(&self.documents, query)
    //             } else {
    //                 {
    //                     panic!()
    //                 };
    //             }
    //         }
    //     }
    //     impl FilterTest {
    //         pub fn create(path: &Path) -> Self {
    //             use perlin_core::page_manager::{RamPageCache, FsPageManager};
    //             use perlin_core::index::vocabulary::SharedVocabulary;
    //             use perlin_core::index::Index;
    //             let body_page_cache =
    //                 RamPageCache::new(
    // FsPageManager::new(&path.join("body_page_cache")));
    //             let number_page_cache =
    //                 RamPageCache::new(
    // FsPageManager::new(&path.join("number_page_cache")));
    //             FilterTest {
    //                 body: Index::new(body_page_cache, SharedVocabulary::new()),
    // number: FilterField::new(Index::new(number_page_cache,
    // SharedVocabulary::new())),
    //             }
    //         }
    //         pub fn commit(&mut self) {
    //             self.body.commit();
    //             self.number.commit();
    //         }
    //     }
    // }

}
<commit_msg>fix commit call<commit_after>use std::hash::Hash;
use std::path::Path;
use std::ops::{Deref, DerefMut};
use std::collections::HashMap;

use perlin_core::index::Index;
use perlin_core::index::posting::DocId;

use language::PipelineBucket;

mod filter;
mod hierarchy;

pub use field::filter::Filter;
pub use field::hierarchy::Hierarchy;

pub enum FieldSupplement<T> {
    None,
    Filter(Filter<T>),
    Hierarchy(Hierarchy<T>),
}

pub struct Field<T: Hash + Eq> {
    index: Index<T>,
    pub term_doc_ratio: f32,
    pub supplement: FieldSupplement<T>,
}

impl<T: Hash + Eq + Ord + Clone + 'static> Field<T> {
    pub fn commit(&mut self) {
        self.index.commit();
        if let FieldSupplement::Filter(ref mut filter) = self.supplement {
            filter.commit(&self.index);
        }
    }
}

impl<TTerm> PipelineBucket<TTerm> for Field<TTerm>
    where TTerm: Hash + Eq + Ord
{
    fn put(&mut self, doc_id: DocId, term: TTerm) {
        self.index.index_term(doc_id, term);
    }
}

impl<T: Hash + Eq> DerefMut for Field<T> {
    fn deref_mut(&mut self) -> &mut Index<T> {
        &mut self.index
    }
}

impl<T: Hash + Eq> Deref for Field<T> {
    type Target = Index<T>;
    fn deref(&self) -> &Index<T> {
        &self.index
    }
}

pub struct Fields<T: Hash + Eq> {
    pub fields: HashMap<String, Field<T>>,
}

impl<T: Hash + Eq + Ord + Clone + 'static> Fields<T> {
    pub fn commit(&mut self) {
        for field in self.fields.values_mut() {
            (field as &mut Field<T>).commit();
        }
    }

    pub fn add_field(&mut self,
                     name: String,
                     path: &Path,
                     supplement: FieldSupplement<T>)
                     -> Result<(), ()> {
        use perlin_core::page_manager::{RamPageCache, FsPageManager};
        use perlin_core::index::vocabulary::SharedVocabulary;
        use perlin_core::index::Index;
        if self.fields.contains_key(&name) {
            return Err(());
        } else {
            let page_cache =
                RamPageCache::new(FsPageManager::new(&path.join(format!("{}_page_cache", name))));
            self.fields.insert(name,
                               Field {
                                   index: Index::new(page_cache, SharedVocabulary::new()),
                                   term_doc_ratio: 1.0,
                                   supplement,
                               });
            return Ok(());
        }
    }

    pub fn new() -> Self {
        Fields { fields: HashMap::new() }
    }
}

#[cfg(test)]
mod tests {
    use perlin_core::index::posting::DocId;
    use field::{Fields};

    use rust_stemmers::Algorithm;

    #[derive(PerlinDocument)]
    pub struct FilterTest {
        body: Fields<String>,
    }
    // pub use self::perlin_impl::FilterTestIndex;
    // mod perlin_impl {
    //     use super::*;
    //     use std::path::{Path, PathBuf};
    //     use std::borrow::Cow;
    //     use document_index::Pipeline;
    //     use document_index::QueryPipeline;
    //     use document_index::QueryResultIterator;
    //     use query::Operand;
    //     use query::Query;
    //     use perlin_core::index::posting::{PostingIterator, DocId};
    //     use perlin_core::index::vocabulary::TermId;
    //     pub struct FilterTestIndex {
    //         pub documents: FilterTest,
    //         pub query_pipeline: Option<QueryPipeline<FilterTest>>,
    //         pub doc_counter: DocId,
    //     }
    //     impl FilterTestIndex {
    //         pub fn create(base_path: PathBuf) -> Self {
    //             FilterTestIndex {
    //                 documents: FilterTest::create(&base_path),
    //                 query_pipeline: None,
    //                 doc_counter: DocId::none(),
    //             }
    //         }
    //         pub fn commit(&mut self) {
    //             self.documents.commit();
    //         }
    // pub fn set_query_pipeline(&mut self, pipe:
    // QueryPipeline<FilterTest>) {
    //             self.query_pipeline = Some(pipe);
    //         }
    //         pub fn run_query<'a>(&'a self, query: Query<'a>) -> Operand<'a> {
    //             if let Some(ref query_pipe) = self.query_pipeline {
    //                 query_pipe(&self.documents, query)
    //             } else {
    //                 {
    //                     panic!()
    //                 };
    //             }
    //         }
    //     }
    //     impl FilterTest {
    //         pub fn create(path: &Path) -> Self {
    //             use perlin_core::page_manager::{RamPageCache, FsPageManager};
    //             use perlin_core::index::vocabulary::SharedVocabulary;
    //             use perlin_core::index::Index;
    //             let body_page_cache =
    //                 RamPageCache::new(
    // FsPageManager::new(&path.join("body_page_cache")));
    //             let number_page_cache =
    //                 RamPageCache::new(
    // FsPageManager::new(&path.join("number_page_cache")));
    //             FilterTest {
    //                 body: Index::new(body_page_cache, SharedVocabulary::new()),
    // number: FilterField::new(Index::new(number_page_cache,
    // SharedVocabulary::new())),
    //             }
    //         }
    //         pub fn commit(&mut self) {
    //             self.body.commit();
    //             self.number.commit();
    //         }
    //     }
    // }

}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{error::StateSyncError, state_replication::StateComputer};
use anyhow::Result;
use consensus_types::{block::Block, executed_block::ExecutedBlock};
use diem_crypto::HashValue;
use diem_infallible::Mutex;
use diem_logger::prelude::*;
use diem_metrics::monitor;
use diem_types::ledger_info::LedgerInfoWithSignatures;
use execution_correctness::ExecutionCorrectness;
use executor_types::{Error as ExecutionError, StateComputeResult};
use fail::fail_point;
use state_sync::client::StateSyncClient;
use std::{boxed::Box, sync::Arc};

/// Basic communication with the Execution module;
/// implements StateComputer traits.
pub struct ExecutionProxy {
    execution_correctness_client: Mutex<Box<dyn ExecutionCorrectness + Send + Sync>>,
    synchronizer: StateSyncClient,
}

impl ExecutionProxy {
    pub fn new(
        execution_correctness_client: Box<dyn ExecutionCorrectness + Send + Sync>,
        synchronizer: StateSyncClient,
    ) -> Self {
        Self {
            execution_correctness_client: Mutex::new(execution_correctness_client),
            synchronizer,
        }
    }
}

#[async_trait::async_trait]
impl StateComputer for ExecutionProxy {
    fn compute(
        &self,
        // The block to be executed.
        block: &Block,
        // The parent block id.
        parent_block_id: HashValue,
    ) -> Result<StateComputeResult, ExecutionError> {
        fail_point!("consensus::compute", |_| {
            Err(ExecutionError::InternalError {
                error: "Injected error in compute".into(),
            })
        });
        debug!(
            block_id = block.id(),
            parent_id = block.parent_id(),
            "Executing block",
        );

        // TODO: figure out error handling for the prologue txn
        monitor!(
            "execute_block",
            self.execution_correctness_client
                .lock()
                .execute_block(block.clone(), parent_block_id)
        )
    }

    /// Send a successful commit. A future is fulfilled when the state is finalized.
    async fn commit(
        &self,
        blocks: &[Arc<ExecutedBlock>],
        finality_proof: LedgerInfoWithSignatures,
    ) -> Result<(), ExecutionError> {
        let mut block_ids = Vec::new();
        let mut txns = Vec::new();
        let mut reconfig_events = Vec::new();

        for block in blocks {
            block_ids.push(block.id());
            txns.extend(block.transactions_to_commit());
            reconfig_events.extend(block.compute_result().reconfig_events().to_vec());
        }

        monitor!(
            "commit_block",
            self.execution_correctness_client
                .lock()
                .commit_blocks(block_ids, finality_proof)?
        );

        if let Err(e) = monitor!(
            "notify_state_sync",
            self.synchronizer.commit(txns, reconfig_events).await
        ) {
            error!(error = ?e, "Failed to notify state synchronizer");
        }
        Ok(())
    }

    /// Synchronize to a commit that not present locally.
    async fn sync_to(&self, target: LedgerInfoWithSignatures) -> Result<(), StateSyncError> {
        fail_point!("consensus::sync_to", |_| {
            Err(anyhow::anyhow!("Injected error in sync_to").into())
        });
        // Here to start to do state synchronization where ChunkExecutor inside will
        // process chunks and commit to Storage. However, after block execution and
        // commitments, the the sync state of ChunkExecutor may be not up to date so
        // it is required to reset the cache of ChunkExecutor in State Sync
        // when requested to sync.
        let res = monitor!("sync_to", self.synchronizer.sync_to(target).await);
        // Similarily, after the state synchronization, we have to reset the cache
        // of BlockExecutor to guarantee the latest committed state is up to date.
        self.execution_correctness_client.lock().reset()?;

        res.map_err(|error| {
            let anyhow_error: anyhow::Error = error.into();
            anyhow_error.into()
        })
    }
}
<commit_msg>[execution] remove mutex after interior mutability<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{error::StateSyncError, state_replication::StateComputer};
use anyhow::Result;
use consensus_types::{block::Block, executed_block::ExecutedBlock};
use diem_crypto::HashValue;
use diem_logger::prelude::*;
use diem_metrics::monitor;
use diem_types::ledger_info::LedgerInfoWithSignatures;
use execution_correctness::ExecutionCorrectness;
use executor_types::{Error as ExecutionError, StateComputeResult};
use fail::fail_point;
use state_sync::client::StateSyncClient;
use std::{boxed::Box, sync::Arc};

/// Basic communication with the Execution module;
/// implements StateComputer traits.
pub struct ExecutionProxy {
    execution_correctness_client: Box<dyn ExecutionCorrectness + Send + Sync>,
    synchronizer: StateSyncClient,
}

impl ExecutionProxy {
    pub fn new(
        execution_correctness_client: Box<dyn ExecutionCorrectness + Send + Sync>,
        synchronizer: StateSyncClient,
    ) -> Self {
        Self {
            execution_correctness_client,
            synchronizer,
        }
    }
}

#[async_trait::async_trait]
impl StateComputer for ExecutionProxy {
    fn compute(
        &self,
        // The block to be executed.
        block: &Block,
        // The parent block id.
        parent_block_id: HashValue,
    ) -> Result<StateComputeResult, ExecutionError> {
        fail_point!("consensus::compute", |_| {
            Err(ExecutionError::InternalError {
                error: "Injected error in compute".into(),
            })
        });
        debug!(
            block_id = block.id(),
            parent_id = block.parent_id(),
            "Executing block",
        );

        // TODO: figure out error handling for the prologue txn
        monitor!(
            "execute_block",
            self.execution_correctness_client
                .execute_block(block.clone(), parent_block_id)
        )
    }

    /// Send a successful commit. A future is fulfilled when the state is finalized.
    async fn commit(
        &self,
        blocks: &[Arc<ExecutedBlock>],
        finality_proof: LedgerInfoWithSignatures,
    ) -> Result<(), ExecutionError> {
        let mut block_ids = Vec::new();
        let mut txns = Vec::new();
        let mut reconfig_events = Vec::new();

        for block in blocks {
            block_ids.push(block.id());
            txns.extend(block.transactions_to_commit());
            reconfig_events.extend(block.compute_result().reconfig_events().to_vec());
        }

        monitor!(
            "commit_block",
            self.execution_correctness_client
                .commit_blocks(block_ids, finality_proof)?
        );

        if let Err(e) = monitor!(
            "notify_state_sync",
            self.synchronizer.commit(txns, reconfig_events).await
        ) {
            error!(error = ?e, "Failed to notify state synchronizer");
        }
        Ok(())
    }

    /// Synchronize to a commit that not present locally.
    async fn sync_to(&self, target: LedgerInfoWithSignatures) -> Result<(), StateSyncError> {
        fail_point!("consensus::sync_to", |_| {
            Err(anyhow::anyhow!("Injected error in sync_to").into())
        });
        // Here to start to do state synchronization where ChunkExecutor inside will
        // process chunks and commit to Storage. However, after block execution and
        // commitments, the the sync state of ChunkExecutor may be not up to date so
        // it is required to reset the cache of ChunkExecutor in State Sync
        // when requested to sync.
        let res = monitor!("sync_to", self.synchronizer.sync_to(target).await);
        // Similarily, after the state synchronization, we have to reset the cache
        // of BlockExecutor to guarantee the latest committed state is up to date.
        self.execution_correctness_client.reset()?;

        res.map_err(|error| {
            let anyhow_error: anyhow::Error = error.into();
            anyhow_error.into()
        })
    }
}
<|endoftext|>