<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! This module provides `Pruner` which manages a thread pruning old data in the background and is
//! meant to be triggered by other threads as they commit new data to the DB.

use crate::{
    metrics::{
        LIBRA_STORAGE_OTHER_TIMERS_SECONDS, LIBRA_STORAGE_PRUNER_LEAST_READABLE_STATE_VERSION,
        LIBRA_STORAGE_PRUNE_WINDOW,
    },
    schema::{
        jellyfish_merkle_node::JellyfishMerkleNodeSchema, stale_node_index::StaleNodeIndexSchema,
    },
};
use anyhow::Result;
use libra_infallible::Mutex;
use libra_jellyfish_merkle::StaleNodeIndex;
use libra_logger::prelude::*;
use libra_types::transaction::Version;
use schemadb::{ReadOptions, SchemaBatch, SchemaIterator, DB};
#[cfg(test)]
use std::thread::sleep;
use std::{
    iter::Peekable,
    sync::{
        atomic::{AtomicU64, Ordering},
        mpsc::{channel, Receiver, Sender},
        Arc,
    },
    thread::JoinHandle,
    time::{Duration, Instant},
};

/// The `Pruner` is meant to be part of a `LibraDB` instance and runs in the background to prune old
/// data.
///
/// It creates a worker thread on construction and joins it on destruction. When destructed, it
/// quits the worker thread eagerly without waiting for all pending work to be done.
#[derive(Debug)]
pub(crate) struct Pruner {
    /// Other than the latest version, how many historical versions to keep being readable. For
    /// example, this being 0 means keep only the latest version.
    historical_versions_to_keep: u64,
    /// The worker thread handle, created upon Pruner instance construction and joined upon its
    /// destruction. It only becomes `None` after joined in `drop()`.
    worker_thread: Option<JoinHandle<()>>,
    /// The sender side of the channel talking to the worker thread.
    command_sender: Mutex<Sender<Command>>,
    /// (For tests) A way for the worker thread to inform the `Pruner` the pruning progress. If it
    /// sets this atomic value to `V`, all versions before `V` can no longer be accessed.
    #[allow(dead_code)]
    worker_progress: Arc<AtomicU64>,
}

impl Pruner {
    /// Creates a worker thread that waits on a channel for pruning commands.
    pub fn new(db: Arc<DB>, historical_versions_to_keep: u64) -> Self {
        let (command_sender, command_receiver) = channel();
        let worker_progress = Arc::new(AtomicU64::new(0));
        let worker_progress_clone = Arc::clone(&worker_progress);

        LIBRA_STORAGE_PRUNE_WINDOW.set(historical_versions_to_keep as i64);
        let worker_thread = std::thread::Builder::new()
            .name("libradb_pruner".into())
            .spawn(move || Worker::new(db, command_receiver, worker_progress_clone).work_loop())
            .expect("Creating pruner thread should succeed.");

        Self {
            historical_versions_to_keep,
            worker_thread: Some(worker_thread),
            command_sender: Mutex::new(command_sender),
            worker_progress,
        }
    }

    /// Sends pruning command to the worker thread when necessary.
    pub fn wake(&self, latest_version: Version) {
        if latest_version > self.historical_versions_to_keep {
            let least_readable_version = latest_version - self.historical_versions_to_keep;
            self.command_sender
                .lock()
                .send(Command::Prune {
                    least_readable_version,
                })
                .expect("Receiver should not destruct prematurely.");
        }
    }

    /// (For tests only.) Notifies the worker thread and waits for it to finish its job by polling
    /// an internal counter.
    #[cfg(test)]
    pub fn wake_and_wait(&self, latest_version: Version) -> Result<()> {
        self.wake(latest_version);

        if latest_version > self.historical_versions_to_keep {
            let least_readable_version = latest_version - self.historical_versions_to_keep;
            // Assuming no big pruning chunks will be issued by a test.
            const TIMEOUT: Duration = Duration::from_secs(10);
            let end = Instant::now() + TIMEOUT;

            while Instant::now() < end {
                if self.worker_progress.load(Ordering::Relaxed) >= least_readable_version {
                    return Ok(());
                }
                sleep(Duration::from_millis(1));
            }
            anyhow::bail!("Timeout waiting for pruner worker.");
        }
        Ok(())
    }
}

impl Drop for Pruner {
    fn drop(&mut self) {
        self.command_sender
            .lock()
            .send(Command::Quit)
            .expect("Receiver should not destruct.");
        self.worker_thread
            .take()
            .expect("Worker thread must exist.")
            .join()
            .expect("Worker thread should join peacefully.");
    }
}

enum Command {
    Quit,
    Prune { least_readable_version: Version },
}

struct Worker {
    db: Arc<DB>,
    command_receiver: Receiver<Command>,
    target_least_readable_version: Version,
    /// Keeps a record of the pruning progress. If this equals to version `V`, we know versions
    /// smaller than `V` are no longer readable.
    /// This being an atomic value is to communicate the info with the Pruner thread (for tests).
    least_readable_version: Arc<AtomicU64>,
    /// Indicates if there's NOT any pending work to do currently, to hint
    /// `Self::receive_commands()` to `recv()` blocking-ly.
    blocking_recv: bool,
    index_min_nonpurged_version: Version,
    index_purged_at: Instant,
}

impl Worker {
    const MAX_VERSIONS_TO_PRUNE_PER_BATCH: usize = 100;

    fn new(
        db: Arc<DB>,
        command_receiver: Receiver<Command>,
        least_readable_version: Arc<AtomicU64>,
    ) -> Self {
        Self {
            db,
            command_receiver,
            least_readable_version,
            target_least_readable_version: 0,
            blocking_recv: true,
            index_min_nonpurged_version: 0,
            index_purged_at: Instant::now(),
        }
    }

    fn work_loop(mut self) {
        while self.receive_commands() {
            // Process a reasonably small batch of work before trying to receive commands again,
            // in case `Command::Quit` is received (that's when we should quit.)
            match prune_state(
                Arc::clone(&self.db),
                self.least_readable_version.load(Ordering::Relaxed),
                self.target_least_readable_version,
                Self::MAX_VERSIONS_TO_PRUNE_PER_BATCH,
            ) {
                Ok(least_readable_version) => {
                    // Make next recv() blocking if all done.
                    self.blocking_recv =
                        least_readable_version == self.target_least_readable_version;

                    // Log the progress.
                    self.least_readable_version
                        .store(least_readable_version, Ordering::Relaxed);
                    LIBRA_STORAGE_PRUNER_LEAST_READABLE_STATE_VERSION
                        .set(least_readable_version as i64);

                    // Try to purge the log.
                    if let Err(e) = self.maybe_purge_index() {
                        warn!(
                            error = ?e,
                            "Failed purging state node index, ignored.",
                        );
                    }
                }
                Err(e) => {
                    error!(
                        error = ?e,
                        "Error pruning stale state nodes.",
                    );
                    // On error, stop retrying vigorously by making next recv() blocking.
                    self.blocking_recv = true;
                }
            }
        }
    }

    /// Tries to receive all pending commands, blocking waits for the next command if no work needs
    /// to be done, otherwise quits with `true` to allow the outer loop to do some work before
    /// getting back here.
    ///
    /// Returns `false` if `Command::Quit` is received, to break the outer loop and let
    /// `work_loop()` return.
    fn receive_commands(&mut self) -> bool {
        loop {
            let command = if self.blocking_recv {
                // Worker has nothing to do, blocking wait for the next command.
                self.command_receiver
                    .recv()
                    .expect("Sender should not destruct prematurely.")
            } else {
                // Worker has pending work to do, non-blocking recv.
                match self.command_receiver.try_recv() {
                    Ok(command) => command,
                    // Channel has drained, yield control to the outer loop.
                    Err(_) => return true,
                }
            };

            match command {
                // On `Command::Quit` inform the outer loop to quit by returning `false`.
                Command::Quit => return false,
                Command::Prune {
                    least_readable_version,
                } => {
                    if least_readable_version > self.target_least_readable_version {
                        self.target_least_readable_version = least_readable_version;
                        // Switch to non-blocking to allow some work to be done after the
                        // channel has drained.
                        self.blocking_recv = false;
                    }
                }
            }
        }
    }

    /// Purge the stale node index so that after restart not too much already pruned stuff is dealt
    /// with again (although no harm is done deleting those then non-existent things.)
    ///
    /// We issue (range) deletes on the index only periodically instead of after every pruning batch
    /// to avoid sending too many deletions to the DB, which takes disk space and slows it down.
    fn maybe_purge_index(&mut self) -> Result<()> {
        const MIN_INTERVAL: Duration = Duration::from_secs(60);
        const MIN_VERSIONS: u64 = 60000;

        // A deletion is issued at most once in one minute and when the pruner has progressed by at
        // least 60000 versions (assuming the pruner deletes as slow as 1000 versions per second,
        // this imposes at most one minute of work in vain after restarting.)
        let now = Instant::now();
        if now - self.index_purged_at > MIN_INTERVAL {
            let least_readable_version = self.least_readable_version.load(Ordering::Relaxed);

            if least_readable_version - self.index_min_nonpurged_version + 1 > MIN_VERSIONS {
                let new_min_non_purged_version = least_readable_version + 1;
                self.db.range_delete::<StaleNodeIndexSchema, Version>(
                    &self.index_min_nonpurged_version,
                    &new_min_non_purged_version, // end is exclusive
                )?;
                self.index_min_nonpurged_version = new_min_non_purged_version;
                self.index_purged_at = now;
            }
        }

        Ok(())
    }
}

struct StaleNodeIndicesByVersionIterator<'a> {
    inner: Peekable<SchemaIterator<'a, StaleNodeIndexSchema>>,
    target_least_readable_version: Version,
}

impl<'a> StaleNodeIndicesByVersionIterator<'a> {
    fn new(
        db: &'a DB,
        least_readable_version: Version,
        target_least_readable_version: Version,
    ) -> Result<Self> {
        let mut iter = db.iter::<StaleNodeIndexSchema>(ReadOptions::default())?;
        iter.seek(&least_readable_version)?;

        Ok(Self {
            inner: iter.peekable(),
            target_least_readable_version,
        })
    }

    fn next_result(&mut self) -> Result<Option<Vec<StaleNodeIndex>>> {
        match self.inner.next().transpose()? {
            None => Ok(None),
            Some((index, _)) => {
                let version = index.stale_since_version;
                if version > self.target_least_readable_version {
                    return Ok(None);
                }

                let mut indices = vec![index];
                while let Some(res) = self.inner.peek() {
                    if let Ok((index_ref, _)) = res {
                        if index_ref.stale_since_version != version {
                            break;
                        }
                    }

                    let (index, _) = self.inner.next().transpose()?.expect("Should be Some.");
                    indices.push(index);
                }

                Ok(Some(indices))
            }
        }
    }
}

impl<'a> Iterator for StaleNodeIndicesByVersionIterator<'a> {
    type Item = Result<Vec<StaleNodeIndex>>;

    fn next(&mut self) -> Option<Self::Item> {
        self.next_result().transpose()
    }
}

pub fn prune_state(
    db: Arc<DB>,
    least_readable_version: Version,
    target_least_readable_version: Version,
    max_versions: usize,
) -> Result<Version> {
    let indices = StaleNodeIndicesByVersionIterator::new(
        &db,
        least_readable_version,
        target_least_readable_version,
    )?
    .take(max_versions) // Iterator<Item = Result<Vec<StaleNodeIndex>>>
    .collect::<Result<Vec<_>>>()? // now Vec<Vec<StaleNodeIndex>>
    .into_iter()
    .flatten()
    .collect::<Vec<_>>();

    if indices.is_empty() {
        Ok(least_readable_version)
    } else {
        let _timer = LIBRA_STORAGE_OTHER_TIMERS_SECONDS
            .with_label_values(&["pruner_commit"])
            .start_timer();
        let new_least_readable_version = indices.last().expect("Should exist.").stale_since_version;
        let mut batch = SchemaBatch::new();
        indices
            .into_iter()
            .map(|index| batch.delete::<JellyfishMerkleNodeSchema>(&index.node_key))
            .collect::<Result<_>>()?;
        db.write_schemas(batch)?;
        Ok(new_least_readable_version)
    }
}

#[cfg(test)]
mod test;
<commit_msg>[storage] avoid waking up the pruner when not necessary<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! This module provides `Pruner` which manages a thread pruning old data in the background and is
//! meant to be triggered by other threads as they commit new data to the DB.

use crate::{
    metrics::{
        LIBRA_STORAGE_OTHER_TIMERS_SECONDS, LIBRA_STORAGE_PRUNER_LEAST_READABLE_STATE_VERSION,
        LIBRA_STORAGE_PRUNE_WINDOW,
    },
    schema::{
        jellyfish_merkle_node::JellyfishMerkleNodeSchema, stale_node_index::StaleNodeIndexSchema,
    },
};
use anyhow::Result;
use libra_infallible::Mutex;
use libra_jellyfish_merkle::StaleNodeIndex;
use libra_logger::prelude::*;
use libra_types::transaction::Version;
use schemadb::{ReadOptions, SchemaBatch, SchemaIterator, DB};
#[cfg(test)]
use std::thread::sleep;
use std::{
    iter::Peekable,
    sync::{
        atomic::{AtomicU64, Ordering},
        mpsc::{channel, Receiver, Sender},
        Arc,
    },
    thread::JoinHandle,
    time::{Duration, Instant},
};

/// The `Pruner` is meant to be part of a `LibraDB` instance and runs in the background to prune old
/// data.
///
/// It creates a worker thread on construction and joins it on destruction. When destructed, it
/// quits the worker thread eagerly without waiting for all pending work to be done.
#[derive(Debug)]
pub(crate) struct Pruner {
    /// Other than the latest version, how many historical versions to keep being readable. For
    /// example, this being 0 means keep only the latest version.
    historical_versions_to_keep: u64,
    /// The worker thread handle, created upon Pruner instance construction and joined upon its
    /// destruction. It only becomes `None` after joined in `drop()`.
    worker_thread: Option<JoinHandle<()>>,
    /// The sender side of the channel talking to the worker thread.
    command_sender: Mutex<Sender<Command>>,
    /// On start up, we query and store where the stale node index starts, before a pruning
    /// request hits beyond that, we don't need to wake the worker thread.
    initial_least_readable_version: Version,
    /// (For tests) A way for the worker thread to inform the `Pruner` the pruning progress. If it
    /// sets this atomic value to `V`, all versions before `V` can no longer be accessed.
    #[allow(dead_code)]
    worker_progress: Arc<AtomicU64>,
}

impl Pruner {
    /// Creates a worker thread that waits on a channel for pruning commands.
    pub fn new(db: Arc<DB>, historical_versions_to_keep: u64) -> Self {
        let (command_sender, command_receiver) = channel();

        let initial_least_readable_version = {
            let mut iter = db
                .iter::<StaleNodeIndexSchema>(ReadOptions::default())
                .expect("Create DB iter should succeed.");
            iter.seek_to_first();
            iter.next()
                .transpose()
                .expect("Seeking stale node index should succeed.")
                .map_or(0, |(index, _)| index.stale_since_version)
        };

        let worker_progress = Arc::new(AtomicU64::new(initial_least_readable_version));
        let worker_progress_clone = Arc::clone(&worker_progress);

        LIBRA_STORAGE_PRUNE_WINDOW.set(historical_versions_to_keep as i64);
        let worker_thread = std::thread::Builder::new()
            .name("libradb_pruner".into())
            .spawn(move || Worker::new(db, command_receiver, worker_progress_clone).work_loop())
            .expect("Creating pruner thread should succeed.");

        Self {
            historical_versions_to_keep,
            worker_thread: Some(worker_thread),
            command_sender: Mutex::new(command_sender),
            initial_least_readable_version,
            worker_progress,
        }
    }

    /// Sends pruning command to the worker thread when necessary.
    pub fn wake(&self, latest_version: Version) {
        if latest_version > self.historical_versions_to_keep {
            let least_readable_version = latest_version - self.historical_versions_to_keep;
            if least_readable_version > self.initial_least_readable_version {
                self.command_sender
                    .lock()
                    .send(Command::Prune {
                        least_readable_version,
                    })
                    .expect("Receiver should not destruct prematurely.");
            }
        }
    }

    /// (For tests only.) Notifies the worker thread and waits for it to finish its job by polling
    /// an internal counter.
    #[cfg(test)]
    pub fn wake_and_wait(&self, latest_version: Version) -> Result<()> {
        self.wake(latest_version);

        if latest_version > self.historical_versions_to_keep {
            let least_readable_version = latest_version - self.historical_versions_to_keep;
            // Assuming no big pruning chunks will be issued by a test.
            const TIMEOUT: Duration = Duration::from_secs(10);
            let end = Instant::now() + TIMEOUT;

            while Instant::now() < end {
                if self.worker_progress.load(Ordering::Relaxed) >= least_readable_version {
                    return Ok(());
                }
                sleep(Duration::from_millis(1));
            }
            anyhow::bail!("Timeout waiting for pruner worker.");
        }
        Ok(())
    }
}

impl Drop for Pruner {
    fn drop(&mut self) {
        self.command_sender
            .lock()
            .send(Command::Quit)
            .expect("Receiver should not destruct.");
        self.worker_thread
            .take()
            .expect("Worker thread must exist.")
            .join()
            .expect("Worker thread should join peacefully.");
    }
}

enum Command {
    Quit,
    Prune { least_readable_version: Version },
}

struct Worker {
    db: Arc<DB>,
    command_receiver: Receiver<Command>,
    target_least_readable_version: Version,
    /// Keeps a record of the pruning progress. If this equals to version `V`, we know versions
    /// smaller than `V` are no longer readable.
    /// This being an atomic value is to communicate the info with the Pruner thread (for tests).
    least_readable_version: Arc<AtomicU64>,
    /// Indicates if there's NOT any pending work to do currently, to hint
    /// `Self::receive_commands()` to `recv()` blocking-ly.
    blocking_recv: bool,
    index_min_nonpurged_version: Version,
    index_purged_at: Instant,
}

impl Worker {
    const MAX_VERSIONS_TO_PRUNE_PER_BATCH: usize = 100;

    fn new(
        db: Arc<DB>,
        command_receiver: Receiver<Command>,
        least_readable_version: Arc<AtomicU64>,
    ) -> Self {
        Self {
            db,
            command_receiver,
            least_readable_version,
            target_least_readable_version: 0,
            blocking_recv: true,
            index_min_nonpurged_version: 0,
            index_purged_at: Instant::now(),
        }
    }

    fn work_loop(mut self) {
        while self.receive_commands() {
            // Process a reasonably small batch of work before trying to receive commands again,
            // in case `Command::Quit` is received (that's when we should quit.)
            match prune_state(
                Arc::clone(&self.db),
                self.least_readable_version.load(Ordering::Relaxed),
                self.target_least_readable_version,
                Self::MAX_VERSIONS_TO_PRUNE_PER_BATCH,
            ) {
                Ok(least_readable_version) => {
                    // Make next recv() blocking if all done.
                    self.blocking_recv =
                        least_readable_version == self.target_least_readable_version;

                    // Log the progress.
                    self.least_readable_version
                        .store(least_readable_version, Ordering::Relaxed);
                    LIBRA_STORAGE_PRUNER_LEAST_READABLE_STATE_VERSION
                        .set(least_readable_version as i64);

                    // Try to purge the log.
                    if let Err(e) = self.maybe_purge_index() {
                        warn!(
                            error = ?e,
                            "Failed purging state node index, ignored.",
                        );
                    }
                }
                Err(e) => {
                    error!(
                        error = ?e,
                        "Error pruning stale state nodes.",
                    );
                    // On error, stop retrying vigorously by making next recv() blocking.
                    self.blocking_recv = true;
                }
            }
        }
    }

    /// Tries to receive all pending commands, blocking waits for the next command if no work needs
    /// to be done, otherwise quits with `true` to allow the outer loop to do some work before
    /// getting back here.
    ///
    /// Returns `false` if `Command::Quit` is received, to break the outer loop and let
    /// `work_loop()` return.
    fn receive_commands(&mut self) -> bool {
        loop {
            let command = if self.blocking_recv {
                // Worker has nothing to do, blocking wait for the next command.
                self.command_receiver
                    .recv()
                    .expect("Sender should not destruct prematurely.")
            } else {
                // Worker has pending work to do, non-blocking recv.
                match self.command_receiver.try_recv() {
                    Ok(command) => command,
                    // Channel has drained, yield control to the outer loop.
                    Err(_) => return true,
                }
            };

            match command {
                // On `Command::Quit` inform the outer loop to quit by returning `false`.
                Command::Quit => return false,
                Command::Prune {
                    least_readable_version,
                } => {
                    if least_readable_version > self.target_least_readable_version {
                        self.target_least_readable_version = least_readable_version;
                        // Switch to non-blocking to allow some work to be done after the
                        // channel has drained.
                        self.blocking_recv = false;
                    }
                }
            }
        }
    }

    /// Purge the stale node index so that after restart not too much already pruned stuff is dealt
    /// with again (although no harm is done deleting those then non-existent things.)
    ///
    /// We issue (range) deletes on the index only periodically instead of after every pruning batch
    /// to avoid sending too many deletions to the DB, which takes disk space and slows it down.
    fn maybe_purge_index(&mut self) -> Result<()> {
        const MIN_INTERVAL: Duration = Duration::from_secs(60);
        const MIN_VERSIONS: u64 = 60000;

        // A deletion is issued at most once in one minute and when the pruner has progressed by at
        // least 60000 versions (assuming the pruner deletes as slow as 1000 versions per second,
        // this imposes at most one minute of work in vain after restarting.)
        let now = Instant::now();
        if now - self.index_purged_at > MIN_INTERVAL {
            let least_readable_version = self.least_readable_version.load(Ordering::Relaxed);

            if least_readable_version - self.index_min_nonpurged_version + 1 > MIN_VERSIONS {
                let new_min_non_purged_version = least_readable_version + 1;
                self.db.range_delete::<StaleNodeIndexSchema, Version>(
                    &self.index_min_nonpurged_version,
                    &new_min_non_purged_version, // end is exclusive
                )?;
                self.index_min_nonpurged_version = new_min_non_purged_version;
                self.index_purged_at = now;
            }
        }

        Ok(())
    }
}

struct StaleNodeIndicesByVersionIterator<'a> {
    inner: Peekable<SchemaIterator<'a, StaleNodeIndexSchema>>,
    target_least_readable_version: Version,
}

impl<'a> StaleNodeIndicesByVersionIterator<'a> {
    fn new(
        db: &'a DB,
        least_readable_version: Version,
        target_least_readable_version: Version,
    ) -> Result<Self> {
        let mut iter = db.iter::<StaleNodeIndexSchema>(ReadOptions::default())?;
        iter.seek(&least_readable_version)?;

        Ok(Self {
            inner: iter.peekable(),
            target_least_readable_version,
        })
    }

    fn next_result(&mut self) -> Result<Option<Vec<StaleNodeIndex>>> {
        match self.inner.next().transpose()? {
            None => Ok(None),
            Some((index, _)) => {
                let version = index.stale_since_version;
                if version > self.target_least_readable_version {
                    return Ok(None);
                }

                let mut indices = vec![index];
                while let Some(res) = self.inner.peek() {
                    if let Ok((index_ref, _)) = res {
                        if index_ref.stale_since_version != version {
                            break;
                        }
                    }

                    let (index, _) = self.inner.next().transpose()?.expect("Should be Some.");
                    indices.push(index);
                }

                Ok(Some(indices))
            }
        }
    }
}

impl<'a> Iterator for StaleNodeIndicesByVersionIterator<'a> {
    type Item = Result<Vec<StaleNodeIndex>>;

    fn next(&mut self) -> Option<Self::Item> {
        self.next_result().transpose()
    }
}

pub fn prune_state(
    db: Arc<DB>,
    least_readable_version: Version,
    target_least_readable_version: Version,
    max_versions: usize,
) -> Result<Version> {
    let indices = StaleNodeIndicesByVersionIterator::new(
        &db,
        least_readable_version,
        target_least_readable_version,
    )?
    .take(max_versions) // Iterator<Item = Result<Vec<StaleNodeIndex>>>
    .collect::<Result<Vec<_>>>()? // now Vec<Vec<StaleNodeIndex>>
    .into_iter()
    .flatten()
    .collect::<Vec<_>>();

    if indices.is_empty() {
        Ok(least_readable_version)
    } else {
        let _timer = LIBRA_STORAGE_OTHER_TIMERS_SECONDS
            .with_label_values(&["pruner_commit"])
            .start_timer();
        let new_least_readable_version = indices.last().expect("Should exist.").stale_since_version;
        let mut batch = SchemaBatch::new();
        indices
            .into_iter()
            .map(|index| batch.delete::<JellyfishMerkleNodeSchema>(&index.node_key))
            .collect::<Result<_>>()?;
        db.write_schemas(batch)?;
        Ok(new_least_readable_version)
    }
}

#[cfg(test)]
mod test;
<|endoftext|>


<commit_before>use std::fmt::{self, Display};
use std::str::FromStr;
use unicase::UniCase;

pub use self::ConnectionOption::{KeepAlive, Close, ConnectionHeader};

/// Values that can be in the `Connection` header.
#[derive(Clone, PartialEq, Debug)]
pub enum ConnectionOption {
    /// The `keep-alive` connection value.
    KeepAlive,
    /// The `close` connection value.
    Close,
    /// Values in the Connection header that are supposed to be names of other Headers.
    ///
    /// > When a header field aside from Connection is used to supply control
    /// > information for or about the current connection, the sender MUST list
    /// > the corresponding field-name within the Connection header field.
    // TODO: it would be nice if these "Strings" could be stronger types, since
    // they are supposed to relate to other Header fields (which we have strong
    // types for).
    ConnectionHeader(UniCase<String>),
}

impl FromStr for ConnectionOption {
    type Err = ();
    fn from_str(s: &str) -> Result<ConnectionOption, ()> {
        match s {
            "keep-alive" => Ok(KeepAlive),
            "close" => Ok(Close),
            s => Ok(ConnectionHeader(UniCase(s.to_owned())))
        }
    }
}

impl Display for ConnectionOption {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(match *self {
            KeepAlive => "keep-alive",
            Close => "close",
            ConnectionHeader(UniCase(ref s)) => s.as_ref()
        })
    }
}

header! {
    #[doc="`Connection` header, defined in"]
    #[doc="[RFC7230](http://tools.ietf.org/html/rfc7230#section-6.1)"]
    #[doc=""]
    #[doc="The `Connection` header field allows the sender to indicate desired"]
    #[doc="control options for the current connection.  In order to avoid"]
    #[doc="confusing downstream recipients, a proxy or gateway MUST remove or"]
    #[doc="replace any received connection options before forwarding the"]
    #[doc="message."]
    #[doc=""]
    #[doc="# ABNF"]
    #[doc="```plain"]
    #[doc="Connection        = 1#connection-option"]
    #[doc="connection-option = token"]
    #[doc=""]
    #[doc="# Example values"]
    #[doc="* `close`"]
    #[doc="* `keep-alive`"]
    #[doc="* `upgrade`"]
    #[doc="```"]
    #[doc=""]
    #[doc="# Examples"]
    #[doc="```"]
    #[doc="use hyper::header::{Headers, Connection};"]
    #[doc=""]
    #[doc="let mut headers = Headers::new();"]
    #[doc="headers.set(Connection::keep_alive());"]
    #[doc="```"]
    #[doc="```"]
    #[doc="# extern crate hyper;"]
    #[doc="# extern crate unicase;"]
    #[doc="# fn main() {"]
    #[doc="// extern crate unicase;"]
    #[doc=""]
    #[doc="use hyper::header::{Headers, Connection, ConnectionOption};"]
    #[doc="use unicase::UniCase;"]
    #[doc=""]
    #[doc="let mut headers = Headers::new();"]
    #[doc="headers.set("]
    #[doc="    Connection(vec!["]
    #[doc="        ConnectionOption::ConnectionHeader(UniCase(\"upgrade\".to_owned())),"]
    #[doc="    ])"]
    #[doc=");"]
    #[doc="# }"]
    #[doc="```"]
    (Connection, "Connection") => (ConnectionOption)+

    test_connection {
        test_header!(test1, vec![b"close"]);
        test_header!(test2, vec![b"keep-alive"]);
        test_header!(test3, vec![b"upgrade"]);
    }
}

impl Connection {
    /// A constructor to easily create a `Connection: close` header.
    #[inline]
    pub fn close() -> Connection {
        Connection(vec![ConnectionOption::Close])
    }

    /// A constructor to easily create a `Connection: keep-alive` header.
    #[inline]
    pub fn keep_alive() -> Connection {
        Connection(vec![ConnectionOption::KeepAlive])
    }
}

bench_header!(close, Connection, { vec![b"close".to_vec()] });
bench_header!(keep_alive, Connection, { vec![b"keep-alive".to_vec()] });
bench_header!(header, Connection, { vec![b"authorization".to_vec()] });

#[cfg(test)]
mod tests {
    use super::{Connection,ConnectionHeader};
    use header::Header;
    use unicase::UniCase;

    fn parse_option(header: Vec<u8>) -> Connection {
        let val = vec![header];
        let connection: Connection = Header::parse_header(&val[..]).unwrap();
        connection
    }

    #[test]
    fn test_parse() {
        assert_eq!(Connection::close(),parse_option(b"close".to_vec()));
        assert_eq!(Connection::keep_alive(),parse_option(b"keep-alive".to_vec()));
        assert_eq!(Connection(vec![ConnectionHeader(UniCase("upgrade".to_owned()))]),
            parse_option(b"upgrade".to_vec()));
    }
}
<commit_msg>fix(headers): case insensitive values for Connection header<commit_after>use std::fmt::{self, Display};
use std::str::FromStr;
use unicase::UniCase;

pub use self::ConnectionOption::{KeepAlive, Close, ConnectionHeader};

const KEEP_ALIVE: UniCase<&'static str> = UniCase("keep-alive");
const CLOSE: UniCase<&'static str> = UniCase("close");

/// Values that can be in the `Connection` header.
#[derive(Clone, PartialEq, Debug)]
pub enum ConnectionOption {
    /// The `keep-alive` connection value.
    KeepAlive,
    /// The `close` connection value.
    Close,
    /// Values in the Connection header that are supposed to be names of other Headers.
    ///
    /// > When a header field aside from Connection is used to supply control
    /// > information for or about the current connection, the sender MUST list
    /// > the corresponding field-name within the Connection header field.
    // TODO: it would be nice if these "Strings" could be stronger types, since
    // they are supposed to relate to other Header fields (which we have strong
    // types for).
    ConnectionHeader(UniCase<String>),
}

impl FromStr for ConnectionOption {
    type Err = ();
    fn from_str(s: &str) -> Result<ConnectionOption, ()> {
        if UniCase(s) == KEEP_ALIVE {
            Ok(KeepAlive)
        } else if UniCase(s) == CLOSE {
            Ok(Close)
        } else {
            Ok(ConnectionHeader(UniCase(s.to_owned())))
        }
    }
}

impl Display for ConnectionOption {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(match *self {
            KeepAlive => "keep-alive",
            Close => "close",
            ConnectionHeader(UniCase(ref s)) => s.as_ref()
        })
    }
}

header! {
    #[doc="`Connection` header, defined in"]
    #[doc="[RFC7230](http://tools.ietf.org/html/rfc7230#section-6.1)"]
    #[doc=""]
    #[doc="The `Connection` header field allows the sender to indicate desired"]
    #[doc="control options for the current connection.  In order to avoid"]
    #[doc="confusing downstream recipients, a proxy or gateway MUST remove or"]
    #[doc="replace any received connection options before forwarding the"]
    #[doc="message."]
    #[doc=""]
    #[doc="# ABNF"]
    #[doc="```plain"]
    #[doc="Connection        = 1#connection-option"]
    #[doc="connection-option = token"]
    #[doc=""]
    #[doc="# Example values"]
    #[doc="* `close`"]
    #[doc="* `keep-alive`"]
    #[doc="* `upgrade`"]
    #[doc="```"]
    #[doc=""]
    #[doc="# Examples"]
    #[doc="```"]
    #[doc="use hyper::header::{Headers, Connection};"]
    #[doc=""]
    #[doc="let mut headers = Headers::new();"]
    #[doc="headers.set(Connection::keep_alive());"]
    #[doc="```"]
    #[doc="```"]
    #[doc="# extern crate hyper;"]
    #[doc="# extern crate unicase;"]
    #[doc="# fn main() {"]
    #[doc="// extern crate unicase;"]
    #[doc=""]
    #[doc="use hyper::header::{Headers, Connection, ConnectionOption};"]
    #[doc="use unicase::UniCase;"]
    #[doc=""]
    #[doc="let mut headers = Headers::new();"]
    #[doc="headers.set("]
    #[doc="    Connection(vec!["]
    #[doc="        ConnectionOption::ConnectionHeader(UniCase(\"upgrade\".to_owned())),"]
    #[doc="    ])"]
    #[doc=");"]
    #[doc="# }"]
    #[doc="```"]
    (Connection, "Connection") => (ConnectionOption)+

    test_connection {
        test_header!(test1, vec![b"close"]);
        test_header!(test2, vec![b"keep-alive"]);
        test_header!(test3, vec![b"upgrade"]);
    }
}

impl Connection {
    /// A constructor to easily create a `Connection: close` header.
    #[inline]
    pub fn close() -> Connection {
        Connection(vec![ConnectionOption::Close])
    }

    /// A constructor to easily create a `Connection: keep-alive` header.
    #[inline]
    pub fn keep_alive() -> Connection {
        Connection(vec![ConnectionOption::KeepAlive])
    }
}

bench_header!(close, Connection, { vec![b"close".to_vec()] });
bench_header!(keep_alive, Connection, { vec![b"keep-alive".to_vec()] });
bench_header!(header, Connection, { vec![b"authorization".to_vec()] });

#[cfg(test)]
mod tests {
    use super::{Connection,ConnectionHeader};
    use header::Header;
    use unicase::UniCase;

    fn parse_option(header: Vec<u8>) -> Connection {
        let val = vec![header];
        let connection: Connection = Header::parse_header(&val[..]).unwrap();
        connection
    }

    #[test]
    fn test_parse() {
        assert_eq!(Connection::close(),parse_option(b"close".to_vec()));
        assert_eq!(Connection::keep_alive(),parse_option(b"keep-alive".to_vec()));
        assert_eq!(Connection::keep_alive(),parse_option(b"Keep-Alive".to_vec()));
        assert_eq!(Connection(vec![ConnectionHeader(UniCase("upgrade".to_owned()))]),
            parse_option(b"upgrade".to_vec()));
    }
}
<|endoftext|>


<commit_before>//! Determining which types for which we can emit `#[derive(Debug)]`.

use super::{ConstrainResult, MonotoneFramework};
use std::collections::HashSet;
use std::collections::HashMap;
use ir::context::{BindgenContext, ItemId};
use ir::item::IsOpaque;
use ir::traversal::EdgeKind;
use ir::ty::RUST_DERIVE_IN_ARRAY_LIMIT;
use ir::ty::TypeKind;
use ir::comp::Field;
use ir::traversal::Trace;
use ir::comp::FieldMethods;
use ir::derive::CanTriviallyDeriveDebug;
use ir::comp::CompKind;

/// An analysis that finds for each IR item whether debug cannot be derived.
///
/// We use the monotone constraint function `cannot_derive_debug`, defined as
/// follows:
///
/// * If T is Opaque and layout of the type is known, get this layout as opaque
///   type and check whether it can be derived using trivial checks.
/// * If T is Array type, debug cannot be derived if the length of the array is
///   larger than the limit or the type of data the array contains cannot derive
///   debug.
/// * If T is a type alias, a templated alias or an indirection to another type,
///   debug cannot be derived if the type T refers to cannot be derived debug.
/// * If T is a compound type, debug cannot be derived if any of its base member
///   or field cannot be derived debug.
/// * If T is a pointer, T cannot be derived debug if T is a function pointer
///   and the function signature cannot be derived debug.
/// * If T is an instantiation of an abstract template definition, T cannot be
///   derived debug if any of the template arguments or template definition
///   cannot derive debug.
#[derive(Debug, Clone)]
pub struct CannotDeriveDebug<'ctx, 'gen>
    where 'gen: 'ctx
{
    ctx: &'ctx BindgenContext<'gen>,

    // The incremental result of this analysis's computation. Everything in this
    // set cannot derive debug.
    cannot_derive_debug: HashSet<ItemId>,

    // Dependencies saying that if a key ItemId has been inserted into the
    // `cannot_derive_debug` set, then each of the ids in Vec<ItemId> need to be
    // considered again.
    //
    // This is a subset of the natural IR graph with reversed edges, where we
    // only include the edges from the IR graph that can affect whether a type
    // can derive debug or not.
    dependencies: HashMap<ItemId, Vec<ItemId>>,
}

impl<'ctx, 'gen> CannotDeriveDebug<'ctx, 'gen> {
    fn consider_edge(kind: EdgeKind) -> bool {
        match kind {
            // These are the only edges that can affect whether a type can derive
            // debug or not.
            EdgeKind::BaseMember |
            EdgeKind::Field |
            EdgeKind::TypeReference |
            EdgeKind::VarType |
            EdgeKind::TemplateArgument |
            EdgeKind::TemplateDeclaration |
            EdgeKind::TemplateParameterDefinition => true,

            EdgeKind::Constructor |
            EdgeKind::Destructor |
            EdgeKind::FunctionReturn |
            EdgeKind::FunctionParameter |
            EdgeKind::InnerType |
            EdgeKind::InnerVar |
            EdgeKind::Method => false,
            EdgeKind::Generic => false,
        }
    }

    fn insert(&mut self, id: ItemId) -> ConstrainResult {
        let was_not_already_in_set = self.cannot_derive_debug.insert(id);
        assert!(
            was_not_already_in_set,
            "We shouldn't try and insert {:?} twice because if it was \
             already in the set, `constrain` should have exited early.",
            id
        );
        ConstrainResult::Changed
    }
}

impl<'ctx, 'gen> MonotoneFramework for CannotDeriveDebug<'ctx, 'gen> {
    type Node = ItemId;
    type Extra = &'ctx BindgenContext<'gen>;
    type Output = HashSet<ItemId>;

    fn new(ctx: &'ctx BindgenContext<'gen>) -> CannotDeriveDebug<'ctx, 'gen> {
        let cannot_derive_debug = HashSet::new();
        let mut dependencies = HashMap::new();

        for &item in ctx.whitelisted_items() {
            dependencies.entry(item).or_insert(vec![]);

            {
                // We reverse our natural IR graph edges to find dependencies
                // between nodes.
                item.trace(ctx, &mut |sub_item: ItemId, edge_kind| {
                    if ctx.whitelisted_items().contains(&sub_item) &&
                       Self::consider_edge(edge_kind) {
                           dependencies.entry(sub_item)
                               .or_insert(vec![])
                               .push(item);
                    }
                }, &());
            }
        }

        CannotDeriveDebug {
            ctx,
            cannot_derive_debug,
            dependencies,
        }
    }

    fn initial_worklist(&self) -> Vec<ItemId> {
        self.ctx.whitelisted_items().iter().cloned().collect()
    }

    fn constrain(&mut self, id: ItemId) -> ConstrainResult {
        if self.cannot_derive_debug.contains(&id) {
            return ConstrainResult::Same;
        }

        let item = self.ctx.resolve_item(id);
        let ty = match item.as_type() {
            None => return ConstrainResult::Same,
            Some(ty) => ty
        };

        if ty.is_opaque(self.ctx, item) {
            let layout_can_derive = ty.layout(self.ctx).map_or(true, |l| {
                l.opaque().can_trivially_derive_debug(self.ctx, ())
            });
            if layout_can_derive {
                return ConstrainResult::Same;
            } else {
                return self.insert(id);
            }
        }

        match *ty.kind() {
            // Handle the simple cases. These can derive debug without further
            // information.
            TypeKind::Void |
            TypeKind::NullPtr |
            TypeKind::Int(..) |
            TypeKind::Float(..) |
            TypeKind::Complex(..) |
            TypeKind::Function(..) |
            TypeKind::Enum(..) |
            TypeKind::Reference(..) |
            TypeKind::BlockPointer |
            TypeKind::Named |
            TypeKind::UnresolvedTypeRef(..) |
            TypeKind::ObjCInterface(..) |
            TypeKind::ObjCId |
            TypeKind::ObjCSel => {
                ConstrainResult::Same
            }

            TypeKind::Array(t, len) => {
                if self.cannot_derive_debug.contains(&t) {
                    return self.insert(id);
                }

                if len <= RUST_DERIVE_IN_ARRAY_LIMIT {
                    ConstrainResult::Same
                } else {
                    self.insert(id)
                }
            }

            TypeKind::ResolvedTypeRef(t) |
            TypeKind::TemplateAlias(t, _) |
            TypeKind::Alias(t) => {
                if self.cannot_derive_debug.contains(&t) {
                    self.insert(id)
                } else {
                    ConstrainResult::Same
                }
            }

            TypeKind::Comp(ref info) => {
                assert!(
                    !info.has_non_type_template_params(),
                    "The early ty.is_opaque check should have handled this case"
                );

                if info.kind() == CompKind::Union {
                    if self.ctx.options().unstable_rust {
                        return self.insert(id);
                    }

                    if ty.layout(self.ctx)
                        .map_or(true,
                                |l| l.opaque().can_trivially_derive_debug(self.ctx, ())) {
                        return ConstrainResult::Same;
                    } else {
                        return self.insert(id);
                    }
                }

                let bases_cannot_derive = info.base_members()
                    .iter()
                    .any(|base| self.cannot_derive_debug.contains(&base.ty));
                if bases_cannot_derive {
                    return self.insert(id);
                }

                let fields_cannot_derive = info.fields()
                    .iter()
                    .any(|f| {
                        match *f {
                            Field::DataMember(ref data) => {
                                self.cannot_derive_debug.contains(&data.ty())
                            }
                            Field::Bitfields(ref bfu) => {
                                bfu.bitfields()
                                    .iter().any(|b| {
                                        self.cannot_derive_debug.contains(&b.ty())
                                    })
                            }
                        }
                    });
                if fields_cannot_derive {
                    return self.insert(id);
                }

                ConstrainResult::Same
            }

            TypeKind::Pointer(inner) => {
                let inner_type = self.ctx.resolve_type(inner).canonical_type(self.ctx);
                if let TypeKind::Function(ref sig) = *inner_type.kind() {
                    if !sig.can_trivially_derive_debug(&self.ctx, ()) {
                        return self.insert(id);
                    }
                }
                ConstrainResult::Same
            }

            TypeKind::TemplateInstantiation(ref template) => {
                let args_cannot_derive = template.template_arguments()
                    .iter()
                    .any(|arg| self.cannot_derive_debug.contains(&arg));
                if args_cannot_derive {
                    return self.insert(id);
                }

                assert!(
                    !template.template_definition().is_opaque(self.ctx, &()),
                    "The early ty.is_opaque check should have handled this case"
                );
                let def_cannot_derive = self.cannot_derive_debug
                    .contains(&template.template_definition());
                if def_cannot_derive {
                    return self.insert(id);
                }

                ConstrainResult::Same
            }

            TypeKind::Opaque => {
                unreachable!(
                    "The early ty.is_opaque check should have handled this case"
                )
            }
        }
    }

    fn each_depending_on<F>(&self, id: ItemId, mut f: F)
        where F: FnMut(ItemId),
    {
        if let Some(edges) = self.dependencies.get(&id) {
            for item in edges {
                trace!("enqueue {:?} into worklist", item);
                f(*item);
            }
        }
    }
}

impl<'ctx, 'gen> From<CannotDeriveDebug<'ctx, 'gen>> for HashSet<ItemId> {
    fn from(analysis: CannotDeriveDebug<'ctx, 'gen>) -> Self {
        analysis.cannot_derive_debug
    }
}
<commit_msg>Add `trace!` logging for the derive(Debug) analysis<commit_after>//! Determining which types for which we can emit `#[derive(Debug)]`.

use super::{ConstrainResult, MonotoneFramework};
use std::collections::HashSet;
use std::collections::HashMap;
use ir::context::{BindgenContext, ItemId};
use ir::item::IsOpaque;
use ir::traversal::EdgeKind;
use ir::ty::RUST_DERIVE_IN_ARRAY_LIMIT;
use ir::ty::TypeKind;
use ir::comp::Field;
use ir::traversal::Trace;
use ir::comp::FieldMethods;
use ir::derive::CanTriviallyDeriveDebug;
use ir::comp::CompKind;

/// An analysis that finds for each IR item whether debug cannot be derived.
///
/// We use the monotone constraint function `cannot_derive_debug`, defined as
/// follows:
///
/// * If T is Opaque and layout of the type is known, get this layout as opaque
///   type and check whether it can be derived using trivial checks.
/// * If T is Array type, debug cannot be derived if the length of the array is
///   larger than the limit or the type of data the array contains cannot derive
///   debug.
/// * If T is a type alias, a templated alias or an indirection to another type,
///   debug cannot be derived if the type T refers to cannot be derived debug.
/// * If T is a compound type, debug cannot be derived if any of its base member
///   or field cannot be derived debug.
/// * If T is a pointer, T cannot be derived debug if T is a function pointer
///   and the function signature cannot be derived debug.
/// * If T is an instantiation of an abstract template definition, T cannot be
///   derived debug if any of the template arguments or template definition
///   cannot derive debug.
#[derive(Debug, Clone)]
pub struct CannotDeriveDebug<'ctx, 'gen>
    where 'gen: 'ctx
{
    ctx: &'ctx BindgenContext<'gen>,

    // The incremental result of this analysis's computation. Everything in this
    // set cannot derive debug.
    cannot_derive_debug: HashSet<ItemId>,

    // Dependencies saying that if a key ItemId has been inserted into the
    // `cannot_derive_debug` set, then each of the ids in Vec<ItemId> need to be
    // considered again.
    //
    // This is a subset of the natural IR graph with reversed edges, where we
    // only include the edges from the IR graph that can affect whether a type
    // can derive debug or not.
    dependencies: HashMap<ItemId, Vec<ItemId>>,
}

impl<'ctx, 'gen> CannotDeriveDebug<'ctx, 'gen> {
    fn consider_edge(kind: EdgeKind) -> bool {
        match kind {
            // These are the only edges that can affect whether a type can derive
            // debug or not.
            EdgeKind::BaseMember |
            EdgeKind::Field |
            EdgeKind::TypeReference |
            EdgeKind::VarType |
            EdgeKind::TemplateArgument |
            EdgeKind::TemplateDeclaration |
            EdgeKind::TemplateParameterDefinition => true,

            EdgeKind::Constructor |
            EdgeKind::Destructor |
            EdgeKind::FunctionReturn |
            EdgeKind::FunctionParameter |
            EdgeKind::InnerType |
            EdgeKind::InnerVar |
            EdgeKind::Method => false,
            EdgeKind::Generic => false,
        }
    }

    fn insert(&mut self, id: ItemId) -> ConstrainResult {
        trace!("inserting {:?} into the cannot_derive_debug set", id);

        let was_not_already_in_set = self.cannot_derive_debug.insert(id);
        assert!(
            was_not_already_in_set,
            "We shouldn't try and insert {:?} twice because if it was \
             already in the set, `constrain` should have exited early.",
            id
        );

        ConstrainResult::Changed
    }
}

impl<'ctx, 'gen> MonotoneFramework for CannotDeriveDebug<'ctx, 'gen> {
    type Node = ItemId;
    type Extra = &'ctx BindgenContext<'gen>;
    type Output = HashSet<ItemId>;

    fn new(ctx: &'ctx BindgenContext<'gen>) -> CannotDeriveDebug<'ctx, 'gen> {
        let cannot_derive_debug = HashSet::new();
        let mut dependencies = HashMap::new();

        for &item in ctx.whitelisted_items() {
            dependencies.entry(item).or_insert(vec![]);

            {
                // We reverse our natural IR graph edges to find dependencies
                // between nodes.
                item.trace(ctx, &mut |sub_item: ItemId, edge_kind| {
                    if ctx.whitelisted_items().contains(&sub_item) &&
                       Self::consider_edge(edge_kind) {
                           dependencies.entry(sub_item)
                               .or_insert(vec![])
                               .push(item);
                    }
                }, &());
            }
        }

        CannotDeriveDebug {
            ctx,
            cannot_derive_debug,
            dependencies,
        }
    }

    fn initial_worklist(&self) -> Vec<ItemId> {
        self.ctx.whitelisted_items().iter().cloned().collect()
    }

    fn constrain(&mut self, id: ItemId) -> ConstrainResult {
        trace!("constrain: {:?}", id);

        if self.cannot_derive_debug.contains(&id) {
            trace!("    already know it cannot derive Debug");
            return ConstrainResult::Same;
        }

        let item = self.ctx.resolve_item(id);
        let ty = match item.as_type() {
            Some(ty) => ty,
            None => {
                trace!("    not a type; ignoring");
                return ConstrainResult::Same;
            }
        };

        if ty.is_opaque(self.ctx, item) {
            let layout_can_derive = ty.layout(self.ctx).map_or(true, |l| {
                l.opaque().can_trivially_derive_debug(self.ctx, ())
            });
            if layout_can_derive {
                trace!("    we can trivially derive Debug for the layout");
                return ConstrainResult::Same;
            } else {
                trace!("    we cannot derive Debug for the layout");
                return self.insert(id);
            }
        }

        match *ty.kind() {
            // Handle the simple cases. These can derive debug without further
            // information.
            TypeKind::Void |
            TypeKind::NullPtr |
            TypeKind::Int(..) |
            TypeKind::Float(..) |
            TypeKind::Complex(..) |
            TypeKind::Function(..) |
            TypeKind::Enum(..) |
            TypeKind::Reference(..) |
            TypeKind::BlockPointer |
            TypeKind::Named |
            TypeKind::UnresolvedTypeRef(..) |
            TypeKind::ObjCInterface(..) |
            TypeKind::ObjCId |
            TypeKind::ObjCSel => {
                trace!("    simple type that can always derive Debug");
                ConstrainResult::Same
            }

            TypeKind::Array(t, len) => {
                if self.cannot_derive_debug.contains(&t) {
                    trace!("    arrays of T for which we cannot derive Debug \
                            also cannot derive Debug");
                    return self.insert(id);
                }

                if len <= RUST_DERIVE_IN_ARRAY_LIMIT {
                    trace!("    array is small enough to derive Debug");
                    ConstrainResult::Same
                } else {
                    trace!("    array is too large to derive Debug");
                    self.insert(id)
                }
            }

            TypeKind::ResolvedTypeRef(t) |
            TypeKind::TemplateAlias(t, _) |
            TypeKind::Alias(t) => {
                if self.cannot_derive_debug.contains(&t) {
                    trace!("    aliases and type refs to T which cannot derive \
                            Debug also cannot derive Debug");
                    self.insert(id)
                } else {
                    trace!("    aliases and type refs to T which can derive \
                            Debug can also derive Debug");
                    ConstrainResult::Same
                }
            }

            TypeKind::Comp(ref info) => {
                assert!(
                    !info.has_non_type_template_params(),
                    "The early ty.is_opaque check should have handled this case"
                );

                if info.kind() == CompKind::Union {
                    if self.ctx.options().unstable_rust {
                        trace!("    cannot derive Debug for Rust unions");
                        return self.insert(id);
                    }

                    if ty.layout(self.ctx)
                        .map_or(true,
                                |l| l.opaque().can_trivially_derive_debug(self.ctx, ())) {
                        trace!("    union layout can trivially derive Debug");
                        return ConstrainResult::Same;
                    } else {
                        trace!("    union layout cannot derive Debug");
                        return self.insert(id);
                    }
                }

                let bases_cannot_derive = info.base_members()
                    .iter()
                    .any(|base| self.cannot_derive_debug.contains(&base.ty));
                if bases_cannot_derive {
                    trace!("    base members cannot derive Debug, so we can't \
                            either");
                    return self.insert(id);
                }

                let fields_cannot_derive = info.fields()
                    .iter()
                    .any(|f| {
                        match *f {
                            Field::DataMember(ref data) => {
                                self.cannot_derive_debug.contains(&data.ty())
                            }
                            Field::Bitfields(ref bfu) => {
                                bfu.bitfields()
                                    .iter().any(|b| {
                                        self.cannot_derive_debug.contains(&b.ty())
                                    })
                            }
                        }
                    });
                if fields_cannot_derive {
                    trace!("    fields cannot derive Debug, so we can't either");
                    return self.insert(id);
                }

                trace!("    comp can derive Debug");
                ConstrainResult::Same
            }

            TypeKind::Pointer(inner) => {
                let inner_type = self.ctx.resolve_type(inner).canonical_type(self.ctx);
                if let TypeKind::Function(ref sig) = *inner_type.kind() {
                    if !sig.can_trivially_derive_debug(&self.ctx, ()) {
                        trace!("    function pointer that can't trivially derive Debug");
                        return self.insert(id);
                    }
                }
                trace!("    pointers can derive Debug");
                ConstrainResult::Same
            }

            TypeKind::TemplateInstantiation(ref template) => {
                let args_cannot_derive = template.template_arguments()
                    .iter()
                    .any(|arg| self.cannot_derive_debug.contains(&arg));
                if args_cannot_derive {
                    trace!("    template args cannot derive Debug, so \
                            insantiation can't either");
                    return self.insert(id);
                }

                assert!(
                    !template.template_definition().is_opaque(self.ctx, &()),
                    "The early ty.is_opaque check should have handled this case"
                );
                let def_cannot_derive = self.cannot_derive_debug
                    .contains(&template.template_definition());
                if def_cannot_derive {
                    trace!("    template definition cannot derive Debug, so \
                            insantiation can't either");
                    return self.insert(id);
                }

                trace!("    template instantiation can derive Debug");
                ConstrainResult::Same
            }

            TypeKind::Opaque => {
                unreachable!(
                    "The early ty.is_opaque check should have handled this case"
                )
            }
        }
    }

    fn each_depending_on<F>(&self, id: ItemId, mut f: F)
        where F: FnMut(ItemId),
    {
        if let Some(edges) = self.dependencies.get(&id) {
            for item in edges {
                trace!("enqueue {:?} into worklist", item);
                f(*item);
            }
        }
    }
}

impl<'ctx, 'gen> From<CannotDeriveDebug<'ctx, 'gen>> for HashSet<ItemId> {
    fn from(analysis: CannotDeriveDebug<'ctx, 'gen>) -> Self {
        analysis.cannot_derive_debug
    }
}
<|endoftext|>


<commit_before>#[macro_export]
macro_rules! async_hostcall_tests {
    ( $( $region_id:ident => $TestRegion:path ),* ) => {

        use lucet_runtime::{vmctx::Vmctx, lucet_hostcall};

        #[lucet_hostcall]
        #[no_mangle]
        pub fn hostcall_containing_block_on(vmctx: &Vmctx, value: u32) {
            let asynced_value = vmctx.block_on(async move { value });
            assert_eq!(asynced_value, value);
        }

        $(
            mod $region_id {
                use lucet_runtime::{DlModule, Error, Limits, Region, RegionCreate, TerminationDetails};
                use std::sync::Arc;
                use $TestRegion as TestRegion;
                use $crate::build::test_module_c;

                #[test]
                fn ensure_linked() {
                    lucet_runtime::lucet_internal_ensure_linked();
                }

                #[test]
                fn load_module() {
                    let _module =
                        test_module_c("async_hostcall", "hostcall_block_on.c").expect("build and load module");
                }

                #[test]
                fn hostcall_yield() {
                    /// Dummy function with the same type signature as
                    /// [`tokio::task::block_in_place`][tokio].
                    ///
                    /// [tokio]: https://docs.rs/tokio/0.2.21/tokio/task/fn.block_in_place.html
                    fn block_in_place<F, R>(f: F) -> R
                    where
                        F: FnOnce() -> R,
                    {
                        f()
                    }

                    let module = test_module_c("async_hostcall", "hostcall_block_on.c")
                        .expect("module compiled and loaded");
                    let region = <TestRegion as RegionCreate>::create(1, &Limits::default())
                        .expect("region can be created");
                    let mut inst = region
                        .new_instance(module)
                        .expect("instance can be created");

                    inst.run_start().expect("start section runs");

                    let correct_run_res =
                        futures_executor::block_on(
                            inst.run_async(
                                "main",
                                &[0u32.into(), 0i32.into()],
                                |f| block_in_place(f),
                            ));
                    match correct_run_res {
                        Ok(_) => {} // expected - UntypedRetVal is (), so no reason to inspect value
                        _ => panic!(
                            "run_async main should return successfully, got {:?}",
                            correct_run_res
                        ),
                    }

                    let incorrect_run_res = inst.run("main", &[0u32.into(), 0i32.into()]);
                    match incorrect_run_res {
                        Err(Error::RuntimeTerminated(term)) => {
                            assert_eq!(term, TerminationDetails::AwaitNeedsAsync);
                        }
                        _ => panic!(
                            "inst.run should fail because its not an async context, got {:?}",
                            incorrect_run_res
                        ),
                    }
                    inst.reset().expect("can reset instance");

                    let correct_run_res_2 =
                        futures_executor::block_on(
                            inst.run_async(
                                "main",
                                &[0u32.into(), 0i32.into()],
                                |f| block_in_place(f),
                            ));
                    match correct_run_res_2 {
                        Ok(_) => {} // expected
                        _ => panic!(
                            "second run_async main should return successfully, got {:?}",
                            correct_run_res_2
                        ),
                    }
                }

            }
        )*
    };
}
<commit_msg>tests: forgot to update name<commit_after>#[macro_export]
macro_rules! async_hostcall_tests {
    ( $( $region_id:ident => $TestRegion:path ),* ) => {

        use lucet_runtime::{vmctx::Vmctx, lucet_hostcall};

        #[lucet_hostcall]
        #[no_mangle]
        pub fn hostcall_containing_block_on(vmctx: &Vmctx, value: u32) {
            let asynced_value = vmctx.block_on(async move { value });
            assert_eq!(asynced_value, value);
        }

        $(
            mod $region_id {
                use lucet_runtime::{DlModule, Error, Limits, Region, RegionCreate, TerminationDetails};
                use std::sync::Arc;
                use $TestRegion as TestRegion;
                use $crate::build::test_module_c;

                #[test]
                fn ensure_linked() {
                    lucet_runtime::lucet_internal_ensure_linked();
                }

                #[test]
                fn load_module() {
                    let _module =
                        test_module_c("async_hostcall", "hostcall_block_on.c").expect("build and load module");
                }

                #[test]
                fn hostcall_yield() {
                    /// Dummy function with the same type signature as
                    /// [`tokio::task::block_in_place`][tokio].
                    ///
                    /// [tokio]: https://docs.rs/tokio/0.2.21/tokio/task/fn.block_in_place.html
                    fn block_in_place<F, R>(f: F) -> R
                    where
                        F: FnOnce() -> R,
                    {
                        f()
                    }

                    let module = test_module_c("async_hostcall", "hostcall_block_on.c")
                        .expect("module compiled and loaded");
                    let region = <TestRegion as RegionCreate>::create(1, &Limits::default())
                        .expect("region can be created");
                    let mut inst = region
                        .new_instance(module)
                        .expect("instance can be created");

                    inst.run_start().expect("start section runs");

                    let correct_run_res =
                        futures_executor::block_on(
                            inst.run_async(
                                "main",
                                &[0u32.into(), 0i32.into()],
                                |f| block_in_place(f),
                            ));
                    match correct_run_res {
                        Ok(_) => {} // expected - UntypedRetVal is (), so no reason to inspect value
                        _ => panic!(
                            "run_async main should return successfully, got {:?}",
                            correct_run_res
                        ),
                    }

                    let incorrect_run_res = inst.run("main", &[0u32.into(), 0i32.into()]);
                    match incorrect_run_res {
                        Err(Error::RuntimeTerminated(term)) => {
                            assert_eq!(term, TerminationDetails::BlockOnNeedsAsync);
                        }
                        _ => panic!(
                            "inst.run should fail because its not an async context, got {:?}",
                            incorrect_run_res
                        ),
                    }
                    inst.reset().expect("can reset instance");

                    let correct_run_res_2 =
                        futures_executor::block_on(
                            inst.run_async(
                                "main",
                                &[0u32.into(), 0i32.into()],
                                |f| block_in_place(f),
                            ));
                    match correct_run_res_2 {
                        Ok(_) => {} // expected
                        _ => panic!(
                            "second run_async main should return successfully, got {:?}",
                            correct_run_res_2
                        ),
                    }
                }

            }
        )*
    };
}
<|endoftext|>


<commit_before>//! Safe _handles_ to managed JavaScript memory.

pub(crate) mod internal;

use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::error::Error;
use std::fmt::{self, Debug, Display};
use neon_runtime;
use neon_runtime::raw;
use types::Value;
use context::Context;
use result::{JsResult, JsResultExt};
use self::internal::SuperType;

/// The trait of data that is managed by the JS garbage collector and can only be accessed via handles.
pub trait Managed: Copy {
    fn to_raw(self) -> raw::Local;

    fn from_raw(h: raw::Local) -> Self;
}

/// A safely rooted _handle_ to a JS value in memory that is managed by the garbage collector.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct Handle<'a, T: Managed + 'a> {
    value: T,
    phantom: PhantomData<&'a T>
}

impl<'a, T: Managed + 'a> PartialEq for Handle<'a, T> {
    fn eq(&self, other: &Self) -> bool {
        unsafe { neon_runtime::mem::same_handle(self.to_raw(), other.to_raw()) }
    }
}

impl<'a, T: Managed + 'a> Eq for Handle<'a, T> { }

impl<'a, T: Managed + 'a> Handle<'a, T> {
    pub(crate) fn new_internal(value: T) -> Handle<'a, T> {
        Handle {
            value: value,
            phantom: PhantomData
        }
    }
}

/// An error representing a failed downcast.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)]
pub struct DowncastError<F: Value, T: Value> {
    phantom_from: PhantomData<F>,
    phantom_to: PhantomData<T>,
    description: String
}

impl<F: Value, T: Value> Debug for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "DowncastError")
    }
}

impl<F: Value, T: Value> DowncastError<F, T> {
    fn new() -> Self {
        DowncastError {
            phantom_from: PhantomData,
            phantom_to: PhantomData,
            description: format!("failed downcast to {}", T::name())
        }
    }
}

impl<F: Value, T: Value> Display for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.description())
    }
}

impl<F: Value, T: Value> Error for DowncastError<F, T> {
    fn description(&self) -> &str {
        &self.description
    }
}

/// The result of a call to `Handle::downcast()`.
pub type DowncastResult<'a, F, T> = Result<Handle<'a, T>, DowncastError<F, T>>;

impl<'a, F: Value, T: Value> JsResultExt<'a, T> for DowncastResult<'a, F, T> {
    fn or_throw<'b, C: Context<'b>>(self, cx: &mut C) -> JsResult<'a, T> {
        match self {
            Ok(v) => Ok(v),
            Err(e) => cx.throw_type_error(&e.description)
        }
    }
}

impl<'a, T: Value> Handle<'a, T> {

    /// Safely upcast a handle to a supertype.
    ///
    /// This method does not require an execution context because it only copies a handle.
    pub fn upcast<U: Value + SuperType<T>>(&self) -> Handle<'a, U> {
        Handle::new_internal(SuperType::upcast_internal(self.value))
    }

    /// Tests whether this value is an instance of the given type.
    ///
    /// # Example:
    ///
    /// ```no_run
    /// # use neon::prelude::*;
    /// # fn my_neon_function(mut cx: FunctionContext) -> JsResult<JsUndefined> {
    /// let v: Handle<JsValue> = cx.number(17).upcast();
    /// v.is_a::<JsString>(); // false
    /// v.is_a::<JsNumber>(); // true
    /// v.is_a::<JsValue>();  // true
    /// # Ok(cx.undefined())
    /// # }
    /// ```
    pub fn is_a<U: Value>(&self) -> bool {
        U::is_typeof(self.value)
    }

    /// Attempts to downcast a handle to another type, which may fail. A failure
    /// to downcast **does not** throw a JavaScript exception, so it's OK to
    /// continue interacting with the JS engine if this method produces an `Err`
    /// result.
    pub fn downcast<U: Value>(&self) -> DowncastResult<'a, T, U> {
        match U::downcast(self.value) {
            Some(v) => Ok(Handle::new_internal(v)),
            None => Err(DowncastError::new())
        }
    }

    /// Attempts to downcast a handle to another type, raising a JavaScript `TypeError`
    /// exception on failure. This method is a convenient shorthand, equivalent to
    /// `self.downcast::<U>().or_throw::<C>(cx)`.
    pub fn downcast_or_throw<'b, U: Value, C: Context<'b>>(&self, cx: &mut C) -> JsResult<'a, U> {
        self.downcast().or_throw(cx)
    }

}

impl<'a, T: Managed> Deref for Handle<'a, T> {
    type Target = T;
    fn deref<'b>(&'b self) -> &'b T {
        &self.value
    }
}

impl<'a, T: Managed> DerefMut for Handle<'a, T> {
    fn deref_mut<'b>(&'b mut self) -> &'b mut T {
        &mut self.value
    }
}
<commit_msg>Fix #503<commit_after>//! Safe _handles_ to managed JavaScript memory.

pub(crate) mod internal;

use std::marker::PhantomData;
use std::ops::{Deref, DerefMut};
use std::error::Error;
use std::fmt::{self, Debug, Display};
use neon_runtime;
use neon_runtime::raw;
use types::Value;
use context::Context;
use result::{JsResult, JsResultExt};
use self::internal::SuperType;

/// The trait of data that is managed by the JS garbage collector and can only be accessed via handles.
pub trait Managed: Copy {
    fn to_raw(self) -> raw::Local;

    fn from_raw(h: raw::Local) -> Self;
}

/// A safely rooted _handle_ to a JS value in memory that is managed by the garbage collector.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct Handle<'a, T: Managed + 'a> {
    value: T,
    phantom: PhantomData<&'a T>
}

impl<'a, T: Managed + 'a> PartialEq for Handle<'a, T> {
    fn eq(&self, other: &Self) -> bool {
        unsafe { neon_runtime::mem::same_handle(self.to_raw(), other.to_raw()) }
    }
}

impl<'a, T: Managed + 'a> Eq for Handle<'a, T> { }

impl<'a, T: Managed + 'a> Handle<'a, T> {
    pub(crate) fn new_internal(value: T) -> Handle<'a, T> {
        Handle {
            value: value,
            phantom: PhantomData
        }
    }
}

/// An error representing a failed downcast.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)]
pub struct DowncastError<F: Value, T: Value> {
    phantom_from: PhantomData<F>,
    phantom_to: PhantomData<T>,
    description: String
}

impl<F: Value, T: Value> Debug for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "DowncastError")
    }
}

impl<F: Value, T: Value> DowncastError<F, T> {
    fn new() -> Self {
        DowncastError {
            phantom_from: PhantomData,
            phantom_to: PhantomData,
            description: format!("failed downcast to {}", T::name())
        }
    }
}

impl<F: Value, T: Value> Display for DowncastError<F, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{}", self.to_string())
    }
}

impl<F: Value, T: Value> Error for DowncastError<F, T> {
    fn description(&self) -> &str {
        &self.description
    }
}

/// The result of a call to `Handle::downcast()`.
pub type DowncastResult<'a, F, T> = Result<Handle<'a, T>, DowncastError<F, T>>;

impl<'a, F: Value, T: Value> JsResultExt<'a, T> for DowncastResult<'a, F, T> {
    fn or_throw<'b, C: Context<'b>>(self, cx: &mut C) -> JsResult<'a, T> {
        match self {
            Ok(v) => Ok(v),
            Err(e) => cx.throw_type_error(&e.description)
        }
    }
}

impl<'a, T: Value> Handle<'a, T> {

    /// Safely upcast a handle to a supertype.
    ///
    /// This method does not require an execution context because it only copies a handle.
    pub fn upcast<U: Value + SuperType<T>>(&self) -> Handle<'a, U> {
        Handle::new_internal(SuperType::upcast_internal(self.value))
    }

    /// Tests whether this value is an instance of the given type.
    ///
    /// # Example:
    ///
    /// ```no_run
    /// # use neon::prelude::*;
    /// # fn my_neon_function(mut cx: FunctionContext) -> JsResult<JsUndefined> {
    /// let v: Handle<JsValue> = cx.number(17).upcast();
    /// v.is_a::<JsString>(); // false
    /// v.is_a::<JsNumber>(); // true
    /// v.is_a::<JsValue>();  // true
    /// # Ok(cx.undefined())
    /// # }
    /// ```
    pub fn is_a<U: Value>(&self) -> bool {
        U::is_typeof(self.value)
    }

    /// Attempts to downcast a handle to another type, which may fail. A failure
    /// to downcast **does not** throw a JavaScript exception, so it's OK to
    /// continue interacting with the JS engine if this method produces an `Err`
    /// result.
    pub fn downcast<U: Value>(&self) -> DowncastResult<'a, T, U> {
        match U::downcast(self.value) {
            Some(v) => Ok(Handle::new_internal(v)),
            None => Err(DowncastError::new())
        }
    }

    /// Attempts to downcast a handle to another type, raising a JavaScript `TypeError`
    /// exception on failure. This method is a convenient shorthand, equivalent to
    /// `self.downcast::<U>().or_throw::<C>(cx)`.
    pub fn downcast_or_throw<'b, U: Value, C: Context<'b>>(&self, cx: &mut C) -> JsResult<'a, U> {
        self.downcast().or_throw(cx)
    }

}

impl<'a, T: Managed> Deref for Handle<'a, T> {
    type Target = T;
    fn deref<'b>(&'b self) -> &'b T {
        &self.value
    }
}

impl<'a, T: Managed> DerefMut for Handle<'a, T> {
    fn deref_mut<'b>(&'b mut self) -> &'b mut T {
        &mut self.value
    }
}
<|endoftext|>


<commit_before>use std::ops::{Mul, AddAssign};
use std::cmp;
use std::f32;
use std::fmt::Debug;

use na;
use num::{zero, Zero};

use ::Point2i;
use ::Point2f;
use ::lerp;
use blockedarray::BlockedArray;
use spectrum::Spectrum;

pub enum WrapMode {
    Repeat,
    Black,
    Clamp,
}

pub struct MIPMap<T> {
    do_trilinear: bool,
    max_anisotropy: f32,
    wrap_mode: WrapMode,
    resolution: Point2i,
    pyramid: Vec<BlockedArray<T>>,
    black: T,
}

impl<T> MIPMap<T>
    where T: Zero,
          T: Clone,
          T: Copy,
          T: Clampable,
          T: Debug,
          T: AddAssign<T>,
          T: Mul<f32, Output = T>
{
    pub fn new(res: &Point2i,
               img: &[T],
               do_trilinear: bool,
               max_anisotropy: f32,
               wrap_mode: WrapMode)
               -> MIPMap<T> {
        info!("Creating MIPMap for texture");
        let mut resolution = *res;
        let mut resampled_image = Vec::new();
        if !res.x.is_power_of_two() || !res.y.is_power_of_two() {
            info!("Texture dimensions are not powers of 2: re-sampling.");
            // resample image to power of two resolution
            let res_pow2 = Point2i::new(res.x.next_power_of_two(), res.y.next_power_of_two());
            // resample image in s direction
            resampled_image.resize(res_pow2.x as usize * res_pow2.y as usize, zero());
            let s_weights = MIPMap::<T>::resample_weights(res.x as usize, res_pow2.x as usize);
            // apply s_weights to zoom in s direction
            for t in 0..res.y as usize {
                for s in 0..res_pow2.x as usize {
                    // Compute texel (s,t) in s-zoomed image
                    resampled_image[t * res_pow2.x as usize + s] = zero();
                    for j in 0..4 {
                        let mut orig_s = s_weights[s].first_texel as usize + j;
                        orig_s = match wrap_mode {
                            WrapMode::Repeat => orig_s % res.x as usize,
                            WrapMode::Clamp => na::clamp(orig_s, 0, res.x as usize - 1),
                            WrapMode::Black => orig_s,
                        };
                        if orig_s < res.x as usize {
                            resampled_image[t * res_pow2.x as usize + s] +=
                                img[(t * res.x as usize + orig_s) as usize] *
                                s_weights[s].weights[j];
                        }
                    }

                }
            }
            // TODO use rayon to parallelize this loop?
            // resample image in t direction
            let t_weights = MIPMap::<T>::resample_weights(res.y as usize, res_pow2.y as usize);
            // apply t_weights to zoom in t direction
            for s in 0..res_pow2.x as usize {
                let mut work_data: Vec<T> = vec![zero(); res_pow2.y as usize];
                for t in 0..res_pow2.y as usize {
                    // work_data[t] = zero();
                    // Compute texel (s,t) in t-zoomed image
                    for j in 0..4 {
                        let mut offset = t_weights[t].first_texel as usize + j;
                        offset = match wrap_mode {
                            WrapMode::Repeat => offset % res.y as usize,
                            WrapMode::Clamp => na::clamp(offset, 0, res.y as usize - 1),
                            WrapMode::Black => offset,
                        };
                        if offset < res.y as usize {
                            work_data[t] += img[(offset * res_pow2.x as usize + s) as usize] *
                                            t_weights[t].weights[j];
                        }
                    }
                }
                for t in 0..res_pow2.y as usize {
                    resampled_image[t * res_pow2.x as usize + s] = work_data[t].clamp(0.0, 1.0);
                }
            }
            resolution = res_pow2;
        }

        let mut mipmap = MIPMap {
            do_trilinear: do_trilinear,
            max_anisotropy: max_anisotropy,
            wrap_mode: wrap_mode,
            resolution: resolution,
            pyramid: Vec::new(),
            black: zero(),
        };

        // initialize levels of MIPMap for image
        let n_levels = 1 + f32::log2(cmp::max(resolution.x, resolution.y) as f32).log2() as usize;
        // Initialize most detailed level of the pyramid
        let img_data = if resampled_image.is_empty() {
            img
        } else {
            &resampled_image[..]
        };
        // level 0
        mipmap.pyramid
            .push(BlockedArray::new_from(resolution.x as usize, resolution.y as usize, img_data));
        for i in 1..n_levels {
            // initialize ith level of the pyramid
            let s_res = cmp::max(1, mipmap.pyramid[i - 1].u_size() / 2);
            let t_res = cmp::max(1, mipmap.pyramid[i - 1].v_size() / 2);
            let mut ba = BlockedArray::new(s_res, t_res);
            // Filter 4 texels from finer level of pyramid
            for t in 0..t_res {
                for s in 0..s_res {
                    let (si, ti) = (s as isize, t as isize);
                    ba[(s, t)] = (*mipmap.texel(i - 1, 2 * si, 2 * ti) +
                                  *mipmap.texel(i - 1, 2 * si + 1, 2 * ti) +
                                  *mipmap.texel(i - 1, 2 * si, 2 * ti + 1) +
                                  *mipmap.texel(i - 1, 2 * si + 1, 2 * ti + 1)) *
                                 0.25;
                    debug!("l={}, ba[({}, {})]={:?}", i, s, t, ba[(s, t)]);
                }
            }
            mipmap.pyramid.push(ba);
        }
        // initialize EWA filter weights if needed

        mipmap
    }

    pub fn width(&self) -> usize {
        self.resolution.x as usize
    }

    pub fn height(&self) -> usize {
        self.resolution.y as usize
    }

    pub fn levels(&self) -> usize {
        self.pyramid.len()
    }

    pub fn texel(&self, level: usize, s: isize, t: isize) -> &T {
        let l = &self.pyramid[level];
        let (u_size, v_size) = (l.u_size() as isize, l.v_size() as isize);
        let (ss, tt): (usize, usize) = match self.wrap_mode {
            WrapMode::Repeat => (modulo(s, u_size), modulo(t, v_size)),
            WrapMode::Clamp => {
                (na::clamp(s, 0, u_size - 1) as usize, na::clamp(t, 0, v_size - 1) as usize)
            }
            WrapMode::Black => {
                if s < 0 || s >= u_size || t < 0 || t >= v_size {
                    return &self.black;
                }
                (s as usize, t as usize)
            }
        };
        &l[(ss, tt)]
    }

    pub fn lookup(&self, st: &Point2f, width: f32) -> T {
        // Compute MIPMap-level for trilinear filtering
        let level = self.levels() as f32 - 1.0 + width.max(1e-8).log2();
        // Perform trilinear interpolation at appropriate MIPMap level
        if level < 0.0 {
            self.triangle(0, st)
        } else if level >= self.levels() as f32 - 1.0 {
            *self.texel(self.levels() - 1, 0, 0)
        } else {
            let i_level = level.floor();
            let delta = level - i_level;
            lerp(delta,
                 self.triangle(i_level as usize, st),
                 self.triangle(i_level as usize + 1, st))
        }
    }

    pub fn triangle(&self, level: usize, st: &Point2f) -> T {
        let level = na::clamp(level, 0, self.levels() - 1);
        let s = st.x * self.pyramid[level].u_size() as f32 - 0.5;
        let t = st.y * self.pyramid[level].v_size() as f32 - 0.5;
        let s0 = s.floor() as isize;
        let t0 = t.floor() as isize;
        let ds = s - s0 as f32;
        let dt = t - t0 as f32;
        debug!("st={:?}, s={}, t={}, s0={}, t0={}, ds={}, dt={}",
               st,
               s,
               t,
               s0,
               t0,
               ds,
               dt);

        *self.texel(level, s0, t0) * (1.0 - ds) * (1.0 - dt) +
        *self.texel(level, s0, t0 + 1) * (1.0 - ds) * dt +
        *self.texel(level, s0 + 1, t0) * ds * (1.0 - dt) +
        *self.texel(level, s0 + 1, t0 + 1) * ds * dt
    }

    fn resample_weights(old_res: usize, new_res: usize) -> Vec<ResampleWeight> {
        assert!(new_res >= old_res);
        let mut wt = Vec::with_capacity(new_res);
        let filter_width = 2.0;
        let mut w = [0.0; 4];
        for i in 0..new_res {
            // compute image resampling weights for ith texel
            let center = (i as f32 + 0.5) * (old_res as f32 / new_res as f32);
            let first_texel = ((center - filter_width) + 0.5).floor() as u32;
            for j in 0..4 {
                let pos = first_texel as f32 + j as f32 + 0.5;
                w[j] = Self::lanczos((pos - center) / filter_width);
            }
            // Normalize filter weights for texel resampling
            let inv_sum_weights = 1.0 / (w[0] + w[1] + w[2] + w[3]);
            for j in 0..4 {
                w[j] *= inv_sum_weights;
                assert!(w[j] <= 1.0);
            }
            wt.push(ResampleWeight {
                first_texel: first_texel,
                weights: w,
            });
        }

        wt
    }

    fn lanczos(f: f32) -> f32 {
        let tau = 2.0;
        let mut x = f.abs();
        if x < 1e-5 {
            return 1.0;
        };
        if x > 1.0 {
            return 0.0;
        };
        x *= f32::consts::PI;
        let s = (x * tau).sin() / (x * tau);
        let lanczos = x.sin() / x;
        s * lanczos
    }
}

struct ResampleWeight {
    pub first_texel: u32,
    pub weights: [f32; 4],
}

pub trait Clampable {
    fn clamp(self, min: f32, max: f32) -> Self;
}

impl Clampable for f32 {
    fn clamp(self, min: f32, max: f32) -> f32 {
        na::clamp(self, min, max)
    }
}

impl Clampable for Spectrum {
    fn clamp(self, min: f32, max: f32) -> Spectrum {
        Spectrum::rgb(self.r.clamp(min, max),
                      self.g.clamp(min, max),
                      self.b.clamp(min, max))
    }
}

fn modulo(a: isize, b: isize) -> usize {
    let result = a % b;
    if result < 0 {
        (result + b) as usize
    } else {
        result as usize
    }
}
<commit_msg>Ooops, I was taking the log2 twice...<commit_after>use std::ops::{Mul, AddAssign};
use std::cmp;
use std::f32;
use std::fmt::Debug;

use na;
use num::{zero, Zero};

use ::Point2i;
use ::Point2f;
use ::lerp;
use blockedarray::BlockedArray;
use spectrum::Spectrum;

pub enum WrapMode {
    Repeat,
    Black,
    Clamp,
}

pub struct MIPMap<T> {
    do_trilinear: bool,
    max_anisotropy: f32,
    wrap_mode: WrapMode,
    resolution: Point2i,
    pyramid: Vec<BlockedArray<T>>,
    black: T,
}

impl<T> MIPMap<T>
    where T: Zero,
          T: Clone,
          T: Copy,
          T: Clampable,
          T: Debug,
          T: AddAssign<T>,
          T: Mul<f32, Output = T>
{
    pub fn new(res: &Point2i,
               img: &[T],
               do_trilinear: bool,
               max_anisotropy: f32,
               wrap_mode: WrapMode)
               -> MIPMap<T> {
        info!("Creating MIPMap for texture");
        let mut resolution = *res;
        let mut resampled_image = Vec::new();
        if !res.x.is_power_of_two() || !res.y.is_power_of_two() {
            info!("Texture dimensions are not powers of 2: re-sampling.");
            // resample image to power of two resolution
            let res_pow2 = Point2i::new(res.x.next_power_of_two(), res.y.next_power_of_two());
            // resample image in s direction
            resampled_image.resize(res_pow2.x as usize * res_pow2.y as usize, zero());
            let s_weights = MIPMap::<T>::resample_weights(res.x as usize, res_pow2.x as usize);
            // apply s_weights to zoom in s direction
            for t in 0..res.y as usize {
                for s in 0..res_pow2.x as usize {
                    // Compute texel (s,t) in s-zoomed image
                    resampled_image[t * res_pow2.x as usize + s] = zero();
                    for j in 0..4 {
                        let mut orig_s = s_weights[s].first_texel as usize + j;
                        orig_s = match wrap_mode {
                            WrapMode::Repeat => orig_s % res.x as usize,
                            WrapMode::Clamp => na::clamp(orig_s, 0, res.x as usize - 1),
                            WrapMode::Black => orig_s,
                        };
                        if orig_s < res.x as usize {
                            resampled_image[t * res_pow2.x as usize + s] +=
                                img[(t * res.x as usize + orig_s) as usize] *
                                s_weights[s].weights[j];
                        }
                    }

                }
            }
            // TODO use rayon to parallelize this loop?
            // resample image in t direction
            let t_weights = MIPMap::<T>::resample_weights(res.y as usize, res_pow2.y as usize);
            // apply t_weights to zoom in t direction
            for s in 0..res_pow2.x as usize {
                let mut work_data: Vec<T> = vec![zero(); res_pow2.y as usize];
                for t in 0..res_pow2.y as usize {
                    // work_data[t] = zero();
                    // Compute texel (s,t) in t-zoomed image
                    for j in 0..4 {
                        let mut offset = t_weights[t].first_texel as usize + j;
                        offset = match wrap_mode {
                            WrapMode::Repeat => offset % res.y as usize,
                            WrapMode::Clamp => na::clamp(offset, 0, res.y as usize - 1),
                            WrapMode::Black => offset,
                        };
                        if offset < res.y as usize {
                            work_data[t] += img[(offset * res_pow2.x as usize + s) as usize] *
                                            t_weights[t].weights[j];
                        }
                    }
                }
                for t in 0..res_pow2.y as usize {
                    resampled_image[t * res_pow2.x as usize + s] = work_data[t].clamp(0.0, 1.0);
                }
            }
            resolution = res_pow2;
        }

        let mut mipmap = MIPMap {
            do_trilinear: do_trilinear,
            max_anisotropy: max_anisotropy,
            wrap_mode: wrap_mode,
            resolution: resolution,
            pyramid: Vec::new(),
            black: zero(),
        };

        // initialize levels of MIPMap for image
        let n_levels = 1 + (cmp::max(resolution.x, resolution.y) as f32).log2() as usize;
        info!("mipmap will have {} levels", n_levels);
        // Initialize most detailed level of the pyramid
        let img_data = if resampled_image.is_empty() {
            img
        } else {
            &resampled_image[..]
        };
        // level 0
        mipmap.pyramid
            .push(BlockedArray::new_from(resolution.x as usize, resolution.y as usize, img_data));
        for i in 1..n_levels {
            // initialize ith level of the pyramid
            let s_res = cmp::max(1, mipmap.pyramid[i - 1].u_size() / 2);
            let t_res = cmp::max(1, mipmap.pyramid[i - 1].v_size() / 2);
            let mut ba = BlockedArray::new(s_res, t_res);
            // Filter 4 texels from finer level of pyramid
            for t in 0..t_res {
                for s in 0..s_res {
                    let (si, ti) = (s as isize, t as isize);
                    ba[(s, t)] = (*mipmap.texel(i - 1, 2 * si, 2 * ti) +
                                  *mipmap.texel(i - 1, 2 * si + 1, 2 * ti) +
                                  *mipmap.texel(i - 1, 2 * si, 2 * ti + 1) +
                                  *mipmap.texel(i - 1, 2 * si + 1, 2 * ti + 1)) *
                                 0.25;
                    debug!("l={}, ba[({}, {})]={:?}", i, s, t, ba[(s, t)]);
                }
            }
            mipmap.pyramid.push(ba);
        }
        // initialize EWA filter weights if needed

        mipmap
    }

    pub fn width(&self) -> usize {
        self.resolution.x as usize
    }

    pub fn height(&self) -> usize {
        self.resolution.y as usize
    }

    pub fn levels(&self) -> usize {
        self.pyramid.len()
    }

    pub fn texel(&self, level: usize, s: isize, t: isize) -> &T {
        let l = &self.pyramid[level];
        let (u_size, v_size) = (l.u_size() as isize, l.v_size() as isize);
        let (ss, tt): (usize, usize) = match self.wrap_mode {
            WrapMode::Repeat => (modulo(s, u_size), modulo(t, v_size)),
            WrapMode::Clamp => {
                (na::clamp(s, 0, u_size - 1) as usize, na::clamp(t, 0, v_size - 1) as usize)
            }
            WrapMode::Black => {
                if s < 0 || s >= u_size || t < 0 || t >= v_size {
                    return &self.black;
                }
                (s as usize, t as usize)
            }
        };
        &l[(ss, tt)]
    }

    pub fn lookup(&self, st: &Point2f, width: f32) -> T {
        // Compute MIPMap-level for trilinear filtering
        let level = self.levels() as f32 - 1.0 + width.max(1e-8).log2();
        // Perform trilinear interpolation at appropriate MIPMap level
        if level < 0.0 {
            self.triangle(0, st)
        } else if level >= self.levels() as f32 - 1.0 {
            *self.texel(self.levels() - 1, 0, 0)
        } else {
            let i_level = level.floor();
            let delta = level - i_level;
            lerp(delta,
                 self.triangle(i_level as usize, st),
                 self.triangle(i_level as usize + 1, st))
        }
    }

    pub fn triangle(&self, level: usize, st: &Point2f) -> T {
        let level = na::clamp(level, 0, self.levels() - 1);
        let s = st.x * self.pyramid[level].u_size() as f32 - 0.5;
        let t = st.y * self.pyramid[level].v_size() as f32 - 0.5;
        let s0 = s.floor() as isize;
        let t0 = t.floor() as isize;
        let ds = s - s0 as f32;
        let dt = t - t0 as f32;
        debug!("st={:?}, s={}, t={}, s0={}, t0={}, ds={}, dt={}",
               st,
               s,
               t,
               s0,
               t0,
               ds,
               dt);

        *self.texel(level, s0, t0) * (1.0 - ds) * (1.0 - dt) +
        *self.texel(level, s0, t0 + 1) * (1.0 - ds) * dt +
        *self.texel(level, s0 + 1, t0) * ds * (1.0 - dt) +
        *self.texel(level, s0 + 1, t0 + 1) * ds * dt
    }

    fn resample_weights(old_res: usize, new_res: usize) -> Vec<ResampleWeight> {
        assert!(new_res >= old_res);
        let mut wt = Vec::with_capacity(new_res);
        let filter_width = 2.0;
        let mut w = [0.0; 4];
        for i in 0..new_res {
            // compute image resampling weights for ith texel
            let center = (i as f32 + 0.5) * (old_res as f32 / new_res as f32);
            let first_texel = ((center - filter_width) + 0.5).floor() as u32;
            for j in 0..4 {
                let pos = first_texel as f32 + j as f32 + 0.5;
                w[j] = Self::lanczos((pos - center) / filter_width);
            }
            // Normalize filter weights for texel resampling
            let inv_sum_weights = 1.0 / (w[0] + w[1] + w[2] + w[3]);
            for j in 0..4 {
                w[j] *= inv_sum_weights;
                assert!(w[j] <= 1.0);
            }
            wt.push(ResampleWeight {
                first_texel: first_texel,
                weights: w,
            });
        }

        wt
    }

    fn lanczos(f: f32) -> f32 {
        let tau = 2.0;
        let mut x = f.abs();
        if x < 1e-5 {
            return 1.0;
        };
        if x > 1.0 {
            return 0.0;
        };
        x *= f32::consts::PI;
        let s = (x * tau).sin() / (x * tau);
        let lanczos = x.sin() / x;
        s * lanczos
    }
}

struct ResampleWeight {
    pub first_texel: u32,
    pub weights: [f32; 4],
}

pub trait Clampable {
    fn clamp(self, min: f32, max: f32) -> Self;
}

impl Clampable for f32 {
    fn clamp(self, min: f32, max: f32) -> f32 {
        na::clamp(self, min, max)
    }
}

impl Clampable for Spectrum {
    fn clamp(self, min: f32, max: f32) -> Spectrum {
        Spectrum::rgb(self.r.clamp(min, max),
                      self.g.clamp(min, max),
                      self.b.clamp(min, max))
    }
}

fn modulo(a: isize, b: isize) -> usize {
    let result = a % b;
    if result < 0 {
        (result + b) as usize
    } else {
        result as usize
    }
}
<|endoftext|>