<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

mod convert;

pub use convert::{FromLossy, IntoLossy};

use log::error;

pub fn convert<T, U>(item: T) -> U
where
    T: IntoLossy<U>,
{
    item.into_lossy()
}

include!(concat!(env!("OUT_DIR"), "/grr.rs"));

impl From<bool> for DataBlob {

    fn from(value: bool) -> DataBlob {
        DataBlob {
            boolean: Some(value),
            ..Default::default()
        }
    }
}

impl From<i64> for DataBlob {

    fn from(value: i64) -> DataBlob {
        DataBlob {
            integer: Some(value),
            ..Default::default()
        }
    }
}

impl From<f32> for DataBlob {

    fn from(value: f32) -> DataBlob {
        DataBlob {
            float: Some(value),
            ..Default::default()
        }
    }
}

impl From<Vec<u8>> for DataBlob {

    fn from(value: Vec<u8>) -> DataBlob {
        DataBlob {
            data: Some(value),
            ..Default::default()
        }
    }
}

impl From<String> for DataBlob {

    fn from(value: String) -> DataBlob  {
        DataBlob {
            string: Some(value),
            ..Default::default()
        }
    }
}

impl KeyValue {

    /// Creates an empty key-value.
    ///
    /// Both the key and the value are going to be equal to `None`.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::empty();
    /// assert_eq!(entry.k, None);
    /// assert_eq!(entry.v, None);
    /// ```
    pub fn empty() -> KeyValue {
        KeyValue {
            k: None,
            v: None,
        }
    }

    /// Creates a key-value pair.
    ///
    /// Both the key and the value are going to be equal to the given values.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::pair(String::from("foo"), 42i64);
    /// assert_eq!(entry.k.unwrap().string, Some(String::from("foo")));
    /// assert_eq!(entry.v.unwrap().integer, Some(42));
    /// ```
    pub fn pair<K, V>(key: K, value: V) -> KeyValue
    where
        K: Into<DataBlob>,
        V: Into<DataBlob>,
    {
        KeyValue {
            k: Some(key.into()),
            v: Some(value.into()),
        }
    }

    /// Creates a key-only key-value.
    ///
    /// The key is going to be equal to the given value and the key will be
    /// `None`.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::key(String::from("quux"));
    /// assert_eq!(entry.k.unwrap().string, Some(String::from("quux")));
    /// assert_eq!(entry.v, None);
    /// ```
    pub fn key<K>(key: K) -> KeyValue
    where
        K: Into<DataBlob>,
    {
        KeyValue {
            k: Some(key.into()),
            v: None,
        }
    }
}

impl std::iter::FromIterator<KeyValue> for AttributedDict {

    fn from_iter<I>(iter: I) -> AttributedDict
    where
        I: IntoIterator<Item = KeyValue>,
    {
        AttributedDict {
            dat: iter.into_iter().collect(),
        }
    }
}

impl<K, V> std::iter::FromIterator<(K, V)> for AttributedDict
where
    K: Into<DataBlob>,
    V: Into<DataBlob>,
{
    fn from_iter<I>(iter: I) -> AttributedDict
    where
        I: IntoIterator<Item = (K, V)>,
    {
        let pair = |(key, value)| KeyValue::pair(key, value);
        iter.into_iter().map(pair).collect()
    }
}

impl FromLossy<std::fs::Metadata> for StatEntry {

    fn from_lossy(metadata: std::fs::Metadata) -> StatEntry {
        use std::convert::{TryFrom, TryInto};
        #[cfg(target_family = "unix")]
        use std::os::unix::fs::MetadataExt;

        // TODO: Fix definition of `StatEntry`.
        // `StatEntry` defines insufficient integer width for some fields. For
        // now we just ignore errors, but the definition should be improved.
        let some = |value: u64| Some(value.try_into().unwrap_or(0));

        let atime_micros = match metadata.accessed().ok().map(micros) {
            Some(Ok(atime_micros)) => Some(atime_micros),
            Some(Err(err)) => {
                error!("failed to convert access time: {}", err);
                None
            },
            None => None,
        };

        let mtime_micros = match metadata.modified().ok().map(micros) {
            Some(Ok(mtime_micros)) => Some(mtime_micros),
            Some(Err(err)) => {
                error!("failed to convert modification time: {}", err);
                None
            },
            None => None,
        };

        #[cfg(target_family = "unix")]
        let ctime_micros = match u64::try_from(metadata.ctime_nsec()) {
            Ok(ctime_nanos) => Some(ctime_nanos / 1000u64),
            Err(err) => {
                error!("negative inode change time: {}", err);
                None
            },
        };

        let crtime_micros = match metadata.created().ok().map(micros) {
            Some(Ok(crtime_micros)) => Some(crtime_micros),
            Some(Err(err)) => {
                error!("failed to convert creation time: {}", err);
                None
            },
            None => None,
        };

        StatEntry {
            #[cfg(target_family = "unix")]
            st_mode: Some(metadata.mode().into()),
            #[cfg(target_family = "unix")]
            st_ino: some(metadata.ino()),
            #[cfg(target_family = "unix")]
            st_dev: some(metadata.dev()),
            #[cfg(target_family = "unix")]
            st_rdev: some(metadata.rdev()),
            #[cfg(target_family = "unix")]
            st_nlink: some(metadata.nlink()),
            #[cfg(target_family = "unix")]
            st_uid: Some(metadata.uid()),
            #[cfg(target_family = "unix")]
            st_gid: Some(metadata.gid()),
            st_size: Some(metadata.len()),
            st_atime: atime_micros,
            st_mtime: mtime_micros,
            #[cfg(target_family = "unix")]
            st_ctime: ctime_micros,
            st_crtime: crtime_micros,
            #[cfg(target_family = "unix")]
            st_blocks: some(metadata.blocks()),
            #[cfg(target_family = "unix")]
            st_blksize: some(metadata.blksize()),
            ..Default::default()
        }
    }
}

/// An error type for failures of converting timestamps to microseconds.
#[derive(Clone, Debug)]
pub enum MicrosError {
    /// Attempted to convert pre-epoch system time.
    Epoch(std::time::SystemTimeError),
    /// Attempted to convert a value outside of 64-bit unsigned integer range.
    Overflow(std::num::TryFromIntError),
}

impl std::fmt::Display for MicrosError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        use MicrosError::*;

        match *self {
            Epoch(ref error) => {
                write!(fmt, "pre-epoch system time: {}", error)
            }
            Overflow(ref error) => {
                write!(fmt, "system time value too big: {}", error)
            }
        }
    }
}

impl std::error::Error for MicrosError {

    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        use MicrosError::*;

        match *self {
            Epoch(ref error) => Some(error),
            Overflow(ref error) => Some(error),
        }
    }
}

impl From<std::time::SystemTimeError> for MicrosError {

    fn from(error: std::time::SystemTimeError) -> MicrosError {
        MicrosError::Epoch(error)
    }
}

impl From<std::num::TryFromIntError> for MicrosError {

    fn from(error: std::num::TryFromIntError) -> MicrosError {
        MicrosError::Overflow(error)
    }
}

/// An error type for failures of converting timestamps to seconds.
#[derive(Clone, Debug)]
pub struct SecsError {
    error: MicrosError,
}

impl std::fmt::Display for SecsError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        self.error.fmt(fmt)
    }
}

impl From<MicrosError> for SecsError {

    fn from(error: MicrosError) -> SecsError {
        SecsError {
            error: error,
        }
    }
}

impl From<SecsError> for MicrosError {

    fn from(error: SecsError) -> MicrosError {
        error.error
    }
}

/// Converts system time into epoch microseconds.
///
/// Because most GRR messages use epoch microseconds for representing timestamps
/// this function be be useful to convert from more idiomatic representations.
///
/// # Examples
///
/// ```
/// use rrg_proto::micros;
///
/// assert_eq!(micros(std::time::UNIX_EPOCH).unwrap(), 0);
/// ```
pub fn micros(time: std::time::SystemTime) -> Result<u64, MicrosError> {
    let time_micros = time.duration_since(std::time::UNIX_EPOCH)?.as_micros();
    Ok(std::convert::TryInto::try_into(time_micros)?)
}

/// Converts system time into epoch seconds.
///
/// Some GRR messages use epoch seconds for representing timestamps. In such
/// cases this function can be useful to convert from more idiomatic types.
///
/// # Examples
///
/// ```
/// use rrg_proto::secs;
///
/// assert_eq!(secs(std::time::UNIX_EPOCH).unwrap(), 0);
/// ```
pub fn secs(time: std::time::SystemTime) -> Result<u64, SecsError> {
    Ok(micros(time)?)
}
<commit_msg>Fix file metadata conversion to use seconds.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

mod convert;

pub use convert::{FromLossy, IntoLossy};

use log::error;

pub fn convert<T, U>(item: T) -> U
where
    T: IntoLossy<U>,
{
    item.into_lossy()
}

include!(concat!(env!("OUT_DIR"), "/grr.rs"));

impl From<bool> for DataBlob {

    fn from(value: bool) -> DataBlob {
        DataBlob {
            boolean: Some(value),
            ..Default::default()
        }
    }
}

impl From<i64> for DataBlob {

    fn from(value: i64) -> DataBlob {
        DataBlob {
            integer: Some(value),
            ..Default::default()
        }
    }
}

impl From<f32> for DataBlob {

    fn from(value: f32) -> DataBlob {
        DataBlob {
            float: Some(value),
            ..Default::default()
        }
    }
}

impl From<Vec<u8>> for DataBlob {

    fn from(value: Vec<u8>) -> DataBlob {
        DataBlob {
            data: Some(value),
            ..Default::default()
        }
    }
}

impl From<String> for DataBlob {

    fn from(value: String) -> DataBlob  {
        DataBlob {
            string: Some(value),
            ..Default::default()
        }
    }
}

impl KeyValue {

    /// Creates an empty key-value.
    ///
    /// Both the key and the value are going to be equal to `None`.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::empty();
    /// assert_eq!(entry.k, None);
    /// assert_eq!(entry.v, None);
    /// ```
    pub fn empty() -> KeyValue {
        KeyValue {
            k: None,
            v: None,
        }
    }

    /// Creates a key-value pair.
    ///
    /// Both the key and the value are going to be equal to the given values.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::pair(String::from("foo"), 42i64);
    /// assert_eq!(entry.k.unwrap().string, Some(String::from("foo")));
    /// assert_eq!(entry.v.unwrap().integer, Some(42));
    /// ```
    pub fn pair<K, V>(key: K, value: V) -> KeyValue
    where
        K: Into<DataBlob>,
        V: Into<DataBlob>,
    {
        KeyValue {
            k: Some(key.into()),
            v: Some(value.into()),
        }
    }

    /// Creates a key-only key-value.
    ///
    /// The key is going to be equal to the given value and the key will be
    /// `None`.
    ///
    /// # Examples
    ///
    /// ```
    /// use rrg_proto::KeyValue;
    ///
    /// let entry = KeyValue::key(String::from("quux"));
    /// assert_eq!(entry.k.unwrap().string, Some(String::from("quux")));
    /// assert_eq!(entry.v, None);
    /// ```
    pub fn key<K>(key: K) -> KeyValue
    where
        K: Into<DataBlob>,
    {
        KeyValue {
            k: Some(key.into()),
            v: None,
        }
    }
}

impl std::iter::FromIterator<KeyValue> for AttributedDict {

    fn from_iter<I>(iter: I) -> AttributedDict
    where
        I: IntoIterator<Item = KeyValue>,
    {
        AttributedDict {
            dat: iter.into_iter().collect(),
        }
    }
}

impl<K, V> std::iter::FromIterator<(K, V)> for AttributedDict
where
    K: Into<DataBlob>,
    V: Into<DataBlob>,
{
    fn from_iter<I>(iter: I) -> AttributedDict
    where
        I: IntoIterator<Item = (K, V)>,
    {
        let pair = |(key, value)| KeyValue::pair(key, value);
        iter.into_iter().map(pair).collect()
    }
}

impl FromLossy<std::fs::Metadata> for StatEntry {

    fn from_lossy(metadata: std::fs::Metadata) -> StatEntry {
        use std::convert::{TryFrom, TryInto};
        #[cfg(target_family = "unix")]
        use std::os::unix::fs::MetadataExt;

        // TODO: Fix definition of `StatEntry`.
        // `StatEntry` defines insufficient integer width for some fields. For
        // now we just ignore errors, but the definition should be improved.
        let some = |value: u64| Some(value.try_into().unwrap_or(0));

        let atime_secs = match metadata.accessed().ok().map(secs) {
            Some(Ok(atime_secs)) => Some(atime_secs),
            Some(Err(err)) => {
                error!("failed to convert access time: {}", err);
                None
            },
            None => None,
        };

        let mtime_secs = match metadata.modified().ok().map(secs) {
            Some(Ok(mtime_secs)) => Some(mtime_secs),
            Some(Err(err)) => {
                error!("failed to convert modification time: {}", err);
                None
            },
            None => None,
        };

        #[cfg(target_family = "unix")]
        let ctime_secs = match u64::try_from(metadata.ctime()) {
            Ok(ctime_secs) => Some(ctime_secs),
            Err(err) => {
                error!("negative inode change time: {}", err);
                None
            },
        };

        let crtime_secs = match metadata.created().ok().map(secs) {
            Some(Ok(crtime_secs)) => Some(crtime_secs),
            Some(Err(err)) => {
                error!("failed to convert creation time: {}", err);
                None
            },
            None => None,
        };

        StatEntry {
            #[cfg(target_family = "unix")]
            st_mode: Some(metadata.mode().into()),
            #[cfg(target_family = "unix")]
            st_ino: some(metadata.ino()),
            #[cfg(target_family = "unix")]
            st_dev: some(metadata.dev()),
            #[cfg(target_family = "unix")]
            st_rdev: some(metadata.rdev()),
            #[cfg(target_family = "unix")]
            st_nlink: some(metadata.nlink()),
            #[cfg(target_family = "unix")]
            st_uid: Some(metadata.uid()),
            #[cfg(target_family = "unix")]
            st_gid: Some(metadata.gid()),
            st_size: Some(metadata.len()),
            st_atime: atime_secs,
            st_mtime: mtime_secs,
            #[cfg(target_family = "unix")]
            st_ctime: ctime_secs,
            st_crtime: crtime_secs,
            #[cfg(target_family = "unix")]
            st_blocks: some(metadata.blocks()),
            #[cfg(target_family = "unix")]
            st_blksize: some(metadata.blksize()),
            ..Default::default()
        }
    }
}

/// An error type for failures of converting timestamps to microseconds.
#[derive(Clone, Debug)]
pub enum MicrosError {
    /// Attempted to convert pre-epoch system time.
    Epoch(std::time::SystemTimeError),
    /// Attempted to convert a value outside of 64-bit unsigned integer range.
    Overflow(std::num::TryFromIntError),
}

impl std::fmt::Display for MicrosError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        use MicrosError::*;

        match *self {
            Epoch(ref error) => {
                write!(fmt, "pre-epoch system time: {}", error)
            }
            Overflow(ref error) => {
                write!(fmt, "system time value too big: {}", error)
            }
        }
    }
}

impl std::error::Error for MicrosError {

    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        use MicrosError::*;

        match *self {
            Epoch(ref error) => Some(error),
            Overflow(ref error) => Some(error),
        }
    }
}

impl From<std::time::SystemTimeError> for MicrosError {

    fn from(error: std::time::SystemTimeError) -> MicrosError {
        MicrosError::Epoch(error)
    }
}

impl From<std::num::TryFromIntError> for MicrosError {

    fn from(error: std::num::TryFromIntError) -> MicrosError {
        MicrosError::Overflow(error)
    }
}

/// An error type for failures of converting timestamps to seconds.
#[derive(Clone, Debug)]
pub struct SecsError {
    error: MicrosError,
}

impl std::fmt::Display for SecsError {

    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        self.error.fmt(fmt)
    }
}

impl From<MicrosError> for SecsError {

    fn from(error: MicrosError) -> SecsError {
        SecsError {
            error: error,
        }
    }
}

impl From<SecsError> for MicrosError {

    fn from(error: SecsError) -> MicrosError {
        error.error
    }
}

/// Converts system time into epoch microseconds.
///
/// Because most GRR messages use epoch microseconds for representing timestamps
/// this function be be useful to convert from more idiomatic representations.
///
/// # Examples
///
/// ```
/// use rrg_proto::micros;
///
/// assert_eq!(micros(std::time::UNIX_EPOCH).unwrap(), 0);
/// ```
pub fn micros(time: std::time::SystemTime) -> Result<u64, MicrosError> {
    let time_micros = time.duration_since(std::time::UNIX_EPOCH)?.as_micros();
    Ok(std::convert::TryInto::try_into(time_micros)?)
}

/// Converts system time into epoch seconds.
///
/// Some GRR messages use epoch seconds for representing timestamps. In such
/// cases this function can be useful to convert from more idiomatic types.
///
/// # Examples
///
/// ```
/// use rrg_proto::secs;
///
/// assert_eq!(secs(std::time::UNIX_EPOCH).unwrap(), 0);
/// ```
pub fn secs(time: std::time::SystemTime) -> Result<u64, SecsError> {
    Ok(micros(time)?)
}
<|endoftext|>


<commit_before>//! This will build the proc macro in `imp`, and copy the resulting dylib artifact into the
//! `OUT_DIR`.
//!
//! `proc-macro-test` itself contains only a path to that artifact.

use std::{
    env, fs,
    path::{Path, PathBuf},
    process::Command,
};

use cargo_metadata::Message;

fn main() {
    println!("cargo:rerun-if-changed=imp");

    let out_dir = env::var_os("OUT_DIR").unwrap();
    let out_dir = Path::new(&out_dir);

    let name = "proc-macro-test-impl";
    let version = "0.0.0";

    let imp_dir = std::env::current_dir().unwrap().join("imp");

    let staging_dir = out_dir.join("proc-macro-test-imp-staging");
    // this'll error out if the staging dir didn't previously. using
    // `std::fs::exists` would suffer from TOCTOU so just do our best to
    // wipe it and ignore errors.
    let _ = std::fs::remove_dir_all(&staging_dir);

    println!("Creating {}", staging_dir.display());
    std::fs::create_dir_all(&staging_dir).unwrap();

    let src_dir = staging_dir.join("src");
    println!("Creating {}", src_dir.display());
    std::fs::create_dir_all(src_dir).unwrap();

    for item_els in [&["Cargo.toml"][..], &["src", "lib.rs"]] {
        let mut src = imp_dir.clone();
        let mut dst = staging_dir.clone();
        for el in item_els {
            src.push(el);
            dst.push(el);
        }
        println!("Copying {} to {}", src.display(), dst.display());
        std::fs::copy(src, dst).unwrap();
    }

    let target_dir = out_dir.join("target");
    let output = Command::new(toolchain::cargo())
        .current_dir(&staging_dir)
        .args(&["build", "-p", "proc-macro-test-impl", "--message-format", "json"])
        // Explicit override the target directory to avoid using the same one which the parent
        // cargo is using, or we'll deadlock.
        // This can happen when `CARGO_TARGET_DIR` is set or global config forces all cargo
        // instance to use the same target directory.
        .arg("--target-dir")
        .arg(&target_dir)
        .output()
        .unwrap();
    if !output.status.success() {
        println!("proc-macro-test-impl failed to build");
        println!("============ stdout ============");
        println!("{}", String::from_utf8_lossy(&output.stdout));
        println!("============ stderr ============");
        println!("{}", String::from_utf8_lossy(&output.stderr));
        panic!("proc-macro-test-impl failed to build");
    }

    let mut artifact_path = None;
    for message in Message::parse_stream(output.stdout.as_slice()) {
        match message.unwrap() {
            Message::CompilerArtifact(artifact) => {
                if artifact.target.kind.contains(&"proc-macro".to_string()) {
                    let repr = format!("{} {}", name, version);
                    if artifact.package_id.repr.starts_with(&repr) {
                        artifact_path = Some(PathBuf::from(&artifact.filenames[0]));
                    }
                }
            }
            _ => (), // Unknown message
        }
    }

    // This file is under `target_dir` and is already under `OUT_DIR`.
    let artifact_path = artifact_path.expect("no dylib for proc-macro-test-impl found");

    let info_path = out_dir.join("proc_macro_test_location.txt");
    fs::write(info_path, artifact_path.to_str().unwrap()).unwrap();
}
<commit_msg>Update crates/proc-macro-test/build.rs<commit_after>//! This will build the proc macro in `imp`, and copy the resulting dylib artifact into the
//! `OUT_DIR`.
//!
//! `proc-macro-test` itself contains only a path to that artifact.

use std::{
    env, fs,
    path::{Path, PathBuf},
    process::Command,
};

use cargo_metadata::Message;

fn main() {
    println!("cargo:rerun-if-changed=imp");

    let out_dir = env::var_os("OUT_DIR").unwrap();
    let out_dir = Path::new(&out_dir);

    let name = "proc-macro-test-impl";
    let version = "0.0.0";

    let imp_dir = std::env::current_dir().unwrap().join("imp");

    let staging_dir = out_dir.join("proc-macro-test-imp-staging");
    // this'll error out if the staging dir didn't previously exist. using
    // `std::fs::exists` would suffer from TOCTOU so just do our best to
    // wipe it and ignore errors.
    let _ = std::fs::remove_dir_all(&staging_dir);

    println!("Creating {}", staging_dir.display());
    std::fs::create_dir_all(&staging_dir).unwrap();

    let src_dir = staging_dir.join("src");
    println!("Creating {}", src_dir.display());
    std::fs::create_dir_all(src_dir).unwrap();

    for item_els in [&["Cargo.toml"][..], &["src", "lib.rs"]] {
        let mut src = imp_dir.clone();
        let mut dst = staging_dir.clone();
        for el in item_els {
            src.push(el);
            dst.push(el);
        }
        println!("Copying {} to {}", src.display(), dst.display());
        std::fs::copy(src, dst).unwrap();
    }

    let target_dir = out_dir.join("target");
    let output = Command::new(toolchain::cargo())
        .current_dir(&staging_dir)
        .args(&["build", "-p", "proc-macro-test-impl", "--message-format", "json"])
        // Explicit override the target directory to avoid using the same one which the parent
        // cargo is using, or we'll deadlock.
        // This can happen when `CARGO_TARGET_DIR` is set or global config forces all cargo
        // instance to use the same target directory.
        .arg("--target-dir")
        .arg(&target_dir)
        .output()
        .unwrap();
    if !output.status.success() {
        println!("proc-macro-test-impl failed to build");
        println!("============ stdout ============");
        println!("{}", String::from_utf8_lossy(&output.stdout));
        println!("============ stderr ============");
        println!("{}", String::from_utf8_lossy(&output.stderr));
        panic!("proc-macro-test-impl failed to build");
    }

    let mut artifact_path = None;
    for message in Message::parse_stream(output.stdout.as_slice()) {
        match message.unwrap() {
            Message::CompilerArtifact(artifact) => {
                if artifact.target.kind.contains(&"proc-macro".to_string()) {
                    let repr = format!("{} {}", name, version);
                    if artifact.package_id.repr.starts_with(&repr) {
                        artifact_path = Some(PathBuf::from(&artifact.filenames[0]));
                    }
                }
            }
            _ => (), // Unknown message
        }
    }

    // This file is under `target_dir` and is already under `OUT_DIR`.
    let artifact_path = artifact_path.expect("no dylib for proc-macro-test-impl found");

    let info_path = out_dir.join("proc_macro_test_location.txt");
    fs::write(info_path, artifact_path.to_str().unwrap()).unwrap();
}
<|endoftext|>


<commit_before>use std::io::net::ip::{Ipv4Addr, SocketAddr};

mod libtorresmo {
    extern crate time;

    use std::io::net::udp::UdpSocket;
    use std::io::net::ip::SocketAddr;
    use std::io::IoResult;
    use std::mem::{to_be32, to_be16, transmute};
    use std::rand::random;

    #[allow(dead_code,non_camel_case_types)]
    enum UtpPacketType {
        ST_DATA  = 0,
        ST_FIN   = 1,
        ST_STATE = 2,
        ST_RESET = 3,
        ST_SYN   = 4,
    }

    #[allow(dead_code)]
    struct UtpPacketHeader {
        type_ver: u8, // type: u4, ver: u4
        extension: u8,
        connection_id: u16,
        timestamp_microseconds: u32,
        timestamp_difference_microseconds: u32,
        wnd_size: u32,
        seq_nr: u16,
        ack_nr: u16,
    }

    impl UtpPacketHeader {
        fn bytes(&self) -> &[u8] {
            unsafe {
                let buf: &[u8, ..20] = transmute(self);
                return buf.as_slice();
            }
        }

        fn len(&self) -> uint {
            static HEADER_SIZE: uint = 20;
            return HEADER_SIZE;
        }
    }

    #[allow(dead_code)]
    struct UtpPacket {
        header: UtpPacketHeader,
        payload: Vec<u8>,
    }

    impl UtpPacket {
        /// Constructs a new, empty UtpPacket.
        fn new() -> UtpPacket {
            UtpPacket {
                header: UtpPacketHeader {
                    type_ver: ST_DATA as u8 << 4 | 1,
                    extension: 0,
                    connection_id: 0,
                    timestamp_microseconds: 0,
                    timestamp_difference_microseconds: 0,
                    wnd_size: 0,
                    seq_nr: 0,
                    ack_nr: 0,
                },
                payload: Vec::new(),
            }
        }

        fn bytes(&self) -> Vec<u8> {
            let mut buf: Vec<u8> = Vec::with_capacity(self.len());
            buf.push_all(self.header.bytes());
            buf.push_all(self.payload.as_slice());
            return buf;
        }

        fn len(&self) -> uint {
            self.header.len() + self.payload.len()
        }
    }

    pub struct UtpSocket {
        socket: UdpSocket,
        seq_nr: u16,
    }

    impl UtpSocket {
        pub fn bind(addr: SocketAddr) -> IoResult<UtpSocket> {
            let skt = UdpSocket::bind(addr);
            match skt {
                Ok(x)  => Ok(UtpSocket { socket: x, seq_nr: 1 }),
                Err(e) => Err(e)
            }
        }

        pub fn sendto(&mut self, buf: &[u8], dst: SocketAddr) -> IoResult<()> {
            let mut packet = UtpPacket::new();
            packet.payload = Vec::from_slice(buf);
            packet.header.seq_nr = to_be16(self.seq_nr);
            packet.header.connection_id = to_be16(random());
            let t = time::get_time();
            packet.header.timestamp_microseconds = to_be32((t.sec * 1_000_000) as u32 + (t.nsec/1000) as u32);

            // Send packet
            let result = self.socket.sendto(packet.bytes().as_slice(), dst);

            self.seq_nr += 1;
            result
        }

        pub fn recvfrom(&mut self, buf: &mut[u8]) -> IoResult<(uint,SocketAddr)> {
            self.socket.recvfrom(buf)
        }
    }

    impl Clone for UtpSocket {
        fn clone(&self) -> UtpSocket {
            UtpSocket {
                socket: self.socket.clone(),
                seq_nr: self.seq_nr,
            }
        }
    }
}

fn usage() {
    println!("Usage: torresmo [-s|-c] <address> <port>");
}

fn main() {
    use libtorresmo::{UtpSocket};
    use std::from_str::FromStr;

    // Defaults
    let mut addr = SocketAddr { ip: Ipv4Addr(127,0,0,1), port: 8080 };

    let args = std::os::args();

    if args.len() == 4 {
        let ip = FromStr::from_str(args.get(2).as_slice()).unwrap();
        let port = FromStr::from_str(args.get(3).as_slice()).unwrap();
        addr = SocketAddr { ip: ip, port: port };
    }

    match args.get(1).as_slice() {
        "-s" => {
            let mut buf = [0, ..512];
            let sock = UtpSocket::bind(addr).unwrap();
            println!("Serving on {}", addr);

            loop {
                let mut sock = sock.clone();
                match sock.recvfrom(buf) {
                    Ok((_, src)) => {
                        spawn(proc() {
                            let mut sock = sock;
                            let payload = String::from_str("Hello\n").into_bytes();

                            // Send uTP packet
                            let _ = sock.sendto(payload.as_slice(), src);
                        })
                    }
                    Err(_) => {}
                }
            }
        }
        "-c" => {
            let mut sock = UtpSocket::bind(SocketAddr { ip: Ipv4Addr(127,0,0,1), port: std::rand::random() }).unwrap();
            let _ = sock.sendto([0], addr);
            drop(sock);
        }
        _ => usage(),
    }
}
<commit_msg>Implement UtpStream.<commit_after>use std::io::net::ip::{Ipv4Addr, SocketAddr};

mod libtorresmo {
    extern crate time;

    use std::io::net::udp::UdpSocket;
    use std::io::net::ip::SocketAddr;
    use std::io::IoResult;
    use std::mem::{to_be32, to_be16, transmute};
    use std::rand::random;

    #[allow(dead_code,non_camel_case_types)]
    enum UtpPacketType {
        ST_DATA  = 0,
        ST_FIN   = 1,
        ST_STATE = 2,
        ST_RESET = 3,
        ST_SYN   = 4,
    }

    #[allow(dead_code)]
    struct UtpPacketHeader {
        type_ver: u8, // type: u4, ver: u4
        extension: u8,
        connection_id: u16,
        timestamp_microseconds: u32,
        timestamp_difference_microseconds: u32,
        wnd_size: u32,
        seq_nr: u16,
        ack_nr: u16,
    }

    impl UtpPacketHeader {
        fn bytes(&self) -> &[u8] {
            unsafe {
                let buf: &[u8, ..20] = transmute(self);
                return buf.as_slice();
            }
        }

        fn len(&self) -> uint {
            static HEADER_SIZE: uint = 20;
            return HEADER_SIZE;
        }
    }

    #[allow(dead_code)]
    struct UtpPacket {
        header: UtpPacketHeader,
        payload: Vec<u8>,
    }

    impl UtpPacket {
        /// Constructs a new, empty UtpPacket.
        fn new() -> UtpPacket {
            UtpPacket {
                header: UtpPacketHeader {
                    type_ver: ST_DATA as u8 << 4 | 1,
                    extension: 0,
                    connection_id: 0,
                    timestamp_microseconds: 0,
                    timestamp_difference_microseconds: 0,
                    wnd_size: 0,
                    seq_nr: 0,
                    ack_nr: 0,
                },
                payload: Vec::new(),
            }
        }

        fn bytes(&self) -> Vec<u8> {
            let mut buf: Vec<u8> = Vec::with_capacity(self.len());
            buf.push_all(self.header.bytes());
            buf.push_all(self.payload.as_slice());
            return buf;
        }

        fn len(&self) -> uint {
            self.header.len() + self.payload.len()
        }
    }

    pub struct UtpSocket {
        socket: UdpSocket,
        seq_nr: u16,
    }

    impl UtpSocket {
        pub fn bind(addr: SocketAddr) -> IoResult<UtpSocket> {
            let skt = UdpSocket::bind(addr);
            match skt {
                Ok(x)  => Ok(UtpSocket { socket: x, seq_nr: 1 }),
                Err(e) => Err(e)
            }
        }

        pub fn sendto(&mut self, buf: &[u8], dst: SocketAddr) -> IoResult<()> {
            let mut packet = UtpPacket::new();
            packet.payload = Vec::from_slice(buf);
            packet.header.seq_nr = to_be16(self.seq_nr);
            packet.header.connection_id = to_be16(random());
            let t = time::get_time();
            packet.header.timestamp_microseconds = to_be32((t.sec * 1_000_000) as u32 + (t.nsec/1000) as u32);

            // Send packet
            let result = self.socket.sendto(packet.bytes().as_slice(), dst);

            self.seq_nr += 1;
            result
        }

        pub fn recvfrom(&mut self, buf: &mut[u8]) -> IoResult<(uint,SocketAddr)> {
            self.socket.recvfrom(buf)
        }
    }

    impl Clone for UtpSocket {
        fn clone(&self) -> UtpSocket {
            UtpSocket {
                socket: self.socket.clone(),
                seq_nr: self.seq_nr,
            }
        }
    }

    #[allow(dead_code)]
    struct UtpStream {
        socket: UdpSocket,
        connected_to: SocketAddr,
        seq_nr: u16,
        ack_nr: u16,
    }

    #[allow(dead_code)]
    impl UtpStream {
        fn new(socket: UdpSocket, conn: SocketAddr) -> UtpStream {
            UtpStream {
                socket: socket,
                connected_to: conn,
                seq_nr: 0,
                ack_nr: 0,
            }
        }

        fn disconnect(self) -> UdpSocket {
            self.socket
        }
    }

    impl Reader for UtpStream {
        fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
            match self.socket.recvfrom(buf) {
                Ok((_nread, src)) if src != self.connected_to => Ok(0),
                Ok((nread, _src)) => Ok(nread),
                Err(e) => Err(e),
            }
        }
    }
    impl Writer for UtpStream {
        fn write(&mut self, buf: &[u8]) -> IoResult<()> {
            self.socket.sendto(buf, self.connected_to)
        }
    }
}

fn usage() {
    println!("Usage: torresmo [-s|-c] <address> <port>");
}

fn main() {
    use libtorresmo::{UtpSocket};
    use std::from_str::FromStr;

    // Defaults
    let mut addr = SocketAddr { ip: Ipv4Addr(127,0,0,1), port: 8080 };

    let args = std::os::args();

    if args.len() == 4 {
        let ip = FromStr::from_str(args.get(2).as_slice()).unwrap();
        let port = FromStr::from_str(args.get(3).as_slice()).unwrap();
        addr = SocketAddr { ip: ip, port: port };
    }

    match args.get(1).as_slice() {
        "-s" => {
            let mut buf = [0, ..512];
            let sock = UtpSocket::bind(addr).unwrap();
            println!("Serving on {}", addr);

            loop {
                let mut sock = sock.clone();
                match sock.recvfrom(buf) {
                    Ok((_, src)) => {
                        spawn(proc() {
                            let mut sock = sock;
                            let payload = String::from_str("Hello\n").into_bytes();

                            // Send uTP packet
                            let _ = sock.sendto(payload.as_slice(), src);
                        })
                    }
                    Err(_) => {}
                }
            }
        }
        "-c" => {
            let mut sock = UtpSocket::bind(SocketAddr { ip: Ipv4Addr(127,0,0,1), port: std::rand::random() }).unwrap();
            let _ = sock.sendto([0], addr);
            drop(sock);
        }
        _ => usage(),
    }
}
<|endoftext|>


<commit_before>//! Toplevel handle for a loaded SVG document.
//!
//! This module provides the primitives on which the public APIs are implemented.

use crate::allowed_url::{AllowedUrl, Href};
use crate::bbox::BoundingBox;
use crate::css::{Origin, Stylesheet};
use crate::document::{AcquiredNodes, Document};
use crate::dpi::Dpi;
use crate::drawing_ctx::{draw_tree, DrawingMode, ViewParams};
use crate::error::{DefsLookupErrorKind, LoadingError, RenderingError};
use crate::length::*;
use crate::node::{CascadedValues, Node, NodeBorrow};
use crate::parsers::Parse;
use crate::rect::Rect;
use crate::structure::IntrinsicDimensions;
use url::Url;

/// Loading options for SVG documents.
#[derive(Clone)]
pub struct LoadOptions {
    /// Base URL; all relative references will be resolved with respect to this.
    pub base_url: Option<Url>,

    /// Whether to turn off size limits in libxml2.
    pub unlimited_size: bool,

    /// Whether to keep original (undecoded) image data to embed in Cairo PDF surfaces.
    pub keep_image_data: bool,
}

impl LoadOptions {
    /// Creates a `LoadOptions` with defaults, and sets the `base_url`.
    pub fn new(base_url: Option<Url>) -> Self {
        LoadOptions {
            base_url,
            unlimited_size: false,
            keep_image_data: false,
        }
    }

    /// Sets whether libxml2's limits on memory usage should be turned off.
    ///
    /// This should only be done for trusted data.
    pub fn with_unlimited_size(mut self, unlimited: bool) -> Self {
        self.unlimited_size = unlimited;
        self
    }

    /// Sets whether to keep the original compressed image data from referenced JPEG/PNG images.
    ///
    /// This is only useful for rendering to Cairo PDF
    /// surfaces, which can embed the original, compressed image data instead of uncompressed
    /// RGB buffers.
    pub fn keep_image_data(mut self, keep: bool) -> Self {
        self.keep_image_data = keep;
        self
    }

    /// Creates a new `LoadOptions` with a different `base_url`.
    ///
    /// This is used when loading a referenced file that may in turn cause other files
    /// to be loaded, for example `<image xlink:href="subimage.svg"/>`
    pub fn copy_with_base_url(&self, base_url: &AllowedUrl) -> Self {
        LoadOptions {
            base_url: Some((**base_url).clone()),
            unlimited_size: self.unlimited_size,
            keep_image_data: self.keep_image_data,
        }
    }
}

/// Main handle to an SVG document.
///
/// This is the main object in librsvg.  It gets created with the [`from_stream`] method
/// and then provides access to all the primitives needed to implement the public APIs.
///
/// [`from_stream`]: #method.from_stream
pub struct Handle {
    document: Document,
}

impl Handle {
    /// Loads an SVG document into a `Handle`.
    pub fn from_stream(
        load_options: &LoadOptions,
        stream: &gio::InputStream,
        cancellable: Option<&gio::Cancellable>,
    ) -> Result<Handle, LoadingError> {
        Ok(Handle {
            document: Document::load_from_stream(load_options, stream, cancellable)?,
        })
    }

    /// Queries whether a document has a certain element `#foo`.
    ///
    /// The `id` must be an URL fragment identifier, i.e. something
    /// like `#element_id`.
    pub fn has_sub(&self, id: &str) -> Result<bool, RenderingError> {
        match self.lookup_node(id) {
            Ok(_) => Ok(true),

            Err(DefsLookupErrorKind::NotFound) => Ok(false),

            Err(e) => Err(RenderingError::InvalidId(e)),
        }
    }

    /// Returns (ink_rect, logical_rect)
    pub fn get_geometry_sub(
        &self,
        id: Option<&str>,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(Rect, Rect), RenderingError> {
        let node = self.get_node_or_root(id)?;
        let root = self.document.root();
        let is_root = node == root;

        if is_root {
            let cascaded = CascadedValues::new_from_node(&node);

            if let Some((w, h)) = get_svg_size(&self.get_intrinsic_dimensions(), &cascaded, dpi) {
                let rect = Rect::from_size(w, h);
                return Ok((rect, rect));
            }
        }

        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let bbox = draw_tree(
            DrawingMode::LimitToStack { node, root },
            &cr,
            unit_rectangle(),
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        Ok((ink_rect, logical_rect))
    }

    fn get_node_or_root(&self, id: Option<&str>) -> Result<Node, RenderingError> {
        if let Some(id) = id {
            self.lookup_node(id).map_err(RenderingError::InvalidId)
        } else {
            Ok(self.document.root())
        }
    }

    pub fn get_geometry_for_layer(
        &self,
        id: Option<&str>,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(cairo::Rectangle, cairo::Rectangle), RenderingError> {
        let node = self.get_node_or_root(id)?;
        let root = self.document.root();

        let viewport = Rect::from(*viewport);

        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let bbox = draw_tree(
            DrawingMode::LimitToStack { node, root },
            &cr,
            viewport,
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        Ok((
            cairo::Rectangle::from(ink_rect),
            cairo::Rectangle::from(logical_rect),
        ))
    }

    fn lookup_node(&self, id: &str) -> Result<Node, DefsLookupErrorKind> {
        match Href::parse(&id).map_err(DefsLookupErrorKind::HrefError)? {
            Href::PlainUrl(_) => Err(DefsLookupErrorKind::CannotLookupExternalReferences),
            Href::WithFragment(fragment) => {
                if let Some(uri) = fragment.uri() {
                    // The public APIs to get geometries of individual elements, or to render
                    // them, should only allow referencing elements within the main handle's
                    // SVG file; that is, only plain "#foo" fragment IDs are allowed here.
                    // Otherwise, a calling program could request "another-file#foo" and cause
                    // another-file to be loaded, even if it is not part of the set of
                    // resources that the main SVG actually references.  In the future we may
                    // relax this requirement to allow lookups within that set, but not to
                    // other random files.

                    let msg = format!(
                        "the public API is not allowed to look up external references: {}#{}",
                        uri,
                        fragment.fragment()
                    );

                    rsvg_log!("{}", msg);

                    return Err(DefsLookupErrorKind::CannotLookupExternalReferences);
                }

                match self.document.lookup_node_by_id(fragment.fragment()) {
                    Some(n) => Ok(n),
                    None => Err(DefsLookupErrorKind::NotFound),
                }
            }
        }
    }

    pub fn render_document(
        &self,
        cr: &cairo::Context,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        self.render_layer(cr, None, viewport, dpi, is_testing)
    }

    pub fn render_layer(
        &self,
        cr: &cairo::Context,
        id: Option<&str>,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        check_cairo_context(cr)?;

        let root = self.document.root();

        let mode = if let Some(id) = id {
            let node = self.lookup_node(id).map_err(RenderingError::InvalidId)?;

            DrawingMode::LimitToStack { node, root }
        } else {
            DrawingMode::OnlyNode(root)
        };

        cr.save();

        let res = draw_tree(
            mode,
            cr,
            Rect::from(*viewport),
            dpi,
            false,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        );

        cr.restore();

        res.map(|_bbox| ())
    }

    fn get_bbox_for_element(
        &self,
        node: &Node,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<BoundingBox, RenderingError> {
        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let node = node.clone();

        draw_tree(
            DrawingMode::OnlyNode(node),
            &cr,
            unit_rectangle(),
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )
    }

    /// Returns (ink_rect, logical_rect)
    pub fn get_geometry_for_element(
        &self,
        id: Option<&str>,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(cairo::Rectangle, cairo::Rectangle), RenderingError> {
        let node = self.get_node_or_root(id)?;

        let bbox = self.get_bbox_for_element(&node, dpi, is_testing)?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        // Translate so ink_rect is always at offset (0, 0)
        let ofs = (-ink_rect.x0, -ink_rect.y0);

        Ok((
            cairo::Rectangle::from(ink_rect.translate(ofs)),
            cairo::Rectangle::from(logical_rect.translate(ofs)),
        ))
    }

    pub fn render_element(
        &self,
        cr: &cairo::Context,
        id: Option<&str>,
        element_viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        check_cairo_context(cr)?;

        let node = self.get_node_or_root(id)?;

        let bbox = self.get_bbox_for_element(&node, dpi, is_testing)?;

        if bbox.ink_rect.is_none() || bbox.rect.is_none() {
            // Nothing to draw
            return Ok(());
        }

        let ink_r = bbox.ink_rect.unwrap_or_default();

        if ink_r.is_empty() {
            return Ok(());
        }

        // Render, transforming so element is at the new viewport's origin

        cr.save();

        let factor =
            (element_viewport.width / ink_r.width()).min(element_viewport.height / ink_r.height());

        cr.translate(element_viewport.x, element_viewport.y);
        cr.scale(factor, factor);
        cr.translate(-ink_r.x0, -ink_r.y0);

        let res = draw_tree(
            DrawingMode::OnlyNode(node),
            &cr,
            unit_rectangle(),
            dpi,
            false,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        );

        cr.restore();

        res.map(|_bbox| ())
    }

    pub fn get_intrinsic_dimensions(&self) -> IntrinsicDimensions {
        borrow_element_as!(self.document.root(), Svg).get_intrinsic_dimensions()
    }

    pub fn set_stylesheet(&mut self, css: &str) -> Result<(), LoadingError> {
        let mut stylesheet = Stylesheet::new(Origin::User);
        stylesheet.parse(css, None)?;
        self.document.cascade(&[stylesheet]);
        Ok(())
    }
}

fn check_cairo_context(cr: &cairo::Context) -> Result<(), RenderingError> {
    let status = cr.status();
    if status == cairo::Status::Success {
        Ok(())
    } else {
        Err(RenderingError::Cairo(status))
    }
}

fn unit_rectangle() -> Rect {
    Rect::from_size(1.0, 1.0)
}

/// Returns the SVG's size suitable for the legacy C API, or None
/// if it must be computed by hand.
///
/// The legacy C API can compute an SVG document's size from the
/// `width`, `height`, and `viewBox` attributes of the toplevel `<svg>`
/// element.  If these are not available, then the size must be computed
/// by actually measuring the geometries of elements in the document.
fn get_svg_size(
    dimensions: &IntrinsicDimensions,
    cascaded: &CascadedValues,
    dpi: Dpi,
) -> Option<(f64, f64)> {
    let values = cascaded.get();

    // these defaults are per the spec
    let w = dimensions
        .width
        .unwrap_or_else(|| Length::<Horizontal>::parse_str("100%").unwrap());
    let h = dimensions
        .height
        .unwrap_or_else(|| Length::<Vertical>::parse_str("100%").unwrap());

    match (w, h, dimensions.vbox) {
        (w, h, Some(vbox)) => {
            let params = ViewParams::new(dpi, vbox.width(), vbox.height());

            Some((w.normalize(values, &params), h.normalize(values, &params)))
        }

        (w, h, None) if w.unit != LengthUnit::Percent && h.unit != LengthUnit::Percent => {
            let params = ViewParams::new(dpi, 0.0, 0.0);

            Some((w.normalize(values, &params), h.normalize(values, &params)))
        }
        (_, _, _) => None,
    }
}
<commit_msg>render_layer: make consistent with get_geometry_for_layer<commit_after>//! Toplevel handle for a loaded SVG document.
//!
//! This module provides the primitives on which the public APIs are implemented.

use crate::allowed_url::{AllowedUrl, Href};
use crate::bbox::BoundingBox;
use crate::css::{Origin, Stylesheet};
use crate::document::{AcquiredNodes, Document};
use crate::dpi::Dpi;
use crate::drawing_ctx::{draw_tree, DrawingMode, ViewParams};
use crate::error::{DefsLookupErrorKind, LoadingError, RenderingError};
use crate::length::*;
use crate::node::{CascadedValues, Node, NodeBorrow};
use crate::parsers::Parse;
use crate::rect::Rect;
use crate::structure::IntrinsicDimensions;
use url::Url;

/// Loading options for SVG documents.
#[derive(Clone)]
pub struct LoadOptions {
    /// Base URL; all relative references will be resolved with respect to this.
    pub base_url: Option<Url>,

    /// Whether to turn off size limits in libxml2.
    pub unlimited_size: bool,

    /// Whether to keep original (undecoded) image data to embed in Cairo PDF surfaces.
    pub keep_image_data: bool,
}

impl LoadOptions {
    /// Creates a `LoadOptions` with defaults, and sets the `base_url`.
    pub fn new(base_url: Option<Url>) -> Self {
        LoadOptions {
            base_url,
            unlimited_size: false,
            keep_image_data: false,
        }
    }

    /// Sets whether libxml2's limits on memory usage should be turned off.
    ///
    /// This should only be done for trusted data.
    pub fn with_unlimited_size(mut self, unlimited: bool) -> Self {
        self.unlimited_size = unlimited;
        self
    }

    /// Sets whether to keep the original compressed image data from referenced JPEG/PNG images.
    ///
    /// This is only useful for rendering to Cairo PDF
    /// surfaces, which can embed the original, compressed image data instead of uncompressed
    /// RGB buffers.
    pub fn keep_image_data(mut self, keep: bool) -> Self {
        self.keep_image_data = keep;
        self
    }

    /// Creates a new `LoadOptions` with a different `base_url`.
    ///
    /// This is used when loading a referenced file that may in turn cause other files
    /// to be loaded, for example `<image xlink:href="subimage.svg"/>`
    pub fn copy_with_base_url(&self, base_url: &AllowedUrl) -> Self {
        LoadOptions {
            base_url: Some((**base_url).clone()),
            unlimited_size: self.unlimited_size,
            keep_image_data: self.keep_image_data,
        }
    }
}

/// Main handle to an SVG document.
///
/// This is the main object in librsvg.  It gets created with the [`from_stream`] method
/// and then provides access to all the primitives needed to implement the public APIs.
///
/// [`from_stream`]: #method.from_stream
pub struct Handle {
    document: Document,
}

impl Handle {
    /// Loads an SVG document into a `Handle`.
    pub fn from_stream(
        load_options: &LoadOptions,
        stream: &gio::InputStream,
        cancellable: Option<&gio::Cancellable>,
    ) -> Result<Handle, LoadingError> {
        Ok(Handle {
            document: Document::load_from_stream(load_options, stream, cancellable)?,
        })
    }

    /// Queries whether a document has a certain element `#foo`.
    ///
    /// The `id` must be an URL fragment identifier, i.e. something
    /// like `#element_id`.
    pub fn has_sub(&self, id: &str) -> Result<bool, RenderingError> {
        match self.lookup_node(id) {
            Ok(_) => Ok(true),

            Err(DefsLookupErrorKind::NotFound) => Ok(false),

            Err(e) => Err(RenderingError::InvalidId(e)),
        }
    }

    /// Returns (ink_rect, logical_rect)
    pub fn get_geometry_sub(
        &self,
        id: Option<&str>,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(Rect, Rect), RenderingError> {
        let node = self.get_node_or_root(id)?;
        let root = self.document.root();
        let is_root = node == root;

        if is_root {
            let cascaded = CascadedValues::new_from_node(&node);

            if let Some((w, h)) = get_svg_size(&self.get_intrinsic_dimensions(), &cascaded, dpi) {
                let rect = Rect::from_size(w, h);
                return Ok((rect, rect));
            }
        }

        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let bbox = draw_tree(
            DrawingMode::LimitToStack { node, root },
            &cr,
            unit_rectangle(),
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        Ok((ink_rect, logical_rect))
    }

    fn get_node_or_root(&self, id: Option<&str>) -> Result<Node, RenderingError> {
        if let Some(id) = id {
            self.lookup_node(id).map_err(RenderingError::InvalidId)
        } else {
            Ok(self.document.root())
        }
    }

    pub fn get_geometry_for_layer(
        &self,
        id: Option<&str>,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(cairo::Rectangle, cairo::Rectangle), RenderingError> {
        let node = self.get_node_or_root(id)?;
        let root = self.document.root();

        let viewport = Rect::from(*viewport);

        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let bbox = draw_tree(
            DrawingMode::LimitToStack { node, root },
            &cr,
            viewport,
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        Ok((
            cairo::Rectangle::from(ink_rect),
            cairo::Rectangle::from(logical_rect),
        ))
    }

    fn lookup_node(&self, id: &str) -> Result<Node, DefsLookupErrorKind> {
        match Href::parse(&id).map_err(DefsLookupErrorKind::HrefError)? {
            Href::PlainUrl(_) => Err(DefsLookupErrorKind::CannotLookupExternalReferences),
            Href::WithFragment(fragment) => {
                if let Some(uri) = fragment.uri() {
                    // The public APIs to get geometries of individual elements, or to render
                    // them, should only allow referencing elements within the main handle's
                    // SVG file; that is, only plain "#foo" fragment IDs are allowed here.
                    // Otherwise, a calling program could request "another-file#foo" and cause
                    // another-file to be loaded, even if it is not part of the set of
                    // resources that the main SVG actually references.  In the future we may
                    // relax this requirement to allow lookups within that set, but not to
                    // other random files.

                    let msg = format!(
                        "the public API is not allowed to look up external references: {}#{}",
                        uri,
                        fragment.fragment()
                    );

                    rsvg_log!("{}", msg);

                    return Err(DefsLookupErrorKind::CannotLookupExternalReferences);
                }

                match self.document.lookup_node_by_id(fragment.fragment()) {
                    Some(n) => Ok(n),
                    None => Err(DefsLookupErrorKind::NotFound),
                }
            }
        }
    }

    pub fn render_document(
        &self,
        cr: &cairo::Context,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        self.render_layer(cr, None, viewport, dpi, is_testing)
    }

    pub fn render_layer(
        &self,
        cr: &cairo::Context,
        id: Option<&str>,
        viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        check_cairo_context(cr)?;

        cr.save();

        let node = self.get_node_or_root(id)?;
        let root = self.document.root();

        let viewport = Rect::from(*viewport);

        let res = draw_tree(
            DrawingMode::LimitToStack { node, root },
            cr,
            viewport,
            dpi,
            false,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        );

        cr.restore();

        res.map(|_bbox| ())
    }

    fn get_bbox_for_element(
        &self,
        node: &Node,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<BoundingBox, RenderingError> {
        let target = cairo::ImageSurface::create(cairo::Format::Rgb24, 1, 1)?;
        let cr = cairo::Context::new(&target);

        let node = node.clone();

        draw_tree(
            DrawingMode::OnlyNode(node),
            &cr,
            unit_rectangle(),
            dpi,
            true,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        )
    }

    /// Returns (ink_rect, logical_rect)
    pub fn get_geometry_for_element(
        &self,
        id: Option<&str>,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(cairo::Rectangle, cairo::Rectangle), RenderingError> {
        let node = self.get_node_or_root(id)?;

        let bbox = self.get_bbox_for_element(&node, dpi, is_testing)?;

        let ink_rect = bbox.ink_rect.unwrap_or_default();
        let logical_rect = bbox.rect.unwrap_or_default();

        // Translate so ink_rect is always at offset (0, 0)
        let ofs = (-ink_rect.x0, -ink_rect.y0);

        Ok((
            cairo::Rectangle::from(ink_rect.translate(ofs)),
            cairo::Rectangle::from(logical_rect.translate(ofs)),
        ))
    }

    pub fn render_element(
        &self,
        cr: &cairo::Context,
        id: Option<&str>,
        element_viewport: &cairo::Rectangle,
        dpi: Dpi,
        is_testing: bool,
    ) -> Result<(), RenderingError> {
        check_cairo_context(cr)?;

        let node = self.get_node_or_root(id)?;

        let bbox = self.get_bbox_for_element(&node, dpi, is_testing)?;

        if bbox.ink_rect.is_none() || bbox.rect.is_none() {
            // Nothing to draw
            return Ok(());
        }

        let ink_r = bbox.ink_rect.unwrap_or_default();

        if ink_r.is_empty() {
            return Ok(());
        }

        // Render, transforming so element is at the new viewport's origin

        cr.save();

        let factor =
            (element_viewport.width / ink_r.width()).min(element_viewport.height / ink_r.height());

        cr.translate(element_viewport.x, element_viewport.y);
        cr.scale(factor, factor);
        cr.translate(-ink_r.x0, -ink_r.y0);

        let res = draw_tree(
            DrawingMode::OnlyNode(node),
            &cr,
            unit_rectangle(),
            dpi,
            false,
            is_testing,
            &mut AcquiredNodes::new(&self.document),
        );

        cr.restore();

        res.map(|_bbox| ())
    }

    pub fn get_intrinsic_dimensions(&self) -> IntrinsicDimensions {
        borrow_element_as!(self.document.root(), Svg).get_intrinsic_dimensions()
    }

    pub fn set_stylesheet(&mut self, css: &str) -> Result<(), LoadingError> {
        let mut stylesheet = Stylesheet::new(Origin::User);
        stylesheet.parse(css, None)?;
        self.document.cascade(&[stylesheet]);
        Ok(())
    }
}

fn check_cairo_context(cr: &cairo::Context) -> Result<(), RenderingError> {
    let status = cr.status();
    if status == cairo::Status::Success {
        Ok(())
    } else {
        Err(RenderingError::Cairo(status))
    }
}

fn unit_rectangle() -> Rect {
    Rect::from_size(1.0, 1.0)
}

/// Returns the SVG's size suitable for the legacy C API, or None
/// if it must be computed by hand.
///
/// The legacy C API can compute an SVG document's size from the
/// `width`, `height`, and `viewBox` attributes of the toplevel `<svg>`
/// element.  If these are not available, then the size must be computed
/// by actually measuring the geometries of elements in the document.
fn get_svg_size(
    dimensions: &IntrinsicDimensions,
    cascaded: &CascadedValues,
    dpi: Dpi,
) -> Option<(f64, f64)> {
    let values = cascaded.get();

    // these defaults are per the spec
    let w = dimensions
        .width
        .unwrap_or_else(|| Length::<Horizontal>::parse_str("100%").unwrap());
    let h = dimensions
        .height
        .unwrap_or_else(|| Length::<Vertical>::parse_str("100%").unwrap());

    match (w, h, dimensions.vbox) {
        (w, h, Some(vbox)) => {
            let params = ViewParams::new(dpi, vbox.width(), vbox.height());

            Some((w.normalize(values, &params), h.normalize(values, &params)))
        }

        (w, h, None) if w.unit != LengthUnit::Percent && h.unit != LengthUnit::Percent => {
            let params = ViewParams::new(dpi, 0.0, 0.0);

            Some((w.normalize(values, &params), h.normalize(values, &params)))
        }
        (_, _, _) => None,
    }
}
<|endoftext|>


<commit_before><commit_msg>Enable more tests on Android<commit_after><|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::utils;
use serde::{Deserialize, Serialize};
use std::{
    net::{IpAddr, Ipv4Addr, SocketAddr},
    path::PathBuf,
};

// Lru cache will consume about 2G RAM based on this default value.
pub const DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD: usize = 1 << 13;

pub const TARGET_SNAPSHOT_SIZE: usize = 100_000;

/// Port selected RocksDB options for tuning underlying rocksdb instance of AptosDB.
/// see <https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h>
/// for detailed explanations.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub struct RocksdbConfig {
    pub max_open_files: i32,
    pub max_total_wal_size: u64,
    pub max_background_jobs: i32,
    pub block_cache_size: u64,
    pub block_size: u64,
    pub cache_index_and_filter_blocks: bool,
}

impl Default for RocksdbConfig {
    fn default() -> Self {
        Self {
            // Allow db to close old sst files, saving memory.
            max_open_files: 5000,
            // For now we set the max total WAL size to be 1G. This config can be useful when column
            // families are updated at non-uniform frequencies.
            max_total_wal_size: 1u64 << 30,
            // This includes threads for flashing and compaction. Rocksdb will decide the # of
            // threads to use internally.
            max_background_jobs: 16,
            // Default block cache size is 8MB,
            block_cache_size: 8 * (1u64 << 20),
            // Default block cache size is 4KB,
            block_size: 4 * (1u64 << 10),
            // Whether cache index and filter blocks into block cache.
            cache_index_and_filter_blocks: false,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct RocksdbConfigs {
    pub ledger_db_config: RocksdbConfig,
    pub state_merkle_db_config: RocksdbConfig,
    pub index_db_config: RocksdbConfig,
}

impl Default for RocksdbConfigs {
    fn default() -> Self {
        Self {
            ledger_db_config: RocksdbConfig::default(),
            state_merkle_db_config: RocksdbConfig::default(),
            index_db_config: RocksdbConfig {
                max_open_files: 1000,
                ..Default::default()
            },
        }
    }
}

#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageConfig {
    pub address: SocketAddr,
    pub backup_service_address: SocketAddr,
    pub dir: PathBuf,
    pub storage_pruner_config: PrunerConfig,
    #[serde(skip)]
    data_dir: PathBuf,
    /// The threshold that determine whether a snapshot should be committed to state merkle db.
    pub target_snapshot_size: usize,
    /// The max # of nodes for a lru cache shard.
    pub max_num_nodes_per_lru_cache_shard: usize,
    /// Rocksdb-specific configurations
    pub rocksdb_configs: RocksdbConfigs,
    /// Try to enable the internal indexer. The indexer expects to have seen all transactions
    /// since genesis. To recover operation after data loss, or to bootstrap a node in fast sync
    /// mode, the indexer db needs to be copied in from another node.
    pub enable_indexer: bool,
}

pub const NO_OP_STORAGE_PRUNER_CONFIG: PrunerConfig = PrunerConfig {
    ledger_pruner_config: LedgerPrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
        user_pruning_window_offset: 0,
    },
    state_merkle_pruner_config: StateMerklePrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
    },
    epoch_snapshot_pruner_config: EpochSnapshotPrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
    },
};

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct LedgerPrunerConfig {
    /// Boolean to enable/disable the ledger pruner. The ledger pruner is responsible for pruning
    /// everything else except for states (e.g. transactions, events etc.)
    pub enable: bool,
    /// This is the default pruning window for any other store except for state store. State store
    /// being big in size, we might want to configure a smaller window for state store vs other
    /// store.
    pub prune_window: u64,
    /// Batch size of the versions to be sent to the ledger pruner - this is to avoid slowdown due to
    /// issuing too many DB calls and batch prune instead. For ledger pruner, this means the number
    /// of versions to prune a time.
    pub batch_size: usize,
    /// The offset for user pruning window to adjust
    pub user_pruning_window_offset: u64,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateMerklePrunerConfig {
    /// Boolean to enable/disable the state store pruner. The state pruner is responsible for
    /// pruning state tree nodes.
    pub enable: bool,
    /// The size of the window should be calculated based on disk space availability and system TPS.
    pub prune_window: u64,
    /// Similar to the variable above but for state store pruner. It means the number of stale
    /// nodes to prune a time.
    pub batch_size: usize,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct EpochSnapshotPrunerConfig {
    pub enable: bool,
    /// Window size in versions, but only the snapshots at epoch ending versions are kept, because
    /// other snapshots are pruned by the state merkle pruner.
    pub prune_window: u64,
    /// Number of stale nodes to prune a time.
    pub batch_size: usize,
}

// Config for the epoch ending state pruner is actually in the same format as the state merkle
// pruner, but it has it's own type hence separate default values. This converts it to the same
// type, to use the same pruner implementation (but parameterized on the stale node index DB schema).
impl From<EpochSnapshotPrunerConfig> for StateMerklePrunerConfig {
    fn from(config: EpochSnapshotPrunerConfig) -> Self {
        Self {
            enable: config.enable,
            prune_window: config.prune_window,
            batch_size: config.batch_size,
        }
    }
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, Default)]
#[serde(default, deny_unknown_fields)]
pub struct PrunerConfig {
    pub ledger_pruner_config: LedgerPrunerConfig,
    pub state_merkle_pruner_config: StateMerklePrunerConfig,
    pub epoch_snapshot_pruner_config: EpochSnapshotPrunerConfig,
}

impl Default for LedgerPrunerConfig {
    fn default() -> Self {
        LedgerPrunerConfig {
            enable: true,
            // This assumes we have 1T disk, minus the space needed by state merkle db and the
            // overhead in storage.
            prune_window: 150_000_000,
            batch_size: 500,
            user_pruning_window_offset: 200_000,
        }
    }
}

impl Default for StateMerklePrunerConfig {
    fn default() -> Self {
        StateMerklePrunerConfig {
            enable: true,
            // This allows a block / chunk being executed to have access to a non-latest state tree.
            // It needs to be greater than the number of versions the state committing thread is
            // able to commit during the execution of the block / chunk.
            prune_window: 100_000,
            // A 10k transaction block (touching 60k state values, in the case of the account
            // creation benchmark) on a 4B items DB (or 1.33B accounts) yields 300k JMT nodes
            batch_size: 1_000,
        }
    }
}

impl Default for EpochSnapshotPrunerConfig {
    fn default() -> Self {
        Self {
            enable: true,
            // This is based on ~5K TPS * 2h/epoch * 2 epochs. -- epoch ending snapshots are used
            // by state sync in fast sync mode.
            // The setting is in versions, not epochs, because this makes it behave more like other
            // pruners: a slower network will have longer history in db with the same pruner
            // settings, but the disk space take will be similar.
            // settings.
            prune_window: 80_000_000,
            // A 10k transaction block (touching 60k state values, in the case of the account
            // creation benchmark) on a 4B items DB (or 1.33B accounts) yields 300k JMT nodes
            batch_size: 1_000,
        }
    }
}

impl Default for StorageConfig {
    fn default() -> StorageConfig {
        StorageConfig {
            address: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6666),
            backup_service_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6186),
            dir: PathBuf::from("db"),
            // The prune window must at least out live a RPC request because its sub requests are
            // to return a consistent view of the DB at exactly same version. Considering a few
            // thousand TPS we are potentially going to achieve, and a few minutes a consistent view
            // of the DB might require, 10k (TPS)  * 100 (seconds)  =  1 Million might be a
            // conservatively safe minimal prune window. It'll take a few Gigabytes of disk space
            // depending on the size of an average account blob.
            storage_pruner_config: PrunerConfig::default(),
            data_dir: PathBuf::from("/opt/aptos/data"),
            rocksdb_configs: RocksdbConfigs::default(),
            enable_indexer: false,
            target_snapshot_size: TARGET_SNAPSHOT_SIZE,
            max_num_nodes_per_lru_cache_shard: DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
        }
    }
}

impl StorageConfig {
    pub fn dir(&self) -> PathBuf {
        if self.dir.is_relative() {
            self.data_dir.join(&self.dir)
        } else {
            self.dir.clone()
        }
    }

    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
        self.data_dir = data_dir;
    }

    pub fn randomize_ports(&mut self) {
        self.address.set_port(utils::get_available_port());
        self.backup_service_address
            .set_port(utils::get_available_port());
    }
}

#[cfg(test)]
mod test {
    use crate::config::PrunerConfig;

    #[test]
    pub fn tset_default_prune_window() {
        // Not that these can't be changed, but think twice -- make them safe for mainnet

        let config = PrunerConfig::default();
        assert!(config.ledger_pruner_config.prune_window >= 50_000_000);
        assert!(config.state_merkle_pruner_config.prune_window >= 100_000);
        assert!(config.epoch_snapshot_pruner_config.prune_window > 50_000_000);
    }
}
<commit_msg>[Config] Remove unused config variable.<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::utils;
use serde::{Deserialize, Serialize};
use std::{
    net::{IpAddr, Ipv4Addr, SocketAddr},
    path::PathBuf,
};

// Lru cache will consume about 2G RAM based on this default value.
pub const DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD: usize = 1 << 13;

pub const TARGET_SNAPSHOT_SIZE: usize = 100_000;

/// Port selected RocksDB options for tuning underlying rocksdb instance of AptosDB.
/// see <https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h>
/// for detailed explanations.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub struct RocksdbConfig {
    pub max_open_files: i32,
    pub max_total_wal_size: u64,
    pub max_background_jobs: i32,
    pub block_cache_size: u64,
    pub block_size: u64,
    pub cache_index_and_filter_blocks: bool,
}

impl Default for RocksdbConfig {
    fn default() -> Self {
        Self {
            // Allow db to close old sst files, saving memory.
            max_open_files: 5000,
            // For now we set the max total WAL size to be 1G. This config can be useful when column
            // families are updated at non-uniform frequencies.
            max_total_wal_size: 1u64 << 30,
            // This includes threads for flashing and compaction. Rocksdb will decide the # of
            // threads to use internally.
            max_background_jobs: 16,
            // Default block cache size is 8MB,
            block_cache_size: 8 * (1u64 << 20),
            // Default block cache size is 4KB,
            block_size: 4 * (1u64 << 10),
            // Whether cache index and filter blocks into block cache.
            cache_index_and_filter_blocks: false,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct RocksdbConfigs {
    pub ledger_db_config: RocksdbConfig,
    pub state_merkle_db_config: RocksdbConfig,
    pub index_db_config: RocksdbConfig,
}

impl Default for RocksdbConfigs {
    fn default() -> Self {
        Self {
            ledger_db_config: RocksdbConfig::default(),
            state_merkle_db_config: RocksdbConfig::default(),
            index_db_config: RocksdbConfig {
                max_open_files: 1000,
                ..Default::default()
            },
        }
    }
}

#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageConfig {
    pub backup_service_address: SocketAddr,
    pub dir: PathBuf,
    pub storage_pruner_config: PrunerConfig,
    #[serde(skip)]
    data_dir: PathBuf,
    /// The threshold that determine whether a snapshot should be committed to state merkle db.
    pub target_snapshot_size: usize,
    /// The max # of nodes for a lru cache shard.
    pub max_num_nodes_per_lru_cache_shard: usize,
    /// Rocksdb-specific configurations
    pub rocksdb_configs: RocksdbConfigs,
    /// Try to enable the internal indexer. The indexer expects to have seen all transactions
    /// since genesis. To recover operation after data loss, or to bootstrap a node in fast sync
    /// mode, the indexer db needs to be copied in from another node.
    pub enable_indexer: bool,
}

pub const NO_OP_STORAGE_PRUNER_CONFIG: PrunerConfig = PrunerConfig {
    ledger_pruner_config: LedgerPrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
        user_pruning_window_offset: 0,
    },
    state_merkle_pruner_config: StateMerklePrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
    },
    epoch_snapshot_pruner_config: EpochSnapshotPrunerConfig {
        enable: false,
        prune_window: 0,
        batch_size: 0,
    },
};

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct LedgerPrunerConfig {
    /// Boolean to enable/disable the ledger pruner. The ledger pruner is responsible for pruning
    /// everything else except for states (e.g. transactions, events etc.)
    pub enable: bool,
    /// This is the default pruning window for any other store except for state store. State store
    /// being big in size, we might want to configure a smaller window for state store vs other
    /// store.
    pub prune_window: u64,
    /// Batch size of the versions to be sent to the ledger pruner - this is to avoid slowdown due to
    /// issuing too many DB calls and batch prune instead. For ledger pruner, this means the number
    /// of versions to prune a time.
    pub batch_size: usize,
    /// The offset for user pruning window to adjust
    pub user_pruning_window_offset: u64,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateMerklePrunerConfig {
    /// Boolean to enable/disable the state store pruner. The state pruner is responsible for
    /// pruning state tree nodes.
    pub enable: bool,
    /// The size of the window should be calculated based on disk space availability and system TPS.
    pub prune_window: u64,
    /// Similar to the variable above but for state store pruner. It means the number of stale
    /// nodes to prune a time.
    pub batch_size: usize,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct EpochSnapshotPrunerConfig {
    pub enable: bool,
    /// Window size in versions, but only the snapshots at epoch ending versions are kept, because
    /// other snapshots are pruned by the state merkle pruner.
    pub prune_window: u64,
    /// Number of stale nodes to prune a time.
    pub batch_size: usize,
}

// Config for the epoch ending state pruner is actually in the same format as the state merkle
// pruner, but it has it's own type hence separate default values. This converts it to the same
// type, to use the same pruner implementation (but parameterized on the stale node index DB schema).
impl From<EpochSnapshotPrunerConfig> for StateMerklePrunerConfig {
    fn from(config: EpochSnapshotPrunerConfig) -> Self {
        Self {
            enable: config.enable,
            prune_window: config.prune_window,
            batch_size: config.batch_size,
        }
    }
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, Default)]
#[serde(default, deny_unknown_fields)]
pub struct PrunerConfig {
    pub ledger_pruner_config: LedgerPrunerConfig,
    pub state_merkle_pruner_config: StateMerklePrunerConfig,
    pub epoch_snapshot_pruner_config: EpochSnapshotPrunerConfig,
}

impl Default for LedgerPrunerConfig {
    fn default() -> Self {
        LedgerPrunerConfig {
            enable: true,
            // This assumes we have 1T disk, minus the space needed by state merkle db and the
            // overhead in storage.
            prune_window: 150_000_000,
            batch_size: 500,
            user_pruning_window_offset: 200_000,
        }
    }
}

impl Default for StateMerklePrunerConfig {
    fn default() -> Self {
        StateMerklePrunerConfig {
            enable: true,
            // This allows a block / chunk being executed to have access to a non-latest state tree.
            // It needs to be greater than the number of versions the state committing thread is
            // able to commit during the execution of the block / chunk.
            prune_window: 100_000,
            // A 10k transaction block (touching 60k state values, in the case of the account
            // creation benchmark) on a 4B items DB (or 1.33B accounts) yields 300k JMT nodes
            batch_size: 1_000,
        }
    }
}

impl Default for EpochSnapshotPrunerConfig {
    fn default() -> Self {
        Self {
            enable: true,
            // This is based on ~5K TPS * 2h/epoch * 2 epochs. -- epoch ending snapshots are used
            // by state sync in fast sync mode.
            // The setting is in versions, not epochs, because this makes it behave more like other
            // pruners: a slower network will have longer history in db with the same pruner
            // settings, but the disk space take will be similar.
            // settings.
            prune_window: 80_000_000,
            // A 10k transaction block (touching 60k state values, in the case of the account
            // creation benchmark) on a 4B items DB (or 1.33B accounts) yields 300k JMT nodes
            batch_size: 1_000,
        }
    }
}

impl Default for StorageConfig {
    fn default() -> StorageConfig {
        StorageConfig {
            backup_service_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6186),
            dir: PathBuf::from("db"),
            // The prune window must at least out live a RPC request because its sub requests are
            // to return a consistent view of the DB at exactly same version. Considering a few
            // thousand TPS we are potentially going to achieve, and a few minutes a consistent view
            // of the DB might require, 10k (TPS)  * 100 (seconds)  =  1 Million might be a
            // conservatively safe minimal prune window. It'll take a few Gigabytes of disk space
            // depending on the size of an average account blob.
            storage_pruner_config: PrunerConfig::default(),
            data_dir: PathBuf::from("/opt/aptos/data"),
            rocksdb_configs: RocksdbConfigs::default(),
            enable_indexer: false,
            target_snapshot_size: TARGET_SNAPSHOT_SIZE,
            max_num_nodes_per_lru_cache_shard: DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD,
        }
    }
}

impl StorageConfig {
    pub fn dir(&self) -> PathBuf {
        if self.dir.is_relative() {
            self.data_dir.join(&self.dir)
        } else {
            self.dir.clone()
        }
    }

    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
        self.data_dir = data_dir;
    }

    pub fn randomize_ports(&mut self) {
        self.backup_service_address
            .set_port(utils::get_available_port());
    }
}

#[cfg(test)]
mod test {
    use crate::config::PrunerConfig;

    #[test]
    pub fn tset_default_prune_window() {
        // Not that these can't be changed, but think twice -- make them safe for mainnet

        let config = PrunerConfig::default();
        assert!(config.ledger_pruner_config.prune_window >= 50_000_000);
        assert!(config.state_merkle_pruner_config.prune_window >= 100_000);
        assert!(config.epoch_snapshot_pruner_config.prune_window > 50_000_000);
    }
}
<|endoftext|>


<commit_before>#![crate_id(name="head", vers="1.0.0", author="Alan Andrade")]
/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Alan Andrade <alan.andradec@gmail.com>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 *
 * Synced with: https://raw.github.com/avsm/src/master/usr.bin/head/head.c
 */

extern crate getopts;

use std::os;
use std::char;
use std::io::{stdin};
use std::io::BufferedReader;
use std::io::fs::File;
use std::path::Path;
use getopts::{optopt, optflag, getopts, usage};

static PROGRAM: &'static str = "head";

fn main () {
    let mut line_count = 10u;

    // handle obsolete -number syntax
    let options = match obsolete(os::args().tail()) {
        (args, Some(n)) => { line_count = n; args },
        (args, None) => args
    };

    let args = options;

    let possible_options = [
        optopt("n", "number", "Number of lines to print", "n"),
        optflag("h", "help", "help"),
        optflag("V", "version", "version")
    ];

    let given_options = match getopts(args.as_slice(), possible_options) {
        Ok (m) => { m }
        Err(_) => {
            println!("{:s}", usage(PROGRAM, possible_options));
            return
        }
    };

    if given_options.opt_present("h") {
        println!("{:s}", usage(PROGRAM, possible_options));
        return;
    }
    if given_options.opt_present("V") { version(); return }

    match given_options.opt_str("n") {
        Some(n) => {
            match from_str(n.as_slice()) {
                Some(m) => { line_count = m }
                None => {}
            }
        }
        None => {}
    };

    let files = given_options.free;

    if files.is_empty() {
        let mut buffer = BufferedReader::new(stdin());
        head(&mut buffer, line_count);
    } else {
        let mut multiple = false;
        let mut firstime = true;

        if files.len() > 1 {
            multiple = true;
        }

        for file in files.iter() {
            if multiple {
                if !firstime { println!(""); }
                println!("==> {:s} <==", file.as_slice());
            }
            firstime = false;

            let path = Path::new(file.as_slice());
            let reader = File::open(&path).unwrap();
            let mut buffer = BufferedReader::new(reader);
            head(&mut buffer, line_count);
        }
    }
}

// It searches for an option in the form of -123123
//
// In case is found, the options vector will get rid of that object so that
// getopts works correctly.
fn obsolete (options: &[String]) -> (Vec<String>, Option<uint>) {
    let mut options: Vec<String> = Vec::from_slice(options);
    let mut a = 0;
    let b = options.len();

    while a < b {
        let current = options.get(a).clone();
        let current = current.as_slice();

        if current.len() > 1 && current[0] == '-' as u8 {
            let len = current.len();
            for pos in range(1, len) {
                // Ensure that the argument is only made out of digits
                if !char::is_digit(current.char_at(pos)) { break; }

                // If this is the last number
                if pos == len - 1 {
                    options.remove(a);
                    let number : Option<uint> = from_str(current.slice(1,len));
                    return (options, Some(number.unwrap()));
                }
            }
        }

        a += 1;
    };

    (options, None)
}

fn head<T: Reader> (reader: &mut BufferedReader<T>, line_count:uint) {
    for line in reader.lines().take(line_count) { print!("{}", line); }
}

fn version () {
    println!("head version 1.0.0");
}
<commit_msg>head: unwrap IoResult before printing<commit_after>#![crate_id(name="head", vers="1.0.0", author="Alan Andrade")]
/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Alan Andrade <alan.andradec@gmail.com>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 *
 * Synced with: https://raw.github.com/avsm/src/master/usr.bin/head/head.c
 */

extern crate getopts;

use std::os;
use std::char;
use std::io::{stdin};
use std::io::BufferedReader;
use std::io::fs::File;
use std::path::Path;
use getopts::{optopt, optflag, getopts, usage};

static PROGRAM: &'static str = "head";

fn main () {
    let mut line_count = 10u;

    // handle obsolete -number syntax
    let options = match obsolete(os::args().tail()) {
        (args, Some(n)) => { line_count = n; args },
        (args, None) => args
    };

    let args = options;

    let possible_options = [
        optopt("n", "number", "Number of lines to print", "n"),
        optflag("h", "help", "help"),
        optflag("V", "version", "version")
    ];

    let given_options = match getopts(args.as_slice(), possible_options) {
        Ok (m) => { m }
        Err(_) => {
            println!("{:s}", usage(PROGRAM, possible_options));
            return
        }
    };

    if given_options.opt_present("h") {
        println!("{:s}", usage(PROGRAM, possible_options));
        return;
    }
    if given_options.opt_present("V") { version(); return }

    match given_options.opt_str("n") {
        Some(n) => {
            match from_str(n.as_slice()) {
                Some(m) => { line_count = m }
                None => {}
            }
        }
        None => {}
    };

    let files = given_options.free;

    if files.is_empty() {
        let mut buffer = BufferedReader::new(stdin());
        head(&mut buffer, line_count);
    } else {
        let mut multiple = false;
        let mut firstime = true;

        if files.len() > 1 {
            multiple = true;
        }

        for file in files.iter() {
            if multiple {
                if !firstime { println!(""); }
                println!("==> {:s} <==", file.as_slice());
            }
            firstime = false;

            let path = Path::new(file.as_slice());
            let reader = File::open(&path).unwrap();
            let mut buffer = BufferedReader::new(reader);
            head(&mut buffer, line_count);
        }
    }
}

// It searches for an option in the form of -123123
//
// In case is found, the options vector will get rid of that object so that
// getopts works correctly.
fn obsolete (options: &[String]) -> (Vec<String>, Option<uint>) {
    let mut options: Vec<String> = Vec::from_slice(options);
    let mut a = 0;
    let b = options.len();

    while a < b {
        let current = options.get(a).clone();
        let current = current.as_slice();

        if current.len() > 1 && current[0] == '-' as u8 {
            let len = current.len();
            for pos in range(1, len) {
                // Ensure that the argument is only made out of digits
                if !char::is_digit(current.char_at(pos)) { break; }

                // If this is the last number
                if pos == len - 1 {
                    options.remove(a);
                    let number : Option<uint> = from_str(current.slice(1,len));
                    return (options, Some(number.unwrap()));
                }
            }
        }

        a += 1;
    };

    (options, None)
}

fn head<T: Reader> (reader: &mut BufferedReader<T>, line_count:uint) {
    for line in reader.lines().take(line_count) { print!("{}", line.unwrap()); }
}

fn version () {
    println!("head version 1.0.0");
}
<|endoftext|>