<commit_before>// In heavy WIP
use std::fs::File;
use std::io::Read;
use toml;

use error::PackageError;

#[derive(Deserialize)]
struct Config {
    buffer_size: Option<u64>,
    config_path: Option<String>,
    download_path: Option<String>,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            buffer_size: Some(65536),
            config_path: Some(String::from("~/.wiz/config/")),
            download_path: Some(String::from("~/.wiz/downloads/")),
        }
    }
}

impl Config {
    /// Read the config file from the default config path.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    fn read() -> Result<Self, PackageError> {
        if let Self { config_path: Some(path), .. } = Self::default() {
            // Read the config file into a string
            let mut config = File::open(path)?;
            let mut content = String::new();
            config.read_to_string(&mut content)?;
            // Try parsing the string, and convert it into a `Config`.
            let config = content.parse::<toml::Value>()?;
            let config = config.try_into::<Self>()?;
            Ok(config)
        } else {
            // This should never, ever, happen.
            Err(PackageError::Parsing(
                String::from("Unable to get config_path from the default cfg.")
            ))
        }
    }
}<commit_msg>Ability to read the config file from a custom path<commit_after>// In heavy WIP
use std::fs::File;
use std::io::Read;
use std::path::Path;
use toml;

use error::PackageError;

/// Represents the config file.
#[derive(Deserialize)]
struct Config {
    buffer_size: Option<u64>,
    config_path: Option<String>,
    download_path: Option<String>,
}

impl Default for Config {
    fn default() -> Self {
        Self {
            buffer_size: Some(65536),
            config_path: Some(String::from("~/.wiz/config/")),
            download_path: Some(String::from("~/.wiz/downloads/")),
        }
    }
}

impl Config {
    /// Read the config file from the config path specified in `path`.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from(path: &Path) -> Result<Self, PackageError> {
        // Read the config file into a string
        let mut config = File::open(path)?;
        let mut content = String::new();
        config.read_to_string(&mut content)?;

        // Try parsing the string, and convert it into a `Config`.
        let config = content.parse::<toml::Value>()?;
        let config = config.try_into::<Self>()?;
        Ok(config)
    }

    /// Read the config file from the default config path.
    /// If the config file is read & parsed properly, it should return
    /// a `Config`.
    pub fn read_from_default() -> Result<Self, PackageError> {
        if let Self { config_path: Some(path), .. } = Self::default() {
            Self::read_from(Path::new(&path))
        } else {
            // This should never, ever, happen.
            Err(PackageError::Parsing(
                String::from("Unable to get config_path from the default cfg.")
            ))
        }
    }

}<|endoftext|>


<commit_before>use std::fmt;

use chrono::{DateTime, Local};
use itertools::Itertools;
use json::JsonValue;

// #[derive(Debug, Default)]
// pub struct Error {
//     pub detail: String,
// }
//
// #[derive(Debug)]
// pub struct CrateLinks {
//     pub owners: Option<String>,
//     pub reverse_dependencies: String,
//     pub version_downloads: String,
//     pub versions: Option<String>,
// }
//
// pub struct Crate {
//     pub created_at: String,
//     pub description: Option<String>,
//     pub documentation: Option<String>,
//     pub downloads: i32,
//     pub homepage: Option<String>,
//     pub id: String,
//     pub keywords: Option<Vec<String>>,
//     pub license: Option<String>,
//     pub links: CrateLinks,
//     pub max_version: String,
//     pub name: String,
//     pub repository: Option<String>,
//     pub updated_at: String,
//     pub versions: Option<Vec<u64>>,
// }
//
// #[derive(Debug)]
// pub struct Keyword {
//     pub crates_cnt: u64,
//     pub created_at: String,
//     pub id: String,
//     pub keyword: String,
// }
//
// #[derive(Debug)]
// pub struct VersionLinks {
//     pub authors: String,
//     pub dependencies: String,
//     pub version_downloads: String,
// }
//
// #[derive(Debug)]
// pub struct Version {
//     pub krate: String,
//     pub created_at: String,
//     pub dl_path: String,
//     pub downloads: i32,
//     pub features: HashMap<String, Vec<String>>,
//     pub id: i32,
//     pub links: VersionLinks,
//     pub num: String,
//     pub updated_at: String,
//     pub yanked: bool,
// }
//
// pub struct Reply {
//     pub errors: Error,
//     pub krate: Crate,
//     pub keywords: Vec<Keyword>,
//     pub versions: Vec<Version>,
// }

struct TimeStamp(Option<DateTime<Local>>);

impl<'a> From<&'a JsonValue> for TimeStamp {
    fn from(jv: &JsonValue) -> Self {
        let parse = |s: &str| s.parse::<DateTime<Local>>().ok();
        TimeStamp(jv.as_str().and_then(parse))
    }
}

impl fmt::Display for TimeStamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if let Some(ts) = self.0 {
            f.pad(&format!("{}", ts))
        } else {
            f.pad("")
        }
    }
}

pub struct Crate {
    krate: JsonValue,
    versions: JsonValue,
    keywords: JsonValue,
}

impl Crate {
    pub fn new(json: &JsonValue) -> Self {
        Crate {
            krate: json["crate"].clone(),
            versions: json["versions"].clone(),
            keywords: json["keywords"].clone(),
        }
    }

    pub fn print_repository(&self, verbose: bool) {
        if let JsonValue::String(ref repository) = self.krate["repository"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Repository:", repository)
            } else {
                repository.clone()
            };
            println!("{}", fmt);
        }
    }

    pub fn print_documentation(&self, verbose: bool) {
        if let JsonValue::String(ref documentation) = self.krate["documentation"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Documentation:", documentation)
            } else {
                documentation.clone()
            };
            println!("{}", fmt);
        }
    }

    pub fn print_downloads(&self, verbose: bool) {
        if let JsonValue::Number(downloads) = self.krate["downloads"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Downloads:", downloads)
            } else {
                format!("{}", downloads)
            };
            println!("{}", fmt);
        }
    }

    pub fn print_homepage(&self, verbose: bool) {
        if let JsonValue::String(ref homepage) = self.krate["homepage"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Homepage:", homepage)
            } else {
                homepage.clone()
            };
            println!("{}", fmt);
        }
    }

    #[allow(unknown_lints)]
    #[allow(bool_comparison)]
    fn print_version(v: &JsonValue, verbose: bool) {
        let created_at = TimeStamp::from(&v["created_at"]);
        print!("{:<10}{:<28}{:<11}", v["num"], created_at, v["downloads"]);

        if v["yanked"] == true {
            print!("(yanked)");
        }

        if verbose {
            // Consider adding some more useful information in verbose mode
            println!("");
        } else {
            println!("");
        }
    }

    fn print_version_header(verbose: bool) {
        print!("{:<10}{:<28}{:<11}\n",
                 "VERSION",
                 "RELEASE DATE",
                 "DOWNLOADS");
        if verbose {
            // Consider adding some more useful information in verbose mode
            println!("");
        } else {
            println!("");
        }
    }

    pub fn print_last_versions(&self, limit: usize, verbose: bool) {
        Crate::print_version_header(verbose);
        self.versions
            .members()
            .take(limit)
            .foreach(|v| Crate::print_version(v, verbose));

        let length = self.versions.len();
        if limit < length {
            println!("\n... use -VV to show all {} versions", length);
        }
    }

    pub fn print_keywords(&self, verbose: bool) {
        let fmt = if verbose {
            format!("{:#}", self.keywords)
        } else {
            format!("{}", self.keywords)
        };
        println!("{}", fmt);
    }
}

impl fmt::Display for Crate {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let empty = "";

        let name = self.krate["name"].as_str().unwrap_or(empty);
        let max_version = self.krate["max_version"].as_str().unwrap_or(empty);
        let downloads = self.krate["downloads"].as_i32().unwrap_or(0);

        let created_at = TimeStamp::from(&self.krate["created_at"]);
        let updated_at = TimeStamp::from(&self.krate["updated_at"]);

        let description = self.krate["description"].as_str().unwrap_or(empty);
        let documentation = self.krate["documentation"].as_str().unwrap_or(empty);
        let homepage = self.krate["homepage"].as_str().unwrap_or(empty);
        let repository = self.krate["repository"].as_str().unwrap_or(empty);
        let license = self.krate["license"].as_str().unwrap_or(empty);

        let keywords = self.krate["keywords"]
            .members()
            .filter_map(|jv| jv.as_str())
            .collect::<Vec<_>>();

        if f.alternate() {
            write!(f,
                   "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
                   format_args!("{:<16}{}", "Crate:", name),
                   format_args!("{:<16}{}", "Version:", max_version),
                   format_args!("{:<16}{}", "Description:", description),
                   format_args!("{:<16}{}", "Downloads:", downloads),
                   format_args!("{:<16}{}", "Homepage:", homepage),
                   format_args!("{:<16}{}", "Documentation:", documentation),
                   format_args!("{:<16}{}", "Repository:", repository),
                   format_args!("{:<16}{}", "License:", license),
                   format_args!("{:<16}{:?}", "Keywords:", keywords),
                   format_args!("{:<16}{}", "Created at:", created_at),
                   format_args!("{:<16}{}", "Updated at:", updated_at))
        } else {
            write!(f,
                   "{}\n{}\n{}\n{}\n{}\n{}\n{}",
                   format_args!("{:<16}{}", "Crate:", name),
                   format_args!("{:<16}{}", "Version:", max_version),
                   format_args!("{:<16}{}", "Description:", description),
                   format_args!("{:<16}{}", "Downloads:", downloads),
                   format_args!("{:<16}{}", "Homepage:", homepage),
                   format_args!("{:<16}{}", "Documentation:", documentation),
                   format_args!("{:<16}{}", "Repository:", repository))
        }
    }
}
<commit_msg>Workaround clippy check<commit_after>use std::fmt;

use chrono::{DateTime, Local};
use itertools::Itertools;
use json::JsonValue;

// #[derive(Debug, Default)]
// pub struct Error {
//     pub detail: String,
// }
//
// #[derive(Debug)]
// pub struct CrateLinks {
//     pub owners: Option<String>,
//     pub reverse_dependencies: String,
//     pub version_downloads: String,
//     pub versions: Option<String>,
// }
//
// pub struct Crate {
//     pub created_at: String,
//     pub description: Option<String>,
//     pub documentation: Option<String>,
//     pub downloads: i32,
//     pub homepage: Option<String>,
//     pub id: String,
//     pub keywords: Option<Vec<String>>,
//     pub license: Option<String>,
//     pub links: CrateLinks,
//     pub max_version: String,
//     pub name: String,
//     pub repository: Option<String>,
//     pub updated_at: String,
//     pub versions: Option<Vec<u64>>,
// }
//
// #[derive(Debug)]
// pub struct Keyword {
//     pub crates_cnt: u64,
//     pub created_at: String,
//     pub id: String,
//     pub keyword: String,
// }
//
// #[derive(Debug)]
// pub struct VersionLinks {
//     pub authors: String,
//     pub dependencies: String,
//     pub version_downloads: String,
// }
//
// #[derive(Debug)]
// pub struct Version {
//     pub krate: String,
//     pub created_at: String,
//     pub dl_path: String,
//     pub downloads: i32,
//     pub features: HashMap<String, Vec<String>>,
//     pub id: i32,
//     pub links: VersionLinks,
//     pub num: String,
//     pub updated_at: String,
//     pub yanked: bool,
// }
//
// pub struct Reply {
//     pub errors: Error,
//     pub krate: Crate,
//     pub keywords: Vec<Keyword>,
//     pub versions: Vec<Version>,
// }

struct TimeStamp(Option<DateTime<Local>>);

impl<'a> From<&'a JsonValue> for TimeStamp {
    fn from(jv: &JsonValue) -> Self {
        let parse = |s: &str| s.parse::<DateTime<Local>>().ok();
        TimeStamp(jv.as_str().and_then(parse))
    }
}

impl fmt::Display for TimeStamp {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if let Some(ts) = self.0 {
            f.pad(&format!("{}", ts))
        } else {
            f.pad("")
        }
    }
}

pub struct Crate {
    krate: JsonValue,
    versions: JsonValue,
    keywords: JsonValue,
}

impl Crate {
    pub fn new(json: &JsonValue) -> Self {
        Crate {
            krate: json["crate"].clone(),
            versions: json["versions"].clone(),
            keywords: json["keywords"].clone(),
        }
    }

    pub fn print_repository(&self, verbose: bool) {
        if let JsonValue::String(ref repository) = self.krate["repository"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Repository:", repository)
            } else {
                repository.clone()
            };
            println!("{}", fmt);
        }
    }

    pub fn print_documentation(&self, verbose: bool) {
        if let JsonValue::String(ref documentation) = self.krate["documentation"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Documentation:", documentation)
            } else {
                documentation.clone()
            };
            println!("{}", fmt);
        }
    }

    pub fn print_downloads(&self, verbose: bool) {
        if let JsonValue::Number(downloads) = self.krate["downloads"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Downloads:", downloads)
            } else {
                format!("{}", downloads)
            };
            println!("{}", fmt);
        }
    }

    pub fn print_homepage(&self, verbose: bool) {
        if let JsonValue::String(ref homepage) = self.krate["homepage"] {
            let fmt = if verbose {
                format!("{:<16}{}", "Homepage:", homepage)
            } else {
                homepage.clone()
            };
            println!("{}", fmt);
        }
    }

    fn print_version(v: &JsonValue, verbose: bool) {
        let created_at = TimeStamp::from(&v["created_at"]);
        print!("{:<10}{:<28}{:<11}", v["num"], created_at, v["downloads"]);

        if v["yanked"] == "true" {
            print!("(yanked)");
        }

        if verbose {
            // Consider adding some more useful information in verbose mode
            println!("");
        } else {
            println!("");
        }
    }

    fn print_version_header(verbose: bool) {
        print!("{:<10}{:<28}{:<11}\n",
                 "VERSION",
                 "RELEASE DATE",
                 "DOWNLOADS");
        if verbose {
            // Consider adding some more useful information in verbose mode
            println!("");
        } else {
            println!("");
        }
    }

    pub fn print_last_versions(&self, limit: usize, verbose: bool) {
        Crate::print_version_header(verbose);
        self.versions
            .members()
            .take(limit)
            .foreach(|v| Crate::print_version(v, verbose));

        let length = self.versions.len();
        if limit < length {
            println!("\n... use -VV to show all {} versions", length);
        }
    }

    pub fn print_keywords(&self, verbose: bool) {
        let fmt = if verbose {
            format!("{:#}", self.keywords)
        } else {
            format!("{}", self.keywords)
        };
        println!("{}", fmt);
    }
}

impl fmt::Display for Crate {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let empty = "";

        let name = self.krate["name"].as_str().unwrap_or(empty);
        let max_version = self.krate["max_version"].as_str().unwrap_or(empty);
        let downloads = self.krate["downloads"].as_i32().unwrap_or(0);

        let created_at = TimeStamp::from(&self.krate["created_at"]);
        let updated_at = TimeStamp::from(&self.krate["updated_at"]);

        let description = self.krate["description"].as_str().unwrap_or(empty);
        let documentation = self.krate["documentation"].as_str().unwrap_or(empty);
        let homepage = self.krate["homepage"].as_str().unwrap_or(empty);
        let repository = self.krate["repository"].as_str().unwrap_or(empty);
        let license = self.krate["license"].as_str().unwrap_or(empty);

        let keywords = self.krate["keywords"]
            .members()
            .filter_map(|jv| jv.as_str())
            .collect::<Vec<_>>();

        if f.alternate() {
            write!(f,
                   "{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
                   format_args!("{:<16}{}", "Crate:", name),
                   format_args!("{:<16}{}", "Version:", max_version),
                   format_args!("{:<16}{}", "Description:", description),
                   format_args!("{:<16}{}", "Downloads:", downloads),
                   format_args!("{:<16}{}", "Homepage:", homepage),
                   format_args!("{:<16}{}", "Documentation:", documentation),
                   format_args!("{:<16}{}", "Repository:", repository),
                   format_args!("{:<16}{}", "License:", license),
                   format_args!("{:<16}{:?}", "Keywords:", keywords),
                   format_args!("{:<16}{}", "Created at:", created_at),
                   format_args!("{:<16}{}", "Updated at:", updated_at))
        } else {
            write!(f,
                   "{}\n{}\n{}\n{}\n{}\n{}\n{}",
                   format_args!("{:<16}{}", "Crate:", name),
                   format_args!("{:<16}{}", "Version:", max_version),
                   format_args!("{:<16}{}", "Description:", description),
                   format_args!("{:<16}{}", "Downloads:", downloads),
                   format_args!("{:<16}{}", "Homepage:", homepage),
                   format_args!("{:<16}{}", "Documentation:", documentation),
                   format_args!("{:<16}{}", "Repository:", repository))
        }
    }
}
<|endoftext|>


<commit_before>use num::Num;

#[inline(always)]
pub fn create<T: Num>(m11: T, m12: T, m21: T, m22: T) -> [T; 4] {[m11, m12, m21, m22]}
#[test]
fn test_create() {
    let m = create(1, 0, 0, 1);
    assert!(m[0] == 1);
    assert!(m[1] == 0);
    assert!(m[2] == 0);
    assert!(m[3] == 1);
}

#[inline(always)]
pub fn clone<T: Num>(m: [T; 4]) -> [T; 4] {create(m[0], m[1], m[2], m[3])}

#[inline(always)]
pub fn copy<T: Num>(out: &mut [T; 4], a: [T; 4]) -> &mut [T; 4] {
    out[0] = a[0];
    out[1] = a[1];
    out[2] = a[2];
    out[3] = a[3];
    out
}
#[test]
fn test_copy() {
    let mut v = [0, 0, 0, 0];
    copy(&mut v, [1, 2, 3, 4]);
    assert!(v == [1, 2, 3, 4]);
}
<commit_msg>updates<commit_after>use num::Num;

#[inline(always)]
pub fn create<T: Num>(m11: T, m12: T, m21: T, m22: T) -> [T; 4] {[m11, m12, m21, m22]}
#[test]
fn test_create() {
    let m = create(1, 0, 0, 1);
    assert!(m[0] == 1);
    assert!(m[1] == 0);
    assert!(m[2] == 0);
    assert!(m[3] == 1);
}

#[inline(always)]
pub fn create_identity<T: Num>() -> [T; 4] {
    create(
        T::one(), T::zero(),
        T::zero(), T::one()
    )
}
#[inline(always)]
pub fn create_zero<T: Num>() -> [T; 4] {
    create(
        T::zero(), T::zero(),
        T::zero(), T::zero()
    )
}

#[inline(always)]
pub fn clone<T: Num>(m: [T; 4]) -> [T; 4] {create(m[0], m[1], m[2], m[3])}

#[inline(always)]
pub fn copy<T: Num>(out: &mut [T; 4], a: [T; 4]) -> &mut [T; 4] {
    out[0] = a[0];
    out[1] = a[1];
    out[2] = a[2];
    out[3] = a[3];
    out
}
#[test]
fn test_copy() {
    let mut v = [0, 0, 0, 0];
    copy(&mut v, [1, 2, 3, 4]);
    assert!(v == [1, 2, 3, 4]);
}
<|endoftext|>


<commit_before>use std::{cmp, iter, mem, raw, task};

/// Parallelizes an `operation` over a mutable slice
///
/// The `data` will be divided in chunks of `granularity` size. A new task will be spawned to
/// "operate" over each chunk.
///
/// `operation` will receive two arguments:
///
/// - The mutable chunk of data, and
/// - The offset of this chunk from the start of `data`
///
/// # Panics
///
/// Panics if any of the underlying tasks panics
///
/// # Example
///
/// Parallel map
///
/// ```
/// use std::rand::{Rng, XorShiftRng, mod};
///
/// let ref mut rng: XorShiftRng = rand::task_rng().gen();
/// let mut v = Vec::from_fn(1_000, |_| rng.gen::<f32>());
/// # let w = v.iter().map(|x| x.sin()).collect();
/// parallel::divide(v.as_mut_slice(), 100, |data, _| {
///     for x in data.iter_mut() {
///         *x = x.sin();
///     }
/// });
/// # assert_eq!(v, w);
/// ```
// TODO (rust-lang/rfcs#387) Use unboxed closures
pub fn divide<'a, T>(
    data: &mut [T],
    granularity: uint,
    operation: |&mut [T], uint|:'a + Sync,
) where
    T: Send,
{
    assert!(granularity > 0);

    let raw::Slice { data, len } = unsafe { mem::transmute::<_, raw::Slice<T>>(data) };
    let closure = unsafe { mem::transmute::<_, raw::Closure>(operation) };

    let futures = iter::range_step(0, len, granularity).map(|offset| {
        task::try_future(proc() {
            // NB Is safe to send the slice/closure because the task won't outlive this function
            let slice = raw::Slice {
                data: unsafe { data.offset(offset as int) },
                len: cmp::min(granularity, len - offset)
            };
            let data = unsafe { mem::transmute::<_, &mut [T]>(slice) };
            let operation = unsafe { mem::transmute::<_, |&mut [T], uint|:'a + Sync>(closure) };

            operation(data, offset);
        })
    }).collect::<Vec<_>>();

    for future in futures.into_iter() {
        if future.unwrap().is_err() {
            panic!();
        }
    }
}

#[cfg(test)]
mod test {
    use quickcheck::TestResult;
    use std::rand::{Rng, XorShiftRng, mod};

    #[quickcheck]
    fn clone(size: uint, granularity: uint) -> TestResult {
        if granularity == 0 {
            return TestResult::discard();
        }

        let mut rng: XorShiftRng = rand::task_rng().gen();
        let original = Vec::from_fn(size, |_| rng.gen::<f64>());
        let mut clone = Vec::from_elem(size, 0f64);

        let original_slice = original[];
        super::divide(clone[mut], granularity, |data, offset| {
            for (i, x) in data.iter_mut().enumerate() {
                *x = original_slice[offset + i]
            }
        });

        TestResult::from_bool(original == clone)
    }

    #[quickcheck]
    fn new(size: uint, granularity: uint) -> TestResult {
        if granularity == 0 {
            return TestResult::discard();
        }

        let mut v = Vec::from_elem(size, None::<f64>);

        super::divide(v[mut], granularity, |data, _| {
            let mut rng: XorShiftRng = rand::task_rng().gen();

            for x in data.iter_mut() {
                *x = Some(rng.gen())
            }
        });

        TestResult::from_bool(v.iter().all(|&x| match x {
            None => false,
            Some(x) => x > 0. && x < 1.
        }))
    }
}
<commit_msg>fix(doctest): fix fallout of numeric reform<commit_after>use std::{cmp, iter, mem, raw, task};

/// Parallelizes an `operation` over a mutable slice
///
/// The `data` will be divided in chunks of `granularity` size. A new task will be spawned to
/// "operate" over each chunk.
///
/// `operation` will receive two arguments:
///
/// - The mutable chunk of data, and
/// - The offset of this chunk from the start of `data`
///
/// # Panics
///
/// Panics if any of the underlying tasks panics
///
/// # Example
///
/// Parallel map
///
/// ```
/// use std::num::FloatMath;
/// use std::rand::{Rng, XorShiftRng, mod};
///
/// let ref mut rng: XorShiftRng = rand::task_rng().gen();
/// let mut v = Vec::from_fn(1_000, |_| rng.gen::<f32>());
/// # let w = v.iter().map(|x| x.sin()).collect();
/// parallel::divide(v.as_mut_slice(), 100, |data, _| {
///     for x in data.iter_mut() {
///         *x = x.sin();
///     }
/// });
/// # assert_eq!(v, w);
/// ```
// TODO (rust-lang/rfcs#387) Use unboxed closures
pub fn divide<'a, T>(
    data: &mut [T],
    granularity: uint,
    operation: |&mut [T], uint|:'a + Sync,
) where
    T: Send,
{
    assert!(granularity > 0);

    let raw::Slice { data, len } = unsafe { mem::transmute::<_, raw::Slice<T>>(data) };
    let closure = unsafe { mem::transmute::<_, raw::Closure>(operation) };

    let futures = iter::range_step(0, len, granularity).map(|offset| {
        task::try_future(proc() {
            // NB Is safe to send the slice/closure because the task won't outlive this function
            let slice = raw::Slice {
                data: unsafe { data.offset(offset as int) },
                len: cmp::min(granularity, len - offset)
            };
            let data = unsafe { mem::transmute::<_, &mut [T]>(slice) };
            let operation = unsafe { mem::transmute::<_, |&mut [T], uint|:'a + Sync>(closure) };

            operation(data, offset);
        })
    }).collect::<Vec<_>>();

    for future in futures.into_iter() {
        if future.unwrap().is_err() {
            panic!();
        }
    }
}

#[cfg(test)]
mod test {
    use quickcheck::TestResult;
    use std::rand::{Rng, XorShiftRng, mod};

    #[quickcheck]
    fn clone(size: uint, granularity: uint) -> TestResult {
        if granularity == 0 {
            return TestResult::discard();
        }

        let mut rng: XorShiftRng = rand::task_rng().gen();
        let original = Vec::from_fn(size, |_| rng.gen::<f64>());
        let mut clone = Vec::from_elem(size, 0f64);

        let original_slice = original[];
        super::divide(clone[mut], granularity, |data, offset| {
            for (i, x) in data.iter_mut().enumerate() {
                *x = original_slice[offset + i]
            }
        });

        TestResult::from_bool(original == clone)
    }

    #[quickcheck]
    fn new(size: uint, granularity: uint) -> TestResult {
        if granularity == 0 {
            return TestResult::discard();
        }

        let mut v = Vec::from_elem(size, None::<f64>);

        super::divide(v[mut], granularity, |data, _| {
            let mut rng: XorShiftRng = rand::task_rng().gen();

            for x in data.iter_mut() {
                *x = Some(rng.gen())
            }
        });

        TestResult::from_bool(v.iter().all(|&x| match x {
            None => false,
            Some(x) => x > 0. && x < 1.
        }))
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use backup_service::start_backup_service;
use consensus::{consensus_provider::start_consensus, gen_consensus_reconfig_subscription};
use debug_interface::node_debug_service::NodeDebugService;
use executor::{db_bootstrapper::bootstrap_db_if_empty, Executor};
use executor_types::ChunkExecutor;
use futures::{channel::mpsc::channel, executor::block_on};
use libra_config::{
    config::{DiscoveryMethod, NetworkConfig, NodeConfig, RoleType},
    utils::get_genesis_txn,
};
use libra_json_rpc::bootstrap_from_config as bootstrap_rpc;
use libra_logger::prelude::*;
use libra_mempool::gen_mempool_reconfig_subscription;
use libra_metrics::metric_server;
use libra_secure_storage::config;
use libra_types::waypoint::Waypoint;
use libra_vm::LibraVM;
use libradb::LibraDB;
use network::validator_network::network_builder::{AuthenticationMode, NetworkBuilder};
use network_simple_onchain_discovery::{
    gen_simple_discovery_reconfig_subscription, ConfigurationChangeListener,
};
use onchain_discovery::builder::OnchainDiscoveryBuilder;
use state_synchronizer::StateSynchronizer;
use std::{boxed::Box, collections::HashMap, net::ToSocketAddrs, sync::Arc, thread, time::Instant};
use storage_interface::{DbReader, DbReaderWriter};
use storage_service::start_storage_service_with_db;
use tokio::runtime::{Builder, Runtime};

const AC_SMP_CHANNEL_BUFFER_SIZE: usize = 1_024;
const INTRA_NODE_CHANNEL_BUFFER_SIZE: usize = 1;

pub struct LibraHandle {
    _rpc: Runtime,
    _mempool: Runtime,
    _state_synchronizer: StateSynchronizer,
    _network_runtimes: Vec<Runtime>,
    _consensus_runtime: Option<Runtime>,
    _debug: NodeDebugService,
    _backup: Runtime,
}

fn setup_chunk_executor(db: DbReaderWriter) -> Box<dyn ChunkExecutor> {
    Box::new(Executor::<LibraVM>::new(db))
}

fn setup_debug_interface(config: &NodeConfig) -> NodeDebugService {
    let addr = format!(
        "{}:{}",
        config.debug_interface.address, config.debug_interface.admission_control_node_debug_port,
    )
    .to_socket_addrs()
    .unwrap()
    .next()
    .unwrap();

    NodeDebugService::new(addr)
}

// TODO(abhayb): Move to network crate (similar to consensus).
pub fn setup_network(
    config: &mut NetworkConfig,
    role: RoleType,
    libra_db: Arc<dyn DbReader>,
    waypoint: Waypoint,
) -> (Runtime, NetworkBuilder) {
    let runtime = Builder::new()
        .thread_name("network-")
        .threaded_scheduler()
        .enable_all()
        .build()
        .expect("Failed to start runtime. Won't be able to start networking.");

    let identity_key = config::identity_key(config);
    let peer_id = config::peer_id(config);

    let mut network_builder = NetworkBuilder::new(
        runtime.handle().clone(),
        config.network_id.clone(),
        peer_id,
        role,
        config.listen_address.clone(),
    );
    network_builder.add_connection_monitoring();

    if config.enable_remote_authentication {
        // Sanity check seed peer addresses.
        config
            .seed_peers
            .verify_libranet_addrs()
            .expect("Seed peer addresses must be well-formed");

        let network_peers = config.network_peers.peers.clone();
        let seed_peers = config.seed_peers.seed_peers.clone();

        let trusted_peers = if role == RoleType::Validator {
            // for validators, trusted_peers is empty will be populated from consensus
            HashMap::new()
        } else {
            network_peers
        };

        info!(
            "network setup: role: {}, seed_peers: {:?}, trusted_peers: {:?}",
            role, seed_peers, trusted_peers,
        );

        network_builder
            .advertised_address(config.advertised_address.clone())
            .authentication_mode(AuthenticationMode::Mutual(identity_key))
            .trusted_peers(trusted_peers)
            .seed_peers(seed_peers)
            .connectivity_check_interval_ms(config.connectivity_check_interval_ms)
            // TODO:  Why is the connectivity manager related to remote_authentication?
            .add_connectivity_manager();
    } else {
        // Even if a network end-point operates without remote authentication, it might want to prove
        // its identity to another peer it connects to. For this, we use TCP + Noise but without
        // enforcing a trusted peers set.
        network_builder
            .authentication_mode(AuthenticationMode::ServerOnly(identity_key))
            .advertised_address(config.advertised_address.clone());
    }

    match config.discovery_method {
        DiscoveryMethod::Gossip => {
            network_builder
                .discovery_interval_ms(config.discovery_interval_ms)
                .add_gossip_discovery();
        }
        DiscoveryMethod::Onchain => {
            let (network_tx, discovery_events) =
                onchain_discovery::network_interface::add_to_network(&mut network_builder);
            let onchain_discovery_builder = OnchainDiscoveryBuilder::build(
                network_builder
                    .conn_mgr_reqs_tx()
                    .expect("ConnectivityManager must be installed"),
                network_tx,
                discovery_events,
                peer_id,
                role,
                libra_db,
                waypoint,
                runtime.handle(),
            );
            onchain_discovery_builder.start(runtime.handle());
        }
        DiscoveryMethod::None => {}
    }

    (runtime, network_builder)
}

pub fn setup_environment(node_config: &mut NodeConfig) -> LibraHandle {
    crash_handler::setup_panic_handler();

    // Some of our code uses the rayon global thread pool. Name the rayon threads so it doesn't
    // cause confusion, otherwise the threads would have their parent's name.
    rayon::ThreadPoolBuilder::new()
        .thread_name(|index| format!("rayon-global-{}", index))
        .build_global()
        .expect("Building rayon global thread pool should work.");

    let mut instant = Instant::now();
    let (libra_db, db_rw) = DbReaderWriter::wrap(
        LibraDB::open(
            &node_config.storage.dir(),
            false, /* readonly */
            node_config.storage.prune_window,
        )
        .expect("DB should open."),
    );
    let _simple_storage_service =
        start_storage_service_with_db(&node_config, Arc::clone(&libra_db));
    let backup_service = start_backup_service(
        node_config.storage.backup_service_port,
        Arc::clone(&libra_db),
    );

    bootstrap_db_if_empty::<LibraVM>(&db_rw, get_genesis_txn(&node_config).unwrap())
        .expect("Db-bootstrapper should not fail.");

    debug!(
        "Storage service started in {} ms",
        instant.elapsed().as_millis()
    );

    instant = Instant::now();
    let chunk_executor = setup_chunk_executor(db_rw.clone());
    debug!(
        "ChunkExecutor setup in {} ms",
        instant.elapsed().as_millis()
    );
    let mut network_runtimes = vec![];
    let mut state_sync_network_handles = vec![];
    let mut mempool_network_handles = vec![];
    let mut consensus_network_handles = None;
    let mut reconfig_subscriptions = vec![];

    let (mempool_reconfig_subscription, mempool_reconfig_events) =
        gen_mempool_reconfig_subscription();
    reconfig_subscriptions.push(mempool_reconfig_subscription);
    // consensus has to subscribe to ALL on-chain configs
    let (consensus_reconfig_subscription, consensus_reconfig_events) =
        gen_consensus_reconfig_subscription();
    reconfig_subscriptions.push(consensus_reconfig_subscription);

    let waypoint = config::waypoint(&node_config.base.waypoint);

    // Gather all network configs into a single vector.
    // TODO:  consider explicitly encoding the role in the NetworkConfig
    let mut network_configs: Vec<(RoleType, &mut NetworkConfig)> = node_config
        .full_node_networks
        .iter_mut()
        .map(|network_config| (RoleType::FullNode, network_config))
        .collect();
    if let Some(network_config) = node_config.validator_network.as_mut() {
        network_configs.push((RoleType::Validator, network_config));
    }

    let mut validator_count = 0;
    // Instantiate every network and collect the requisite endpoints for state_sync, mempool, and consensus.
    for (role, network_config) in network_configs {
        // Perform common instantiation steps
        let (runtime, mut network_builder) =
            setup_network(network_config, role, Arc::clone(&db_rw.reader), waypoint);
        let peer_id = network_builder.peer_id();

        // Create the endpoints to connect the Network to StateSynchronizer.
        let (state_sync_sender, state_sync_events) =
            state_synchronizer::network::add_to_network(&mut network_builder);
        state_sync_network_handles.push((peer_id, state_sync_sender, state_sync_events));

        // Create the endpoints to connect the network to MemPool.
        let (mempool_sender, mempool_events) = libra_mempool::network::add_to_network(
            &mut network_builder,
            node_config.mempool.max_broadcasts_per_peer,
        );
        mempool_network_handles.push((peer_id, mempool_sender, mempool_events));

        match role {
            // Perform steps relevant specifically to Validator networks.
            RoleType::Validator => {
                validator_count += 1;
                // A valid config is allowed to have exactly one ValidatorNetwork
                assert_eq!(1, validator_count);
                // Set up to listen for network configuration changes from StateSync.
                // TODO:  move this inside network_builder.
                if let Some(conn_mgr_reqs_tx) = network_builder.conn_mgr_reqs_tx() {
                    let (simple_discovery_reconfig_subscription, simple_discovery_reconfig_rx) =
                        gen_simple_discovery_reconfig_subscription();
                    reconfig_subscriptions.push(simple_discovery_reconfig_subscription);
                    let network_config_listener =
                        ConfigurationChangeListener::new(conn_mgr_reqs_tx, RoleType::Validator);
                    runtime
                        .handle()
                        .spawn(network_config_listener.start(simple_discovery_reconfig_rx));
                };

                consensus_network_handles = Some(consensus::network_interface::add_to_network(
                    &mut network_builder,
                ));
            }
            // Currently no FullNode network specific steps.
            RoleType::FullNode => (),
        }

        // Start the network and cache the runtime so it does not go out of scope.
        // TODO:  move all 'start' commands to a second phase at the end of setup_environment.  Target is to have one pass to wire the pieces together and a second pass to start processing in an appropriate order.
        let _listen_addr = network_builder.build();
        network_runtimes.push(runtime);
        debug!("Network started for peer_id: {}", peer_id);
    }

    // TODO set up on-chain discovery network based on UpstreamConfig.fallback_network
    // and pass network handles to mempool/state sync

    // for state sync to send requests to mempool
    let (state_sync_to_mempool_sender, state_sync_requests) =
        channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);
    let state_synchronizer = StateSynchronizer::bootstrap(
        state_sync_network_handles,
        state_sync_to_mempool_sender,
        Arc::clone(&db_rw.reader),
        chunk_executor,
        &node_config,
        waypoint,
        reconfig_subscriptions,
    );
    let (mp_client_sender, mp_client_events) = channel(AC_SMP_CHANNEL_BUFFER_SIZE);

    let rpc_runtime = bootstrap_rpc(&node_config, libra_db.clone(), mp_client_sender);

    let mut consensus_runtime = None;
    let (consensus_to_mempool_sender, consensus_requests) = channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);

    instant = Instant::now();
    let mempool = libra_mempool::bootstrap(
        node_config,
        Arc::clone(&db_rw.reader),
        mempool_network_handles,
        mp_client_events,
        consensus_requests,
        state_sync_requests,
        mempool_reconfig_events,
    );
    debug!("Mempool started in {} ms", instant.elapsed().as_millis());

    // Note: We need to start network provider before consensus, because the consensus
    // initialization is blocked on state synchronizer to sync to the initial root ledger
    // info, which in turn cannot make progress before network initialization
    // because the NewPeer events which state synchronizer uses to know its
    // peers are delivered by network provider. If we were to start network
    // provider after consensus, we create a cyclic dependency from
    // network provider -> consensus -> state synchronizer -> network provider. This deadlock
    // was observed in GitHub Issue #749. A long term fix might be make
    // consensus initialization async instead of blocking on state synchronizer.
    if let Some((consensus_network_sender, consensus_network_events)) = consensus_network_handles {
        // Make sure that state synchronizer is caught up at least to its waypoint
        // (in case it's present). There is no sense to start consensus prior to that.
        // TODO: Note that we need the networking layer to be able to discover & connect to the
        // peers with potentially outdated network identity public keys.
        debug!("Wait until state synchronizer is initialized");
        block_on(state_synchronizer.wait_until_initialized())
            .expect("State synchronizer initialization failure");
        debug!("State synchronizer initialization complete.");

        // Initialize and start consensus.
        instant = Instant::now();
        consensus_runtime = Some(start_consensus(
            node_config,
            consensus_network_sender,
            consensus_network_events,
            state_synchronizer.create_client(),
            consensus_to_mempool_sender,
            libra_db,
            consensus_reconfig_events,
        ));
        debug!("Consensus started in {} ms", instant.elapsed().as_millis());
    }

    let debug_if = setup_debug_interface(&node_config);

    let metrics_port = node_config.debug_interface.metrics_server_port;
    let metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || metric_server::start_server(metric_host, metrics_port, false));
    let public_metrics_port = node_config.debug_interface.public_metrics_server_port;
    let public_metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || {
        metric_server::start_server(public_metric_host, public_metrics_port, true)
    });

    LibraHandle {
        _network_runtimes: network_runtimes,
        _rpc: rpc_runtime,
        _mempool: mempool,
        _state_synchronizer: state_synchronizer,
        _consensus_runtime: consensus_runtime,
        _debug: debug_if,
        _backup: backup_service,
    }
}
<commit_msg>[node] eliminate awkward counting of validator networks<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use backup_service::start_backup_service;
use consensus::{consensus_provider::start_consensus, gen_consensus_reconfig_subscription};
use debug_interface::node_debug_service::NodeDebugService;
use executor::{db_bootstrapper::bootstrap_db_if_empty, Executor};
use executor_types::ChunkExecutor;
use futures::{channel::mpsc::channel, executor::block_on};
use libra_config::{
    config::{DiscoveryMethod, NetworkConfig, NodeConfig, RoleType},
    utils::get_genesis_txn,
};
use libra_json_rpc::bootstrap_from_config as bootstrap_rpc;
use libra_logger::prelude::*;
use libra_mempool::gen_mempool_reconfig_subscription;
use libra_metrics::metric_server;
use libra_secure_storage::config;
use libra_types::waypoint::Waypoint;
use libra_vm::LibraVM;
use libradb::LibraDB;
use network::validator_network::network_builder::{AuthenticationMode, NetworkBuilder};
use network_simple_onchain_discovery::{
    gen_simple_discovery_reconfig_subscription, ConfigurationChangeListener,
};
use onchain_discovery::builder::OnchainDiscoveryBuilder;
use state_synchronizer::StateSynchronizer;
use std::{boxed::Box, collections::HashMap, net::ToSocketAddrs, sync::Arc, thread, time::Instant};
use storage_interface::{DbReader, DbReaderWriter};
use storage_service::start_storage_service_with_db;
use tokio::runtime::{Builder, Runtime};

const AC_SMP_CHANNEL_BUFFER_SIZE: usize = 1_024;
const INTRA_NODE_CHANNEL_BUFFER_SIZE: usize = 1;

pub struct LibraHandle {
    _rpc: Runtime,
    _mempool: Runtime,
    _state_synchronizer: StateSynchronizer,
    _network_runtimes: Vec<Runtime>,
    _consensus_runtime: Option<Runtime>,
    _debug: NodeDebugService,
    _backup: Runtime,
}

fn setup_chunk_executor(db: DbReaderWriter) -> Box<dyn ChunkExecutor> {
    Box::new(Executor::<LibraVM>::new(db))
}

fn setup_debug_interface(config: &NodeConfig) -> NodeDebugService {
    let addr = format!(
        "{}:{}",
        config.debug_interface.address, config.debug_interface.admission_control_node_debug_port,
    )
    .to_socket_addrs()
    .unwrap()
    .next()
    .unwrap();

    NodeDebugService::new(addr)
}

// TODO(abhayb): Move to network crate (similar to consensus).
pub fn setup_network(
    config: &mut NetworkConfig,
    role: RoleType,
    libra_db: Arc<dyn DbReader>,
    waypoint: Waypoint,
) -> (Runtime, NetworkBuilder) {
    let runtime = Builder::new()
        .thread_name("network-")
        .threaded_scheduler()
        .enable_all()
        .build()
        .expect("Failed to start runtime. Won't be able to start networking.");

    let identity_key = config::identity_key(config);
    let peer_id = config::peer_id(config);

    let mut network_builder = NetworkBuilder::new(
        runtime.handle().clone(),
        config.network_id.clone(),
        peer_id,
        role,
        config.listen_address.clone(),
    );
    network_builder.add_connection_monitoring();

    if config.enable_remote_authentication {
        // Sanity check seed peer addresses.
        config
            .seed_peers
            .verify_libranet_addrs()
            .expect("Seed peer addresses must be well-formed");

        let network_peers = config.network_peers.peers.clone();
        let seed_peers = config.seed_peers.seed_peers.clone();

        let trusted_peers = if role == RoleType::Validator {
            // for validators, trusted_peers is empty will be populated from consensus
            HashMap::new()
        } else {
            network_peers
        };

        info!(
            "network setup: role: {}, seed_peers: {:?}, trusted_peers: {:?}",
            role, seed_peers, trusted_peers,
        );

        network_builder
            .advertised_address(config.advertised_address.clone())
            .authentication_mode(AuthenticationMode::Mutual(identity_key))
            .trusted_peers(trusted_peers)
            .seed_peers(seed_peers)
            .connectivity_check_interval_ms(config.connectivity_check_interval_ms)
            // TODO:  Why is the connectivity manager related to remote_authentication?
            .add_connectivity_manager();
    } else {
        // Even if a network end-point operates without remote authentication, it might want to prove
        // its identity to another peer it connects to. For this, we use TCP + Noise but without
        // enforcing a trusted peers set.
        network_builder
            .authentication_mode(AuthenticationMode::ServerOnly(identity_key))
            .advertised_address(config.advertised_address.clone());
    }

    match config.discovery_method {
        DiscoveryMethod::Gossip => {
            network_builder
                .discovery_interval_ms(config.discovery_interval_ms)
                .add_gossip_discovery();
        }
        DiscoveryMethod::Onchain => {
            let (network_tx, discovery_events) =
                onchain_discovery::network_interface::add_to_network(&mut network_builder);
            let onchain_discovery_builder = OnchainDiscoveryBuilder::build(
                network_builder
                    .conn_mgr_reqs_tx()
                    .expect("ConnectivityManager must be installed"),
                network_tx,
                discovery_events,
                peer_id,
                role,
                libra_db,
                waypoint,
                runtime.handle(),
            );
            onchain_discovery_builder.start(runtime.handle());
        }
        DiscoveryMethod::None => {}
    }

    (runtime, network_builder)
}

pub fn setup_environment(node_config: &mut NodeConfig) -> LibraHandle {
    crash_handler::setup_panic_handler();

    // Some of our code uses the rayon global thread pool. Name the rayon threads so it doesn't
    // cause confusion, otherwise the threads would have their parent's name.
    rayon::ThreadPoolBuilder::new()
        .thread_name(|index| format!("rayon-global-{}", index))
        .build_global()
        .expect("Building rayon global thread pool should work.");

    let mut instant = Instant::now();
    let (libra_db, db_rw) = DbReaderWriter::wrap(
        LibraDB::open(
            &node_config.storage.dir(),
            false, /* readonly */
            node_config.storage.prune_window,
        )
        .expect("DB should open."),
    );
    let _simple_storage_service =
        start_storage_service_with_db(&node_config, Arc::clone(&libra_db));
    let backup_service = start_backup_service(
        node_config.storage.backup_service_port,
        Arc::clone(&libra_db),
    );

    bootstrap_db_if_empty::<LibraVM>(&db_rw, get_genesis_txn(&node_config).unwrap())
        .expect("Db-bootstrapper should not fail.");

    debug!(
        "Storage service started in {} ms",
        instant.elapsed().as_millis()
    );

    instant = Instant::now();
    let chunk_executor = setup_chunk_executor(db_rw.clone());
    debug!(
        "ChunkExecutor setup in {} ms",
        instant.elapsed().as_millis()
    );
    let mut network_runtimes = vec![];
    let mut state_sync_network_handles = vec![];
    let mut mempool_network_handles = vec![];
    let mut consensus_network_handles = None;
    let mut reconfig_subscriptions = vec![];

    let (mempool_reconfig_subscription, mempool_reconfig_events) =
        gen_mempool_reconfig_subscription();
    reconfig_subscriptions.push(mempool_reconfig_subscription);
    // consensus has to subscribe to ALL on-chain configs
    let (consensus_reconfig_subscription, consensus_reconfig_events) =
        gen_consensus_reconfig_subscription();
    reconfig_subscriptions.push(consensus_reconfig_subscription);

    let waypoint = config::waypoint(&node_config.base.waypoint);

    // Gather all network configs into a single vector.
    // TODO:  consider explicitly encoding the role in the NetworkConfig
    let mut network_configs: Vec<(RoleType, &mut NetworkConfig)> = node_config
        .full_node_networks
        .iter_mut()
        .map(|network_config| (RoleType::FullNode, network_config))
        .collect();
    if let Some(network_config) = node_config.validator_network.as_mut() {
        network_configs.push((RoleType::Validator, network_config));
    }

    // Instantiate every network and collect the requisite endpoints for state_sync, mempool, and consensus.
    for (role, network_config) in network_configs {
        // Perform common instantiation steps
        let (runtime, mut network_builder) =
            setup_network(network_config, role, Arc::clone(&db_rw.reader), waypoint);
        let peer_id = network_builder.peer_id();

        // Create the endpoints to connect the Network to StateSynchronizer.
        let (state_sync_sender, state_sync_events) =
            state_synchronizer::network::add_to_network(&mut network_builder);
        state_sync_network_handles.push((peer_id, state_sync_sender, state_sync_events));

        // Create the endpoints to connect the network to MemPool.
        let (mempool_sender, mempool_events) = libra_mempool::network::add_to_network(
            &mut network_builder,
            node_config.mempool.max_broadcasts_per_peer,
        );
        mempool_network_handles.push((peer_id, mempool_sender, mempool_events));

        match role {
            // Perform steps relevant specifically to Validator networks.
            RoleType::Validator => {
                // A valid config is allowed to have at most one ValidatorNetwork
                // TODO:  `expect_none` would be perfect here, once it is stable.
                if consensus_network_handles.is_some() {
                    panic!("There can be at most one validator network!");
                }

                // Set up to listen for network configuration changes from StateSync.
                // TODO:  move this inside network_builder.
                if let Some(conn_mgr_reqs_tx) = network_builder.conn_mgr_reqs_tx() {
                    let (simple_discovery_reconfig_subscription, simple_discovery_reconfig_rx) =
                        gen_simple_discovery_reconfig_subscription();
                    reconfig_subscriptions.push(simple_discovery_reconfig_subscription);
                    let network_config_listener =
                        ConfigurationChangeListener::new(conn_mgr_reqs_tx, RoleType::Validator);
                    runtime
                        .handle()
                        .spawn(network_config_listener.start(simple_discovery_reconfig_rx));
                };

                consensus_network_handles = Some(consensus::network_interface::add_to_network(
                    &mut network_builder,
                ));
            }
            // Currently no FullNode network specific steps.
            RoleType::FullNode => (),
        }

        // Start the network and cache the runtime so it does not go out of scope.
        // TODO:  move all 'start' commands to a second phase at the end of setup_environment.  Target is to have one pass to wire the pieces together and a second pass to start processing in an appropriate order.
        let _listen_addr = network_builder.build();
        network_runtimes.push(runtime);
        debug!("Network started for peer_id: {}", peer_id);
    }

    // TODO set up on-chain discovery network based on UpstreamConfig.fallback_network
    // and pass network handles to mempool/state sync

    // for state sync to send requests to mempool
    let (state_sync_to_mempool_sender, state_sync_requests) =
        channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);
    let state_synchronizer = StateSynchronizer::bootstrap(
        state_sync_network_handles,
        state_sync_to_mempool_sender,
        Arc::clone(&db_rw.reader),
        chunk_executor,
        &node_config,
        waypoint,
        reconfig_subscriptions,
    );
    let (mp_client_sender, mp_client_events) = channel(AC_SMP_CHANNEL_BUFFER_SIZE);

    let rpc_runtime = bootstrap_rpc(&node_config, libra_db.clone(), mp_client_sender);

    let mut consensus_runtime = None;
    let (consensus_to_mempool_sender, consensus_requests) = channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);

    instant = Instant::now();
    let mempool = libra_mempool::bootstrap(
        node_config,
        Arc::clone(&db_rw.reader),
        mempool_network_handles,
        mp_client_events,
        consensus_requests,
        state_sync_requests,
        mempool_reconfig_events,
    );
    debug!("Mempool started in {} ms", instant.elapsed().as_millis());

    // Note: We need to start network provider before consensus, because the consensus
    // initialization is blocked on state synchronizer to sync to the initial root ledger
    // info, which in turn cannot make progress before network initialization
    // because the NewPeer events which state synchronizer uses to know its
    // peers are delivered by network provider. If we were to start network
    // provider after consensus, we create a cyclic dependency from
    // network provider -> consensus -> state synchronizer -> network provider. This deadlock
    // was observed in GitHub Issue #749. A long term fix might be make
    // consensus initialization async instead of blocking on state synchronizer.
    if let Some((consensus_network_sender, consensus_network_events)) = consensus_network_handles {
        // Make sure that state synchronizer is caught up at least to its waypoint
        // (in case it's present). There is no sense to start consensus prior to that.
        // TODO: Note that we need the networking layer to be able to discover & connect to the
        // peers with potentially outdated network identity public keys.
        debug!("Wait until state synchronizer is initialized");
        block_on(state_synchronizer.wait_until_initialized())
            .expect("State synchronizer initialization failure");
        debug!("State synchronizer initialization complete.");

        // Initialize and start consensus.
        instant = Instant::now();
        consensus_runtime = Some(start_consensus(
            node_config,
            consensus_network_sender,
            consensus_network_events,
            state_synchronizer.create_client(),
            consensus_to_mempool_sender,
            libra_db,
            consensus_reconfig_events,
        ));
        debug!("Consensus started in {} ms", instant.elapsed().as_millis());
    }

    let debug_if = setup_debug_interface(&node_config);

    let metrics_port = node_config.debug_interface.metrics_server_port;
    let metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || metric_server::start_server(metric_host, metrics_port, false));
    let public_metrics_port = node_config.debug_interface.public_metrics_server_port;
    let public_metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || {
        metric_server::start_server(public_metric_host, public_metrics_port, true)
    });

    LibraHandle {
        _network_runtimes: network_runtimes,
        _rpc: rpc_runtime,
        _mempool: mempool,
        _state_synchronizer: state_synchronizer,
        _consensus_runtime: consensus_runtime,
        _debug: debug_if,
        _backup: backup_service,
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use rpc_capnp::{message, return_};

use std;
use std::io::Acceptor;
use std::collections::hash_map::HashMap;
use capnp::{any_pointer, MessageBuilder, MallocMessageBuilder};
use capnp::capability::{ClientHook, FromClientHook, Server};
use rpc::{RpcConnectionState, RpcEvent, SturdyRefRestorer};
use capability::{LocalClient};

pub struct EzRpcClient {
    rpc_chan : std::comm::Sender<RpcEvent>,
    tcp : std::io::net::tcp::TcpStream,
}

impl Drop for EzRpcClient {
    fn drop(&mut self) {
        self.rpc_chan.send_opt(RpcEvent::Shutdown).is_ok();
        self.tcp.close_read().is_ok();
    }
}

impl EzRpcClient {
    pub fn new(server_address : &str) -> std::io::IoResult<EzRpcClient> {
        use std::io::net::{ip, tcp};

        let addr : ip::SocketAddr = std::str::FromStr::from_str(server_address).expect("bad server address");

        let tcp = try!(tcp::TcpStream::connect(addr));

        let connection_state = RpcConnectionState::new();

        let chan = connection_state.run(tcp.clone(), tcp.clone(), ());

        return Ok(EzRpcClient { rpc_chan : chan, tcp : tcp });
    }

    pub fn import_cap<T : FromClientHook>(&mut self, name : &str) -> T {
        let mut message = box MallocMessageBuilder::new_default();
        {
            let restore = message.init_root::<message::Builder>().init_bootstrap();
            restore.init_deprecated_object_id().set_as_text(name);
        }

        let (outgoing, answer_port, _question_port) = RpcEvent::new_outgoing(message);
        self.rpc_chan.send(RpcEvent::Outgoing(outgoing));

        let mut response_hook = answer_port.recv();
        let message : message::Reader = response_hook.get().get_as();
        let client = match message.which() {
            Some(message::Return(ret)) => {
                match ret.which() {
                    Some(return_::Results(payload)) => {
                        payload.get_content().get_as_capability::<T>()
                    }
                    _ => { panic!() }
                }
            }
            _ => {panic!()}
        };

        return client;
    }
}

enum ExportEvent {
    Restore(String, std::comm::Sender<Option<Box<ClientHook+Send>>>),
    Register(String, Box<Server+Send>),
}

struct ExportedCaps {
    objects : HashMap<String, Box<ClientHook+Send>>,
}

impl ExportedCaps {
    pub fn new() -> std::comm::Sender<ExportEvent> {
        let (chan, port) = std::comm::channel::<ExportEvent>();

        std::task::spawn(proc() {
                let mut vat = ExportedCaps { objects : HashMap::new() };

                loop {
                    match port.recv_opt() {
                        Ok(ExportEvent::Register(name, server)) => {
                            vat.objects.insert(name, box LocalClient::new(server) as Box<ClientHook+Send>);
                        }
                        Ok(ExportEvent::Restore(name, return_chan)) => {
                            return_chan.send(Some(vat.objects[name].copy()));
                        }
                        Err(_) => break,
                    }
                }
            });

        chan
    }
}

pub struct Restorer {
    sender : std::comm::Sender<ExportEvent>,
}

impl Restorer {
    fn new(sender : std::comm::Sender<ExportEvent>) -> Restorer {
        Restorer { sender : sender }
    }
}

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : any_pointer::Reader) -> Option<Box<ClientHook+Send>> {
        let (tx, rx) = std::comm::channel();
        self.sender.send(ExportEvent::Restore(obj_id.get_as::<::capnp::text::Reader>().to_string(), tx));
        return rx.recv();
    }
}

pub struct EzRpcServer {
    sender : std::comm::Sender<ExportEvent>,
    tcp_acceptor : std::io::net::tcp::TcpAcceptor,
}

impl EzRpcServer {
    pub fn new(bind_address : &str) -> std::io::IoResult<EzRpcServer> {
        use std::io::net::{ip, tcp};
        use std::io::Listener;

        let addr : ip::SocketAddr = std::str::FromStr::from_str(bind_address).expect("bad bind address");

        let tcp_listener = try!(tcp::TcpListener::bind(addr));

        let tcp_acceptor = try!(tcp_listener.listen());

        let sender = ExportedCaps::new();

        Ok(EzRpcServer { sender : sender, tcp_acceptor : tcp_acceptor  })
    }

    pub fn export_cap(&self, name : &str, server : Box<Server+Send>) {
        self.sender.send(ExportEvent::Register(name.to_string(), server))
    }

    pub fn serve(self) {
        std::task::spawn(proc() {
            let mut server = self;
            for res in server.incoming() {
                match res {
                    Ok(()) => {}
                    Err(e) => {
                        println!("error: {}", e)
                    }
                }
            }

        });
    }
}

impl std::io::Acceptor<()> for EzRpcServer {
    fn accept(&mut self) -> std::io::IoResult<()> {

        let sender2 = self.sender.clone();
        let tcp = try!(self.tcp_acceptor.accept());
        std::task::spawn(proc() {
            let connection_state = RpcConnectionState::new();
            let _rpc_chan = connection_state.run(tcp.clone(), tcp, Restorer::new(sender2));
        });
        Ok(())
    }
}
<commit_msg>set_as_text -> set_as<commit_after>/*
 * Copyright (c) 2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use rpc_capnp::{message, return_};

use std;
use std::io::Acceptor;
use std::collections::hash_map::HashMap;
use capnp::{any_pointer, MessageBuilder, MallocMessageBuilder};
use capnp::capability::{ClientHook, FromClientHook, Server};
use rpc::{RpcConnectionState, RpcEvent, SturdyRefRestorer};
use capability::{LocalClient};

pub struct EzRpcClient {
    rpc_chan : std::comm::Sender<RpcEvent>,
    tcp : std::io::net::tcp::TcpStream,
}

impl Drop for EzRpcClient {
    fn drop(&mut self) {
        self.rpc_chan.send_opt(RpcEvent::Shutdown).is_ok();
        self.tcp.close_read().is_ok();
    }
}

impl EzRpcClient {
    pub fn new(server_address : &str) -> std::io::IoResult<EzRpcClient> {
        use std::io::net::{ip, tcp};

        let addr : ip::SocketAddr = std::str::FromStr::from_str(server_address).expect("bad server address");

        let tcp = try!(tcp::TcpStream::connect(addr));

        let connection_state = RpcConnectionState::new();

        let chan = connection_state.run(tcp.clone(), tcp.clone(), ());

        return Ok(EzRpcClient { rpc_chan : chan, tcp : tcp });
    }

    pub fn import_cap<T : FromClientHook>(&mut self, name : &str) -> T {
        let mut message = box MallocMessageBuilder::new_default();
        {
            let restore = message.init_root::<message::Builder>().init_bootstrap();
            restore.init_deprecated_object_id().set_as(name);
        }

        let (outgoing, answer_port, _question_port) = RpcEvent::new_outgoing(message);
        self.rpc_chan.send(RpcEvent::Outgoing(outgoing));

        let mut response_hook = answer_port.recv();
        let message : message::Reader = response_hook.get().get_as();
        let client = match message.which() {
            Some(message::Return(ret)) => {
                match ret.which() {
                    Some(return_::Results(payload)) => {
                        payload.get_content().get_as_capability::<T>()
                    }
                    _ => { panic!() }
                }
            }
            _ => {panic!()}
        };

        return client;
    }
}

enum ExportEvent {
    Restore(String, std::comm::Sender<Option<Box<ClientHook+Send>>>),
    Register(String, Box<Server+Send>),
}

struct ExportedCaps {
    objects : HashMap<String, Box<ClientHook+Send>>,
}

impl ExportedCaps {
    pub fn new() -> std::comm::Sender<ExportEvent> {
        let (chan, port) = std::comm::channel::<ExportEvent>();

        std::task::spawn(proc() {
                let mut vat = ExportedCaps { objects : HashMap::new() };

                loop {
                    match port.recv_opt() {
                        Ok(ExportEvent::Register(name, server)) => {
                            vat.objects.insert(name, box LocalClient::new(server) as Box<ClientHook+Send>);
                        }
                        Ok(ExportEvent::Restore(name, return_chan)) => {
                            return_chan.send(Some(vat.objects[name].copy()));
                        }
                        Err(_) => break,
                    }
                }
            });

        chan
    }
}

pub struct Restorer {
    sender : std::comm::Sender<ExportEvent>,
}

impl Restorer {
    fn new(sender : std::comm::Sender<ExportEvent>) -> Restorer {
        Restorer { sender : sender }
    }
}

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : any_pointer::Reader) -> Option<Box<ClientHook+Send>> {
        let (tx, rx) = std::comm::channel();
        self.sender.send(ExportEvent::Restore(obj_id.get_as::<::capnp::text::Reader>().to_string(), tx));
        return rx.recv();
    }
}

pub struct EzRpcServer {
    sender : std::comm::Sender<ExportEvent>,
    tcp_acceptor : std::io::net::tcp::TcpAcceptor,
}

impl EzRpcServer {
    pub fn new(bind_address : &str) -> std::io::IoResult<EzRpcServer> {
        use std::io::net::{ip, tcp};
        use std::io::Listener;

        let addr : ip::SocketAddr = std::str::FromStr::from_str(bind_address).expect("bad bind address");

        let tcp_listener = try!(tcp::TcpListener::bind(addr));

        let tcp_acceptor = try!(tcp_listener.listen());

        let sender = ExportedCaps::new();

        Ok(EzRpcServer { sender : sender, tcp_acceptor : tcp_acceptor  })
    }

    pub fn export_cap(&self, name : &str, server : Box<Server+Send>) {
        self.sender.send(ExportEvent::Register(name.to_string(), server))
    }

    pub fn serve(self) {
        std::task::spawn(proc() {
            let mut server = self;
            for res in server.incoming() {
                match res {
                    Ok(()) => {}
                    Err(e) => {
                        println!("error: {}", e)
                    }
                }
            }

        });
    }
}

impl std::io::Acceptor<()> for EzRpcServer {
    fn accept(&mut self) -> std::io::IoResult<()> {

        let sender2 = self.sender.clone();
        let tcp = try!(self.tcp_acceptor.accept());
        std::task::spawn(proc() {
            let connection_state = RpcConnectionState::new();
            let _rpc_chan = connection_state.run(tcp.clone(), tcp, Restorer::new(sender2));
        });
        Ok(())
    }
}
<|endoftext|>


<commit_before>use crate::{channel::Reply, Error, Promise, PromiseResolver};
use amq_protocol::frame::AMQPFrame;
use log::{log_enabled, trace, Level::Trace};
use parking_lot::Mutex;
use pinky_swear::Cancellable;
use std::{
    collections::{HashMap, VecDeque},
    fmt,
    sync::Arc,
};

pub(crate) struct ExpectedReply(
    pub(crate) Reply,
    pub(crate) Box<dyn Cancellable<Error> + Send>,
);

impl fmt::Debug for ExpectedReply {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_tuple("ExpectedReply").field(&self.0).finish()
    }
}

#[derive(Clone, Default)]
pub(crate) struct Frames {
    inner: Arc<Mutex<Inner>>,
}

impl Frames {
    pub(crate) fn push(
        &self,
        channel_id: u16,
        frame: AMQPFrame,
        resolver: PromiseResolver<()>,
        expected_reply: Option<ExpectedReply>,
    ) {
        self.inner
            .lock()
            .push(channel_id, frame, resolver, expected_reply);
    }

    pub(crate) fn push_frames(&self, frames: Vec<AMQPFrame>) -> Promise<()> {
        self.inner.lock().push_frames(frames)
    }

    pub(crate) fn retry(&self, frame: (AMQPFrame, Option<PromiseResolver<()>>)) {
        self.inner.lock().retry_frames.push_back(frame);
    }

    pub(crate) fn pop(&self, flow: bool) -> Option<(AMQPFrame, Option<PromiseResolver<()>>)> {
        self.inner.lock().pop(flow)
    }

    pub(crate) fn next_expected_reply(&self, channel_id: u16) -> Option<Reply> {
        self.inner
            .lock()
            .expected_replies
            .get_mut(&channel_id)
            .and_then(|replies| replies.pop_front())
            .map(|t| t.0)
    }

    pub(crate) fn next_expected_close_ok_reply(&self, channel_id: u16, error: Error) -> Option<Reply> {
        self.inner.lock().next_expected_close_ok_reply(channel_id, error)
    }

    pub(crate) fn has_pending(&self) -> bool {
        self.inner.lock().has_pending()
    }

    pub(crate) fn drop_pending(&self, error: Error) {
        self.inner.lock().drop_pending(error);
    }

    pub(crate) fn clear_expected_replies(&self, channel_id: u16, error: Error) {
        self.inner.lock().clear_expected_replies(channel_id, error);
    }
}

struct Inner {
    /* Header frames must follow basic.publish frames directly, otherwise RabbitMQ-server send us an UNEXPECTED_FRAME */
    /* After sending the Header frame, we need to send the associated Body frames before anything else for the same reason */
    publish_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    retry_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    low_prio_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    expected_replies: HashMap<u16, VecDeque<ExpectedReply>>,
}

impl Default for Inner {
    fn default() -> Self {
        Self {
            publish_frames: VecDeque::default(),
            retry_frames: VecDeque::default(),
            frames: VecDeque::default(),
            low_prio_frames: VecDeque::default(),
            expected_replies: HashMap::default(),
        }
    }
}

impl fmt::Debug for Frames {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_struct("Frames");
        if let Some(inner) = self.inner.try_lock() {
            debug.field("expected_replies", &inner.expected_replies);
        }
        debug.finish()
    }
}

impl Inner {
    fn push(
        &mut self,
        channel_id: u16,
        frame: AMQPFrame,
        resolver: PromiseResolver<()>,
        expected_reply: Option<ExpectedReply>,
    ) {
        self.frames.push_back((frame, Some(resolver)));
        if let Some(reply) = expected_reply {
            trace!(
                "channel {} state is now waiting for {:?}",
                channel_id,
                reply
            );
            self.expected_replies
                .entry(channel_id)
                .or_default()
                .push_back(reply);
        }
    }

    fn push_frames(&mut self, mut frames: Vec<AMQPFrame>) -> Promise<()> {
        let (promise, resolver) = Promise::new();
        let last_frame = frames.pop();

        if log_enabled!(Trace) {
            promise.set_marker("Frames".into());
        }

        for frame in frames {
            self.low_prio_frames.push_back((frame, None));
        }
        if let Some(last_frame) = last_frame {
            self.low_prio_frames.push_back((last_frame, Some(resolver)));
        } else {
            resolver.swear(Ok(()));
        }

        promise
    }

    fn pop(&mut self, flow: bool) -> Option<(AMQPFrame, Option<PromiseResolver<()>>)> {
        if let Some(frame) = self
            .retry_frames
            .pop_front()
            .or_else(|| self.publish_frames.pop_front())
            .or_else(|| self.frames.pop_front())
        {
            return Some(frame);
        }
        if flow {
            if let Some(frame) = self.low_prio_frames.pop_front() {
                // If the next frame is a header, that means we're a basic.publish
                // Header frame needs to follow directly the basic.publish frame, and Body frames
                // need to be sent just after those or the AMQP server will close the connection.
                // Push the header into publish_frames which is there to handle just that.
                if self
                    .low_prio_frames
                    .front()
                    .map(|(frame, _)| frame.is_header())
                    .unwrap_or(false)
                {
                    // Yes, this will always be Some() with a Header frame, but let's keep our unwrap() count low
                    if let Some(next_frame) = self.low_prio_frames.pop_front() {
                        self.publish_frames.push_back(next_frame);
                    }
                    while let Some(next_frame) = self.low_prio_frames.pop_front() {
                        match next_frame.0 {
                            AMQPFrame::Body(..) => {
                                self.publish_frames.push_back(next_frame);
                            }
                            _ => {
                                // We've exhausted Body frames for this publish, push back the next one and exit
                                self.low_prio_frames.push_front(next_frame);
                                break;
                            }
                        }
                    }
                }
                return Some(frame);
            }
        }
        None
    }

    fn has_pending(&self) -> bool {
        !(self.retry_frames.is_empty()
            && self.publish_frames.is_empty()
            && self.frames.is_empty()
            && self.low_prio_frames.is_empty())
    }

    fn drop_pending(&mut self, error: Error) {
        Self::drop_pending_frames(&mut self.retry_frames, error.clone());
        Self::drop_pending_frames(&mut self.publish_frames, error.clone());
        Self::drop_pending_frames(&mut self.frames, error.clone());
        Self::drop_pending_frames(&mut self.low_prio_frames, error.clone());
        for (_, replies) in self.expected_replies.drain() {
            Self::cancel_expected_replies(replies, error.clone());
        }
    }

    fn drop_pending_frames(
        frames: &mut VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
        error: Error,
    ) {
        for (_, resolver) in std::mem::take(frames) {
            if let Some(resolver) = resolver {
                resolver.swear(Err(error.clone()));
            }
        }
    }

    fn next_expected_close_ok_reply(&mut self, channel_id: u16, error: Error) -> Option<Reply> {
        let expected_replies = self.expected_replies.get_mut(&channel_id)?;
        while let Some(reply) = expected_replies.pop_front() {
            match &reply.0 {
                Reply::ChannelCloseOk(_) => return Some(reply.0),
                Reply::BasicCancelOk(pinky) => pinky.swear(Ok(())), // Channel close means consumer is canceled automatically
                _ => reply.1.cancel(error.clone()),
            }
        }
        None
    }

    fn clear_expected_replies(&mut self, channel_id: u16, error: Error) {
        if let Some(replies) = self.expected_replies.remove(&channel_id) {
            Self::cancel_expected_replies(replies, error);
        }
    }

    fn cancel_expected_replies(replies: VecDeque<ExpectedReply>, error: Error) {
        for ExpectedReply(reply, cancel) in replies {
            match reply {
                Reply::BasicCancelOk(pinky) => pinky.swear(Ok(())),
                _ => cancel.cancel(error.clone()),
            }
        }
    }
}
<commit_msg>consider errors for basic cancel as success<commit_after>use crate::{channel::Reply, Error, Promise, PromiseResolver};
use amq_protocol::{frame::AMQPFrame, protocol::{AMQPClass, basic::AMQPMethod}};
use log::{log_enabled, trace, Level::Trace};
use parking_lot::Mutex;
use pinky_swear::Cancellable;
use std::{
    collections::{HashMap, VecDeque},
    fmt,
    sync::Arc,
};

pub(crate) struct ExpectedReply(
    pub(crate) Reply,
    pub(crate) Box<dyn Cancellable<Error> + Send>,
);

impl fmt::Debug for ExpectedReply {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_tuple("ExpectedReply").field(&self.0).finish()
    }
}

#[derive(Clone, Default)]
pub(crate) struct Frames {
    inner: Arc<Mutex<Inner>>,
}

impl Frames {
    pub(crate) fn push(
        &self,
        channel_id: u16,
        frame: AMQPFrame,
        resolver: PromiseResolver<()>,
        expected_reply: Option<ExpectedReply>,
    ) {
        self.inner
            .lock()
            .push(channel_id, frame, resolver, expected_reply);
    }

    pub(crate) fn push_frames(&self, frames: Vec<AMQPFrame>) -> Promise<()> {
        self.inner.lock().push_frames(frames)
    }

    pub(crate) fn retry(&self, frame: (AMQPFrame, Option<PromiseResolver<()>>)) {
        self.inner.lock().retry_frames.push_back(frame);
    }

    pub(crate) fn pop(&self, flow: bool) -> Option<(AMQPFrame, Option<PromiseResolver<()>>)> {
        self.inner.lock().pop(flow)
    }

    pub(crate) fn next_expected_reply(&self, channel_id: u16) -> Option<Reply> {
        self.inner
            .lock()
            .expected_replies
            .get_mut(&channel_id)
            .and_then(|replies| replies.pop_front())
            .map(|t| t.0)
    }

    pub(crate) fn next_expected_close_ok_reply(&self, channel_id: u16, error: Error) -> Option<Reply> {
        self.inner.lock().next_expected_close_ok_reply(channel_id, error)
    }

    pub(crate) fn has_pending(&self) -> bool {
        self.inner.lock().has_pending()
    }

    pub(crate) fn drop_pending(&self, error: Error) {
        self.inner.lock().drop_pending(error);
    }

    pub(crate) fn clear_expected_replies(&self, channel_id: u16, error: Error) {
        self.inner.lock().clear_expected_replies(channel_id, error);
    }
}

struct Inner {
    /* Header frames must follow basic.publish frames directly, otherwise RabbitMQ-server send us an UNEXPECTED_FRAME */
    /* After sending the Header frame, we need to send the associated Body frames before anything else for the same reason */
    publish_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    retry_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    low_prio_frames: VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
    expected_replies: HashMap<u16, VecDeque<ExpectedReply>>,
}

impl Default for Inner {
    fn default() -> Self {
        Self {
            publish_frames: VecDeque::default(),
            retry_frames: VecDeque::default(),
            frames: VecDeque::default(),
            low_prio_frames: VecDeque::default(),
            expected_replies: HashMap::default(),
        }
    }
}

impl fmt::Debug for Frames {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_struct("Frames");
        if let Some(inner) = self.inner.try_lock() {
            debug.field("expected_replies", &inner.expected_replies);
        }
        debug.finish()
    }
}

impl Inner {
    fn push(
        &mut self,
        channel_id: u16,
        frame: AMQPFrame,
        resolver: PromiseResolver<()>,
        expected_reply: Option<ExpectedReply>,
    ) {
        self.frames.push_back((frame, Some(resolver)));
        if let Some(reply) = expected_reply {
            trace!(
                "channel {} state is now waiting for {:?}",
                channel_id,
                reply
            );
            self.expected_replies
                .entry(channel_id)
                .or_default()
                .push_back(reply);
        }
    }

    fn push_frames(&mut self, mut frames: Vec<AMQPFrame>) -> Promise<()> {
        let (promise, resolver) = Promise::new();
        let last_frame = frames.pop();

        if log_enabled!(Trace) {
            promise.set_marker("Frames".into());
        }

        for frame in frames {
            self.low_prio_frames.push_back((frame, None));
        }
        if let Some(last_frame) = last_frame {
            self.low_prio_frames.push_back((last_frame, Some(resolver)));
        } else {
            resolver.swear(Ok(()));
        }

        promise
    }

    fn pop(&mut self, flow: bool) -> Option<(AMQPFrame, Option<PromiseResolver<()>>)> {
        if let Some(frame) = self
            .retry_frames
            .pop_front()
            .or_else(|| self.publish_frames.pop_front())
            .or_else(|| self.frames.pop_front())
        {
            return Some(frame);
        }
        if flow {
            if let Some(frame) = self.low_prio_frames.pop_front() {
                // If the next frame is a header, that means we're a basic.publish
                // Header frame needs to follow directly the basic.publish frame, and Body frames
                // need to be sent just after those or the AMQP server will close the connection.
                // Push the header into publish_frames which is there to handle just that.
                if self
                    .low_prio_frames
                    .front()
                    .map(|(frame, _)| frame.is_header())
                    .unwrap_or(false)
                {
                    // Yes, this will always be Some() with a Header frame, but let's keep our unwrap() count low
                    if let Some(next_frame) = self.low_prio_frames.pop_front() {
                        self.publish_frames.push_back(next_frame);
                    }
                    while let Some(next_frame) = self.low_prio_frames.pop_front() {
                        match next_frame.0 {
                            AMQPFrame::Body(..) => {
                                self.publish_frames.push_back(next_frame);
                            }
                            _ => {
                                // We've exhausted Body frames for this publish, push back the next one and exit
                                self.low_prio_frames.push_front(next_frame);
                                break;
                            }
                        }
                    }
                }
                return Some(frame);
            }
        }
        None
    }

    fn has_pending(&self) -> bool {
        !(self.retry_frames.is_empty()
            && self.publish_frames.is_empty()
            && self.frames.is_empty()
            && self.low_prio_frames.is_empty())
    }

    fn drop_pending(&mut self, error: Error) {
        Self::drop_pending_frames(&mut self.retry_frames, error.clone());
        Self::drop_pending_frames(&mut self.publish_frames, error.clone());
        Self::drop_pending_frames(&mut self.frames, error.clone());
        Self::drop_pending_frames(&mut self.low_prio_frames, error.clone());
        for (_, replies) in self.expected_replies.drain() {
            Self::cancel_expected_replies(replies, error.clone());
        }
    }

    fn drop_pending_frames(
        frames: &mut VecDeque<(AMQPFrame, Option<PromiseResolver<()>>)>,
        error: Error,
    ) {
        for (frame, resolver) in std::mem::take(frames) {
            if let Some(resolver) = resolver {
                match frame {
                    AMQPFrame::Method(_, AMQPClass::Basic(AMQPMethod::Cancel(_))) => resolver.swear(Ok(())),
                    _ => resolver.swear(Err(error.clone())),
                }
            }
        }
    }

    fn next_expected_close_ok_reply(&mut self, channel_id: u16, error: Error) -> Option<Reply> {
        let expected_replies = self.expected_replies.get_mut(&channel_id)?;
        while let Some(reply) = expected_replies.pop_front() {
            match &reply.0 {
                Reply::ChannelCloseOk(_) => return Some(reply.0),
                Reply::BasicCancelOk(pinky) => pinky.swear(Ok(())), // Channel close means consumer is canceled automatically
                _ => reply.1.cancel(error.clone()),
            }
        }
        None
    }

    fn clear_expected_replies(&mut self, channel_id: u16, error: Error) {
        if let Some(replies) = self.expected_replies.remove(&channel_id) {
            Self::cancel_expected_replies(replies, error);
        }
    }

    fn cancel_expected_replies(replies: VecDeque<ExpectedReply>, error: Error) {
        for ExpectedReply(reply, cancel) in replies {
            match reply {
                Reply::BasicCancelOk(pinky) => pinky.swear(Ok(())),
                _ => cancel.cancel(error.clone()),
            }
        }
    }
}
<|endoftext|>


<commit_before>use crate::message::{
    header::{self, ContentType},
    IntoBody, SinglePart,
};

/// `SinglePart` builder for attachments
///
/// Allows building attachment parts easily.
#[derive(Clone)]
pub struct Attachment {
    disposition: Disposition,
}

#[derive(Clone)]
enum Disposition {
    /// file name
    Attached(String),
    /// content id
    Inline(String),
}

impl Attachment {
    /// Creates a new attachment
    pub fn new(filename: String) -> Self {
        Attachment {
            disposition: Disposition::Attached(filename),
        }
    }

    /// Creates a new inline attachment
    pub fn new_inline(content_id: String) -> Self {
        Attachment {
            disposition: Disposition::Inline(content_id),
        }
    }

    /// Build the attachment part
    pub fn body<T: IntoBody>(self, content: T, content_type: ContentType) -> SinglePart {
        let mut builder = SinglePart::builder();
        builder = match self.disposition {
            Disposition::Attached(filename) => {
                builder.header(header::ContentDisposition::attachment(&filename))
            }
            Disposition::Inline(content_id) => builder
                .header(header::ContentId::from(format!("<{}>", content_id)))
                .header(header::ContentDisposition::inline()),
        };
        builder = builder.header(content_type);
        builder.body(content)
    }
}

#[cfg(test)]
mod tests {
    use crate::message::header::ContentType;

    #[test]
    fn attachment() {
        let part = super::Attachment::new(String::from("test.txt")).body(
            String::from("Hello world!"),
            ContentType::parse("text/plain").unwrap(),
        );
        assert_eq!(
            &String::from_utf8_lossy(&part.formatted()),
            concat!(
                "Content-Disposition: attachment; filename=\"test.txt\"\r\n",
                "Content-Type: text/plain\r\n",
                "Content-Transfer-Encoding: 7bit\r\n\r\n",
                "Hello world!\r\n",
            )
        );
    }

    #[test]
    fn attachment_inline() {
        let part = super::Attachment::new_inline(String::from("id")).body(
            String::from("Hello world!"),
            ContentType::parse("text/plain").unwrap(),
        );
        assert_eq!(
            &String::from_utf8_lossy(&part.formatted()),
            concat!(
                "Content-ID: <id>\r\n",
                "Content-Disposition: inline\r\n",
                "Content-Type: text/plain\r\n",
                "Content-Transfer-Encoding: 7bit\r\n\r\n",
                "Hello world!\r\n"
            )
        );
    }
}
<commit_msg>Better document how Attachments can be used (#658)<commit_after>use crate::message::{
    header::{self, ContentType},
    IntoBody, SinglePart,
};

/// `SinglePart` builder for attachments
///
/// Allows building attachment parts easily.
#[derive(Clone)]
pub struct Attachment {
    disposition: Disposition,
}

#[derive(Clone)]
enum Disposition {
    /// file name
    Attached(String),
    /// content id
    Inline(String),
}

impl Attachment {
    /// Create a new attachment
    ///
    /// This attachment will be displayed as a normal attachment,
    /// with the chosen `filename` appearing as the file name.
    ///
    /// ```rust
    /// # use std::error::Error;
    /// use std::fs;
    ///
    /// use lettre::message::{Attachment, header::ContentType};
    ///
    /// # fn main() -> Result<(), Box<dyn Error>> {
    /// let filename = String::from("invoice.pdf");
    /// # if false {
    /// let filebody = fs::read("invoice.pdf")?;
    /// # }
    /// # let filebody = fs::read("docs/lettre.png")?;
    /// let content_type = ContentType::parse("application/pdf").unwrap();
    /// let attachment = Attachment::new(filename).body(filebody, content_type);
    ///
    /// // The document `attachment` will show up as a normal attachment.
    /// # Ok(())
    /// # }
    /// ```
    pub fn new(filename: String) -> Self {
        Attachment {
            disposition: Disposition::Attached(filename),
        }
    }

    /// Create a new inline attachment
    ///
    /// This attachment should be displayed inline into the message
    /// body:
    ///
    /// ```html
    /// <img src="cid:123">
    /// ```
    ///
    ///
    /// ```rust
    /// # use std::error::Error;
    /// use std::fs;
    ///
    /// use lettre::message::{Attachment, header::ContentType};
    ///
    /// # fn main() -> Result<(), Box<dyn Error>> {
    /// let content_id = String::from("123");
    /// # if false {
    /// let filebody = fs::read("image.jpg")?;
    /// # }
    /// # let filebody = fs::read("docs/lettre.png")?;
    /// let content_type = ContentType::parse("image/jpeg").unwrap();
    /// let attachment = Attachment::new_inline(content_id).body(filebody, content_type);
    ///
    /// // The image `attachment` will display inline into the email.
    /// # Ok(())
    /// # }
    /// ```
    pub fn new_inline(content_id: String) -> Self {
        Attachment {
            disposition: Disposition::Inline(content_id),
        }
    }

    /// Build the attachment into a [`SinglePart`] which can then be used to build the rest of the email
    ///
    /// Look at the [Complex MIME body example](crate::message#complex-mime-body)
    /// to see how [`SinglePart`] can be put into the email.
    pub fn body<T: IntoBody>(self, content: T, content_type: ContentType) -> SinglePart {
        let mut builder = SinglePart::builder();
        builder = match self.disposition {
            Disposition::Attached(filename) => {
                builder.header(header::ContentDisposition::attachment(&filename))
            }
            Disposition::Inline(content_id) => builder
                .header(header::ContentId::from(format!("<{}>", content_id)))
                .header(header::ContentDisposition::inline()),
        };
        builder = builder.header(content_type);
        builder.body(content)
    }
}

#[cfg(test)]
mod tests {
    use crate::message::header::ContentType;

    #[test]
    fn attachment() {
        let part = super::Attachment::new(String::from("test.txt")).body(
            String::from("Hello world!"),
            ContentType::parse("text/plain").unwrap(),
        );
        assert_eq!(
            &String::from_utf8_lossy(&part.formatted()),
            concat!(
                "Content-Disposition: attachment; filename=\"test.txt\"\r\n",
                "Content-Type: text/plain\r\n",
                "Content-Transfer-Encoding: 7bit\r\n\r\n",
                "Hello world!\r\n",
            )
        );
    }

    #[test]
    fn attachment_inline() {
        let part = super::Attachment::new_inline(String::from("id")).body(
            String::from("Hello world!"),
            ContentType::parse("text/plain").unwrap(),
        );
        assert_eq!(
            &String::from_utf8_lossy(&part.formatted()),
            concat!(
                "Content-ID: <id>\r\n",
                "Content-Disposition: inline\r\n",
                "Content-Type: text/plain\r\n",
                "Content-Transfer-Encoding: 7bit\r\n\r\n",
                "Hello world!\r\n"
            )
        );
    }
}
<|endoftext|>