<commit_before>// Copyright 2015-2017 Vadim Nazarov, Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Various subroutines for computing a distance between sequences.

use itertools::multizip;
use std::cmp::min;

use utils::TextSlice;

/// Compute the Hamming distance between two strings. Complexity: O(n).
///
/// # Example
///
/// ```
/// use bio::alignment::distance::*;
///
/// let x = b"GTCTGCATGCG";
/// let y = b"TTTAGCTAGCG";
/// // GTCTGCATGCG
/// //  |  ||  |||
/// // TTTAGCTAGCG
/// assert_eq!(hamming(x, y), 5);
/// ```
pub fn hamming(alpha: TextSlice, beta: TextSlice) -> u64 {
    assert_eq!(alpha.len(),
               beta.len(),
               "hamming distance cannot be calculated for texts of different length ({}!={})",
               alpha.len(),
               beta.len());
    let mut dist = 0;
    for (a, b) in multizip((alpha, beta)) {
        if a != b {
            dist += 1;
        }
    }
    dist
}

/// Compute the Levenshtein (or Edit) distance between two strings. Complexity: O(n * m) with
/// n and m being the length of the given texts.
///
/// # Example
///
/// ```
/// use bio::alignment::distance::*;
///
/// let x = b"ACCGTGGAT";
/// let y = b"AAAAACCGTTGAT";
/// // ----ACCGTGGAT
/// //     ||||| |||
/// // AAAAACCGTTGAT
/// let ldist = levenshtein(x, y);  // Distance is 5
/// assert_eq!(ldist, 5);
/// ```
#[allow(unused_assignments)]
pub fn levenshtein(alpha: TextSlice, beta: TextSlice) -> u32 {
    let mut columns = [vec!(0u32; alpha.len() + 1), vec!(0u32; alpha.len() + 1)];
    let mut i_prev = 0;
    let mut i_cur = 1;

    for i in 0..columns[0].len() {
        columns[0][i] = i as u32;
    }

    for (j, item) in beta.iter().enumerate() {
        i_cur %= 2;
        i_prev = 1 - i_cur;

        columns[i_cur][0] = 1 + j as u32;
        for i in 1..columns[0].len() {
            columns[i_cur][i] = min(columns[i_prev][i - 1] +
                                    if alpha[i - 1] == *item { 0 } else { 1 },
                                    min(columns[i_cur][i - 1] + 1, columns[i_prev][i] + 1));
        }

        i_cur += 1;
    }

    columns[i_cur - 1][columns[0].len() - 1]
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_hamming_dist_good() {
        let x = b"GTCTGCATGCG";
        let y = b"TTTAGCTAGCG";
        // GTCTGCATGCG
        //  |  ||  |||
        // TTTAGCTAGCG
        assert_eq!(hamming(x, y), 5);
    }

    #[test]
    #[should_panic(expected = "hamming distance cannot be calculated for texts of different length (11!=8)")]
    fn test_hamming_dist_bad() {
        let x = b"GACTATATCGA";
        let y = b"TTTAGCTC";
        hamming(x, y);
    }

    #[test]
    fn test_levenshtein_dist() {
        let x = b"ACCGTGGAT";
        let y = b"AAAAACCGTTGAT";
        // ----ACCGTGGAT
        //     ||||| |||
        // AAAAACCGTTGAT
        assert_eq!(levenshtein(x, y), 5);
        assert_eq!(levenshtein(x, y), levenshtein(y, x));
        assert_eq!(levenshtein(b"AAA", b"TTTT"), 4);
        assert_eq!(levenshtein(b"TTTT", b"AAA"), 4);
    }
}
<commit_msg>distance: hamming: Use zip from stdlib rather than itertools' multizip. (#158)<commit_after>// Copyright 2015-2017 Vadim Nazarov, Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Various subroutines for computing a distance between sequences.

use std::cmp::min;

use utils::TextSlice;

/// Compute the Hamming distance between two strings. Complexity: O(n).
///
/// # Example
///
/// ```
/// use bio::alignment::distance::*;
///
/// let x = b"GTCTGCATGCG";
/// let y = b"TTTAGCTAGCG";
/// // GTCTGCATGCG
/// //  |  ||  |||
/// // TTTAGCTAGCG
/// assert_eq!(hamming(x, y), 5);
/// ```
pub fn hamming(alpha: TextSlice, beta: TextSlice) -> u64 {
    assert_eq!(alpha.len(),
               beta.len(),
               "hamming distance cannot be calculated for texts of different length ({}!={})",
               alpha.len(),
               beta.len());
    let mut dist = 0;
    for (a, b) in alpha.iter().zip(beta) {
        if a != b {
            dist += 1;
        }
    }
    dist
}

/// Compute the Levenshtein (or Edit) distance between two strings. Complexity: O(n * m) with
/// n and m being the length of the given texts.
///
/// # Example
///
/// ```
/// use bio::alignment::distance::*;
///
/// let x = b"ACCGTGGAT";
/// let y = b"AAAAACCGTTGAT";
/// // ----ACCGTGGAT
/// //     ||||| |||
/// // AAAAACCGTTGAT
/// let ldist = levenshtein(x, y);  // Distance is 5
/// assert_eq!(ldist, 5);
/// ```
#[allow(unused_assignments)]
pub fn levenshtein(alpha: TextSlice, beta: TextSlice) -> u32 {
    let mut columns = [vec!(0u32; alpha.len() + 1), vec!(0u32; alpha.len() + 1)];
    let mut i_prev = 0;
    let mut i_cur = 1;

    for i in 0..columns[0].len() {
        columns[0][i] = i as u32;
    }

    for (j, item) in beta.iter().enumerate() {
        i_cur %= 2;
        i_prev = 1 - i_cur;

        columns[i_cur][0] = 1 + j as u32;
        for i in 1..columns[0].len() {
            columns[i_cur][i] = min(columns[i_prev][i - 1] +
                                    if alpha[i - 1] == *item { 0 } else { 1 },
                                    min(columns[i_cur][i - 1] + 1, columns[i_prev][i] + 1));
        }

        i_cur += 1;
    }

    columns[i_cur - 1][columns[0].len() - 1]
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_hamming_dist_good() {
        let x = b"GTCTGCATGCG";
        let y = b"TTTAGCTAGCG";
        // GTCTGCATGCG
        //  |  ||  |||
        // TTTAGCTAGCG
        assert_eq!(hamming(x, y), 5);
    }

    #[test]
    #[should_panic(expected = "hamming distance cannot be calculated for texts of different length (11!=8)")]
    fn test_hamming_dist_bad() {
        let x = b"GACTATATCGA";
        let y = b"TTTAGCTC";
        hamming(x, y);
    }

    #[test]
    fn test_levenshtein_dist() {
        let x = b"ACCGTGGAT";
        let y = b"AAAAACCGTTGAT";
        // ----ACCGTGGAT
        //     ||||| |||
        // AAAAACCGTTGAT
        assert_eq!(levenshtein(x, y), 5);
        assert_eq!(levenshtein(x, y), levenshtein(y, x));
        assert_eq!(levenshtein(b"AAA", b"TTTT"), 4);
        assert_eq!(levenshtein(b"TTTT", b"AAA"), 4);
    }
}
<|endoftext|>


<commit_before>use crate::command_prelude::*;

use cargo::core::{GitReference, SourceId};
use cargo::ops;
use cargo::util::IntoUrl;

pub fn cli() -> App {
    subcommand("install")
        .about("Install a Rust binary. Default location is $HOME/.cargo/bin")
        .arg(opt("quiet", "No output printed to stdout").short("q"))
        .arg(Arg::with_name("crate").empty_values(false).multiple(true))
        .arg(
            opt("version", "Specify a version to install")
                .alias("vers")
                .value_name("VERSION")
                .requires("crate"),
        )
        .arg(
            opt("git", "Git URL to install the specified crate from")
                .value_name("URL")
                .conflicts_with_all(&["path", "index", "registry"]),
        )
        .arg(
            opt("branch", "Branch to use when installing from git")
                .value_name("BRANCH")
                .requires("git"),
        )
        .arg(
            opt("tag", "Tag to use when installing from git")
                .value_name("TAG")
                .requires("git"),
        )
        .arg(
            opt("rev", "Specific commit to use when installing from git")
                .value_name("SHA")
                .requires("git"),
        )
        .arg(
            opt("path", "Filesystem path to local crate to install")
                .value_name("PATH")
                .conflicts_with_all(&["git", "index", "registry"]),
        )
        .arg(opt(
            "list",
            "list all installed packages and their versions",
        ))
        .arg_jobs()
        .arg(opt("force", "Force overwriting existing crates or binaries").short("f"))
        .arg(opt("no-track", "Do not save tracking information"))
        .arg_features()
        .arg_profile("Install artifacts with the specified profile")
        .arg(opt("debug", "Build in debug mode instead of release mode"))
        .arg_targets_bins_examples(
            "Install only the specified binary",
            "Install all binaries",
            "Install only the specified example",
            "Install all examples",
        )
        .arg_target_triple("Build for the target triple")
        .arg(opt("root", "Directory to install packages into").value_name("DIR"))
        .arg(
            opt("index", "Registry index to install from")
                .value_name("INDEX")
                .requires("crate")
                .conflicts_with_all(&["git", "path", "registry"]),
        )
        .arg(
            opt("registry", "Registry to use")
                .value_name("REGISTRY")
                .requires("crate")
                .conflicts_with_all(&["git", "path", "index"]),
        )
        .after_help(
            "\
This command manages Cargo's local set of installed binary crates. Only
packages which have executable [[bin]] or [[example]] targets can be
installed, and all executables are installed into the installation root's
`bin` folder. The installation root is determined, in order of precedence, by
`--root`, `$CARGO_INSTALL_ROOT`, the `install.root` configuration key, and
finally the home directory (which is either `$CARGO_HOME` if set or
`$HOME/.cargo` by default).

There are multiple sources from which a crate can be installed. The default
location is crates.io but the `--git`, `--path`, and `--registry` flags can
change this source. If the source contains more than one package (such as
crates.io or a git repository with multiple crates) the `<crate>` argument is
required to indicate which crate should be installed.

Crates from crates.io can optionally specify the version they wish to install
via the `--version` flags, and similarly packages from git repositories can
optionally specify the branch, tag, or revision that should be installed. If a
crate has multiple binaries, the `--bin` argument can selectively install only
one of them, and if you'd rather install examples the `--example` argument can
be used as well.

If the package is already installed, Cargo will reinstall it if the installed
version does not appear to be up-to-date. Installing with `--path` will always
build and install, unless there are conflicting binaries from another package.

If the source is crates.io or `--git` then by default the crate will be built
in a temporary target directory. To avoid this, the target directory can be
specified by setting the `CARGO_TARGET_DIR` environment variable to a relative
path. In particular, this can be useful for caching build artifacts on
continuous integration systems.",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    if let Some(path) = args.value_of_path("path", config) {
        config.reload_rooted_at(path)?;
    } else {
        config.reload_rooted_at(config.home().clone().into_path_unlocked())?;
    }

    let workspace = args.workspace(config).ok();
    let mut compile_opts = args.compile_options(
        config,
        CompileMode::Build,
        workspace.as_ref(),
        ProfileChecking::Checked,
    )?;

    compile_opts.build_config.requested_profile =
        args.get_profile_name(config, "release", ProfileChecking::Checked)?;

    let krates = args
        .values_of("crate")
        .unwrap_or_default()
        .collect::<Vec<_>>();

    let mut from_cwd = false;

    let source = if let Some(url) = args.value_of("git") {
        let url = url.into_url()?;
        let gitref = if let Some(branch) = args.value_of("branch") {
            GitReference::Branch(branch.to_string())
        } else if let Some(tag) = args.value_of("tag") {
            GitReference::Tag(tag.to_string())
        } else if let Some(rev) = args.value_of("rev") {
            GitReference::Rev(rev.to_string())
        } else {
            GitReference::Branch("master".to_string())
        };
        SourceId::for_git(&url, gitref)?
    } else if let Some(path) = args.value_of_path("path", config) {
        SourceId::for_path(&path)?
    } else if krates.is_empty() {
        from_cwd = true;
        SourceId::for_path(config.cwd())?
    } else if let Some(registry) = args.registry(config)? {
        SourceId::alt_registry(config, &registry)?
    } else if let Some(index) = args.value_of("index") {
        SourceId::for_registry(&index.into_url()?)?
    } else {
        SourceId::crates_io(config)?
    };

    let version = args.value_of("version");
    let root = args.value_of("root");

    if args.is_present("list") {
        ops::install_list(root, config)?;
    } else {
        ops::install(
            config,
            root,
            krates,
            source,
            from_cwd,
            version,
            &compile_opts,
            args.is_present("force"),
            args.is_present("no-track"),
        )?;
    }
    Ok(())
}
<commit_msg>Enable `--target-dir` support in `install` subcommand<commit_after>use crate::command_prelude::*;

use cargo::core::{GitReference, SourceId};
use cargo::ops;
use cargo::util::IntoUrl;

pub fn cli() -> App {
    subcommand("install")
        .about("Install a Rust binary. Default location is $HOME/.cargo/bin")
        .arg(opt("quiet", "No output printed to stdout").short("q"))
        .arg(Arg::with_name("crate").empty_values(false).multiple(true))
        .arg(
            opt("version", "Specify a version to install")
                .alias("vers")
                .value_name("VERSION")
                .requires("crate"),
        )
        .arg(
            opt("git", "Git URL to install the specified crate from")
                .value_name("URL")
                .conflicts_with_all(&["path", "index", "registry"]),
        )
        .arg(
            opt("branch", "Branch to use when installing from git")
                .value_name("BRANCH")
                .requires("git"),
        )
        .arg(
            opt("tag", "Tag to use when installing from git")
                .value_name("TAG")
                .requires("git"),
        )
        .arg(
            opt("rev", "Specific commit to use when installing from git")
                .value_name("SHA")
                .requires("git"),
        )
        .arg(
            opt("path", "Filesystem path to local crate to install")
                .value_name("PATH")
                .conflicts_with_all(&["git", "index", "registry"]),
        )
        .arg(opt(
            "list",
            "list all installed packages and their versions",
        ))
        .arg_jobs()
        .arg(opt("force", "Force overwriting existing crates or binaries").short("f"))
        .arg(opt("no-track", "Do not save tracking information"))
        .arg_features()
        .arg_profile("Install artifacts with the specified profile")
        .arg(opt("debug", "Build in debug mode instead of release mode"))
        .arg_targets_bins_examples(
            "Install only the specified binary",
            "Install all binaries",
            "Install only the specified example",
            "Install all examples",
        )
        .arg_target_triple("Build for the target triple")
        .arg_target_dir()
        .arg(opt("root", "Directory to install packages into").value_name("DIR"))
        .arg(
            opt("index", "Registry index to install from")
                .value_name("INDEX")
                .requires("crate")
                .conflicts_with_all(&["git", "path", "registry"]),
        )
        .arg(
            opt("registry", "Registry to use")
                .value_name("REGISTRY")
                .requires("crate")
                .conflicts_with_all(&["git", "path", "index"]),
        )
        .after_help(
            "\
This command manages Cargo's local set of installed binary crates. Only
packages which have executable [[bin]] or [[example]] targets can be
installed, and all executables are installed into the installation root's
`bin` folder. The installation root is determined, in order of precedence, by
`--root`, `$CARGO_INSTALL_ROOT`, the `install.root` configuration key, and
finally the home directory (which is either `$CARGO_HOME` if set or
`$HOME/.cargo` by default).

There are multiple sources from which a crate can be installed. The default
location is crates.io but the `--git`, `--path`, and `--registry` flags can
change this source. If the source contains more than one package (such as
crates.io or a git repository with multiple crates) the `<crate>` argument is
required to indicate which crate should be installed.

Crates from crates.io can optionally specify the version they wish to install
via the `--version` flags, and similarly packages from git repositories can
optionally specify the branch, tag, or revision that should be installed. If a
crate has multiple binaries, the `--bin` argument can selectively install only
one of them, and if you'd rather install examples the `--example` argument can
be used as well.

If the package is already installed, Cargo will reinstall it if the installed
version does not appear to be up-to-date. Installing with `--path` will always
build and install, unless there are conflicting binaries from another package.

If the source is crates.io or `--git` then by default the crate will be built
in a temporary target directory. To avoid this, the target directory can be
specified by setting the `CARGO_TARGET_DIR` environment variable to a relative
path. In particular, this can be useful for caching build artifacts on
continuous integration systems.",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    if let Some(path) = args.value_of_path("path", config) {
        config.reload_rooted_at(path)?;
    } else {
        config.reload_rooted_at(config.home().clone().into_path_unlocked())?;
    }

    let workspace = args.workspace(config).ok();
    let mut compile_opts = args.compile_options(
        config,
        CompileMode::Build,
        workspace.as_ref(),
        ProfileChecking::Checked,
    )?;

    compile_opts.build_config.requested_profile =
        args.get_profile_name(config, "release", ProfileChecking::Checked)?;

    let krates = args
        .values_of("crate")
        .unwrap_or_default()
        .collect::<Vec<_>>();

    let mut from_cwd = false;

    let source = if let Some(url) = args.value_of("git") {
        let url = url.into_url()?;
        let gitref = if let Some(branch) = args.value_of("branch") {
            GitReference::Branch(branch.to_string())
        } else if let Some(tag) = args.value_of("tag") {
            GitReference::Tag(tag.to_string())
        } else if let Some(rev) = args.value_of("rev") {
            GitReference::Rev(rev.to_string())
        } else {
            GitReference::Branch("master".to_string())
        };
        SourceId::for_git(&url, gitref)?
    } else if let Some(path) = args.value_of_path("path", config) {
        SourceId::for_path(&path)?
    } else if krates.is_empty() {
        from_cwd = true;
        SourceId::for_path(config.cwd())?
    } else if let Some(registry) = args.registry(config)? {
        SourceId::alt_registry(config, &registry)?
    } else if let Some(index) = args.value_of("index") {
        SourceId::for_registry(&index.into_url()?)?
    } else {
        SourceId::crates_io(config)?
    };

    let version = args.value_of("version");
    let root = args.value_of("root");

    if args.is_present("list") {
        ops::install_list(root, config)?;
    } else {
        ops::install(
            config,
            root,
            krates,
            source,
            from_cwd,
            version,
            &compile_opts,
            args.is_present("force"),
            args.is_present("no-track"),
        )?;
    }
    Ok(())
}
<|endoftext|>


<commit_before>#![allow(unstable)]
#![feature(box_syntax)]

use std::io::{BufferedWriter, File};
use std::ops::{Add, Sub, Mul};
use std::num::Float;
use std::default::Default;
use std::rand::random;
use std::sync::TaskPool;
use std::sync::mpsc::channel;
use std::sync::mpsc::{Sender, Receiver};

#[derive(Show, Copy, Clone, Default)]
struct Vector {
    x: f64,
    y: f64,
    z: f64
}

#[derive(Show, Copy, Clone, Default)]
struct Ray {
    o: Vector,
    d: Vector
}

#[derive(Show, Clone, Default)]
struct Sphere {
    radius: f64,
    position: Vector,
    emission: Vector,
    color: Vector,
}

#[derive(Show, Default, Clone)]
struct Camera {
    eye: Ray, // origin and direction of cam
    // Field of view:
    right: Vector, // right vector
    up: Vector, // up vector
}

trait Shape {
    fn intersect(self, r: Ray) -> f64;
}

trait ShapeRef {
    fn color(self, r: &Ray, t: f64) -> Vector;
}

impl Shape for Sphere {
    fn intersect(self, r: Ray) -> f64 {
        // Solve t^2*d.d + 2*t*(o-p).d + (o-p).(o-p)-R^2 = 0
        let eps = 1e-4;
        let op = &self.position - &r.o;
        let b = op.dot(&r.d);
        let mut det = b * b - op.dot(&op) + self.radius * self.radius;

        if det < 0.0 {
            return 0.0;
        } else {
            det = det.sqrt();
        }

        if (b - det) > eps {
            return b-det;
        }

        if (b + det) > eps {
            return b+det;
        }

        return 0.0;
    }
}

impl<'a> Add for &'a Vector {
    type Output = Vector;

    fn add(self, other: &'a Vector) -> Vector {
        Vector {x: self.x + other.x, y: self.y + other.y, z: self.z + other.z}
    }
}

impl<'a> Sub for &'a Vector {
    type Output = Vector;

    fn sub(self, other: &'a Vector) -> Vector {
        Vector {x: self.x - other.x, y: self.y - other.y, z: self.z - other.z}
    }
}

impl<'a> Mul for &'a Vector {
    type Output = Vector;

    fn mul(self, other: &'a Vector) -> Vector {
        Vector {x: self.x * other.x, y: self.y * other.y, z: self.z * other.z}
    }
}

trait VectorOps {
    fn smul(self, rhs: f64) -> Vector;
    fn norm(self) -> Vector;
    fn cross(self, rhs: Vector) -> Vector;
    fn dot(&self, rhs: &Vector) -> f64;
}

impl VectorOps for Vector {

    fn smul(self, other: f64) -> Vector {
        Vector {x: self.x * other, y: self.y * other, z: self.z * other}
    }

    fn norm(self) -> Vector {
        let normalize = 1.0 / (self.x * self.x + self.y * self.y + self.z * self.z).sqrt() ;
        self.smul( normalize )
    }

    fn cross(self, b: Vector) -> Vector {
        Vector{x: self.y * b.z - self.z * b.y, y: self.z * b.x - self.x * b.z, z: self.x * b.y - self.y * b.x}
    }

    fn dot(&self, other: &Vector) -> f64 {
        (*self).x * (*other).x + (*self).y * (*other).y + (*self).z * (*other).z
    }
}

fn clamp(x: f64) -> f64
{
    if x < 0.0 {
        return 0.0;
    }
    if x > 1.0 {
        return 1.0;
    }

    x
}

fn to_int(x: f64) -> i64
{
    (clamp(x).powf(1.0 / 2.2) * 255.0 + 0.5) as i64
}

fn intersect(r: Ray, t: &mut f64, id: &mut usize) -> bool
{
    let inf = 10e20f64;
    *t = inf;
    for (i, sphere) in SPHERES.iter().enumerate() {
        let d: f64 = sphere.clone().intersect(r.clone());
        if d != 0.0 && d < *t {
            *t = d;
            *id = i;
        }

    }
    return *t < inf;

}

fn get_ray(cam: &Camera, a: usize, b: usize) -> Ray {

    let w = cam.eye.d.norm().smul(-1.0);
    let u = cam.up.cross(w).norm();
    let v = w.cross(u);

    let u0 = -1.0;
    let v0 = -1.0;
    let u1 = 1.0;
    let v1 = 1.0;
    let d = 2.0;

    let across = u.smul(u1-u0);
    let up = v.smul(v1-v0);
    let an = (a as f64) / HEIGHT as f64;
    let bn = (b as f64) / WIDTH as f64;

    let corner = &(&(&cam.eye.o + &u.smul(u0)) + &v.smul(v0)) - &w.smul(d);
    let target = &( &corner + &across.smul(an)) + &up.smul(bn);
    Ray{o: cam.eye.o, d: (&target-&cam.eye.o).norm()}
}

fn get_light(ray: Ray, depth: usize) -> Vector{
    let mut t: f64 = 0.0;
    let mut id: usize = 0;
    if intersect(ray, &mut t, &mut id) {
        if depth > 5 {
            return SPHERES[id].emission;
        }

        let r1: f64 = 2.0 * PI* std::rand::random();
        let r2: f64 = std::rand::random();
        let r2s: f64 = r2.sqrt();

        // Hitpoint
        let x: Vector = &ray.o + &ray.d.smul(t);
        let n: Vector = (&x - &SPHERES[id].position).norm();
        let nl = if n.dot(&ray.d) < 0.0 { n } else { n.smul(-1.0) };

        let w = nl;
        let u = if w.x.abs() > 0.1 { Vector{x: 0.0, y: 1.0, z: 0.0} } else { Vector{x: 1.0, y: 0.0, z: 0.0 } }.cross(w).norm();
        let v = w.cross(u);

        let d = (&(&u.smul( r1.cos()*r2s )  + &v.smul(r1.sin()*r2s)) + &w.smul((1.0-r2).sqrt())).norm();
        return &SPHERES[id].emission + &(&SPHERES[id].color * &get_light(Ray{o: x, d: d}, depth+1));
    }

    return Default::default();
}

static SPHERES: [Sphere; 9] = [
    Sphere{radius:1e5 as f64,  position: Vector{ x: (1e5 + 1.0) as f64, y: 40.8 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.25,z: 0.25}}, // Left
    Sphere{radius:1e5 as f64,  position: Vector{ x:  -1e5 as f64 + 99.0,y: 40.8 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.25,y: 0.25,z: 0.75}}, // Rght
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: 40.8 as f64, z: 1e5 as f64}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Back
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: 40.8 as f64, z: -1e5+600 as f64}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 1.0, y: 1.0, z: 1.0 }}, // Frnt
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y:  1e5 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Botm
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: -1e5+81.6 as f64,z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Top
    Sphere{radius:16.5, position: Vector{ x: 27.0, y: 16.5 as f64, z: 47.0}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.999, y: 0.999, z: 0.999}}, // Mirr
    Sphere{radius:16.5, position: Vector{ x: 73.0, y: 16.5 as f64, z: 78.0}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.999, y: 0.999, z: 0.999}}, // Glas
    Sphere{radius:600 as f64,  position: Vector{ x: 50 as f64, y: 681.6-0.27 as f64, z: 81.6}, emission: Vector{x: 12.0, y: 12.0, z: 12.0}, color: Vector{x: 1.0, y: 1.0, z: 1.0}}, //Lite
];

const WIDTH: usize = 1024;
const HEIGHT: usize = 768;
static NTHREADS: usize = 4;
static PI: f64 = 3.14159265358979323846264338327950288_f64;

fn main() {

    let mut cam: Camera = Default::default();
    cam.eye.o = Vector {x: 50.0, y: 52.0, z: 295.6};
    cam.eye.d = Vector {x: 0.0, y: -0.042612, z: -1.0};
    cam.up = Vector{x: 1.0, y: 0.0, z: 0.0};

    let pool = TaskPool::new(NTHREADS);
    let (tx, rx):  (Sender<(usize, usize, Vector)>, Receiver<(usize, usize, Vector)>) = channel();

    let samples: usize = 50;
    let mut output =  box [[Vector{x: 0.0, y: 0.0, z: 0.0}; WIDTH]; HEIGHT];

    for i in range(0, HEIGHT) {
        for j in range(0, WIDTH) {
            let tx = tx.clone();
            let cam = cam.clone();
            pool.execute(move|| {
                let mut r: Vector = Default::default();
                for _ in range(0, samples) {
                    let ray: Ray = get_ray(&cam, i, j);
                    r = &r + &get_light(ray, 0).smul(1.0/samples as f64);
                }
                tx.send((i, j, Vector{ x: clamp(r.x), y: clamp(r.y), z: clamp(r.z) })).unwrap();
            });
        }
    }

    for p in range(0,WIDTH*HEIGHT-1) {
        print!("\rRaytracing... ({:.0}%)", (p as f64) / ((WIDTH*HEIGHT) as f64) * 100.0);
        let (i, j, color) = rx.recv().unwrap();
        output[i][j] = color;
    }

    println!("\nWriting Image...");
    let file = File::create(&Path::new("image.ppm"));
    let mut writer = BufferedWriter::new(file);

    writer.write(format!("P3\n{} {}\n{}\n", WIDTH, HEIGHT, 255).as_bytes()).ok();
    for i in range(0, HEIGHT) {
        for j in range(0, WIDTH) {
            writer.write(format!("{} {} {} ", to_int(output[i][j].x), to_int(output[i][j].y), to_int(output[i][j].z)).as_bytes()).ok();
        }
    }
}<commit_msg>Use amount of cores for TaskPool.<commit_after>#![allow(unstable)]
#![feature(box_syntax)]

use std::io::{BufferedWriter, File};
use std::ops::{Add, Sub, Mul};
use std::num::Float;
use std::default::Default;
use std::rand::random;
use std::sync::TaskPool;
use std::sync::mpsc::channel;
use std::sync::mpsc::{Sender, Receiver};

#[derive(Show, Copy, Clone, Default)]
struct Vector {
    x: f64,
    y: f64,
    z: f64
}

#[derive(Show, Copy, Clone, Default)]
struct Ray {
    o: Vector,
    d: Vector
}

#[derive(Show, Clone, Default)]
struct Sphere {
    radius: f64,
    position: Vector,
    emission: Vector,
    color: Vector,
}

#[derive(Show, Default, Clone)]
struct Camera {
    eye: Ray, // origin and direction of cam
    // Field of view:
    right: Vector, // right vector
    up: Vector, // up vector
}

trait Shape {
    fn intersect(self, r: Ray) -> f64;
}

trait ShapeRef {
    fn color(self, r: &Ray, t: f64) -> Vector;
}

impl Shape for Sphere {
    fn intersect(self, r: Ray) -> f64 {
        // Solve t^2*d.d + 2*t*(o-p).d + (o-p).(o-p)-R^2 = 0
        let eps = 1e-4;
        let op = &self.position - &r.o;
        let b = op.dot(&r.d);
        let mut det = b * b - op.dot(&op) + self.radius * self.radius;

        if det < 0.0 {
            return 0.0;
        } else {
            det = det.sqrt();
        }

        if (b - det) > eps {
            return b-det;
        }

        if (b + det) > eps {
            return b+det;
        }

        return 0.0;
    }
}

impl<'a> Add for &'a Vector {
    type Output = Vector;

    fn add(self, other: &'a Vector) -> Vector {
        Vector {x: self.x + other.x, y: self.y + other.y, z: self.z + other.z}
    }
}

impl<'a> Sub for &'a Vector {
    type Output = Vector;

    fn sub(self, other: &'a Vector) -> Vector {
        Vector {x: self.x - other.x, y: self.y - other.y, z: self.z - other.z}
    }
}

impl<'a> Mul for &'a Vector {
    type Output = Vector;

    fn mul(self, other: &'a Vector) -> Vector {
        Vector {x: self.x * other.x, y: self.y * other.y, z: self.z * other.z}
    }
}

trait VectorOps {
    fn smul(self, rhs: f64) -> Vector;
    fn norm(self) -> Vector;
    fn cross(self, rhs: Vector) -> Vector;
    fn dot(&self, rhs: &Vector) -> f64;
}

impl VectorOps for Vector {

    fn smul(self, other: f64) -> Vector {
        Vector {x: self.x * other, y: self.y * other, z: self.z * other}
    }

    fn norm(self) -> Vector {
        let normalize = 1.0 / (self.x * self.x + self.y * self.y + self.z * self.z).sqrt() ;
        self.smul( normalize )
    }

    fn cross(self, b: Vector) -> Vector {
        Vector{x: self.y * b.z - self.z * b.y, y: self.z * b.x - self.x * b.z, z: self.x * b.y - self.y * b.x}
    }

    fn dot(&self, other: &Vector) -> f64 {
        (*self).x * (*other).x + (*self).y * (*other).y + (*self).z * (*other).z
    }
}

fn clamp(x: f64) -> f64
{
    if x < 0.0 {
        return 0.0;
    }
    if x > 1.0 {
        return 1.0;
    }

    x
}

fn to_int(x: f64) -> i64
{
    (clamp(x).powf(1.0 / 2.2) * 255.0 + 0.5) as i64
}

fn intersect(r: Ray, t: &mut f64, id: &mut usize) -> bool
{
    let inf = 10e20f64;
    *t = inf;
    for (i, sphere) in SPHERES.iter().enumerate() {
        let d: f64 = sphere.clone().intersect(r.clone());
        if d != 0.0 && d < *t {
            *t = d;
            *id = i;
        }

    }
    return *t < inf;

}

fn get_ray(cam: &Camera, a: usize, b: usize) -> Ray {

    let w = cam.eye.d.norm().smul(-1.0);
    let u = cam.up.cross(w).norm();
    let v = w.cross(u);

    let u0 = -1.0;
    let v0 = -1.0;
    let u1 = 1.0;
    let v1 = 1.0;
    let d = 2.0;

    let across = u.smul(u1-u0);
    let up = v.smul(v1-v0);
    let an = (a as f64) / HEIGHT as f64;
    let bn = (b as f64) / WIDTH as f64;

    let corner = &(&(&cam.eye.o + &u.smul(u0)) + &v.smul(v0)) - &w.smul(d);
    let target = &( &corner + &across.smul(an)) + &up.smul(bn);
    Ray{o: cam.eye.o, d: (&target-&cam.eye.o).norm()}
}

fn get_light(ray: Ray, depth: usize) -> Vector{
    let mut t: f64 = 0.0;
    let mut id: usize = 0;
    if intersect(ray, &mut t, &mut id) {
        if depth > 5 {
            return SPHERES[id].emission;
        }

        let r1: f64 = 2.0 * PI* std::rand::random();
        let r2: f64 = std::rand::random();
        let r2s: f64 = r2.sqrt();

        // Hitpoint
        let x: Vector = &ray.o + &ray.d.smul(t);
        let n: Vector = (&x - &SPHERES[id].position).norm();
        let nl = if n.dot(&ray.d) < 0.0 { n } else { n.smul(-1.0) };

        let w = nl;
        let u = if w.x.abs() > 0.1 { Vector{x: 0.0, y: 1.0, z: 0.0} } else { Vector{x: 1.0, y: 0.0, z: 0.0 } }.cross(w).norm();
        let v = w.cross(u);

        let d = (&(&u.smul( r1.cos()*r2s )  + &v.smul(r1.sin()*r2s)) + &w.smul((1.0-r2).sqrt())).norm();
        return &SPHERES[id].emission + &(&SPHERES[id].color * &get_light(Ray{o: x, d: d}, depth+1));
    }

    return Default::default();
}

static SPHERES: [Sphere; 9] = [
    Sphere{radius:1e5 as f64,  position: Vector{ x: (1e5 + 1.0) as f64, y: 40.8 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.25,z: 0.25}}, // Left
    Sphere{radius:1e5 as f64,  position: Vector{ x:  -1e5 as f64 + 99.0,y: 40.8 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.25,y: 0.25,z: 0.75}}, // Rght
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: 40.8 as f64, z: 1e5 as f64}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Back
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: 40.8 as f64, z: -1e5+600 as f64}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 1.0, y: 1.0, z: 1.0 }}, // Frnt
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y:  1e5 as f64, z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Botm
    Sphere{radius:1e5 as f64,  position: Vector{ x: 50 as f64, y: -1e5+81.6 as f64,z: 81.6}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.75,y: 0.75,z: 0.75}}, // Top
    Sphere{radius:16.5, position: Vector{ x: 27.0, y: 16.5 as f64, z: 47.0}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.999, y: 0.999, z: 0.999}}, // Mirr
    Sphere{radius:16.5, position: Vector{ x: 73.0, y: 16.5 as f64, z: 78.0}, emission: Vector{x: 0.0, y: 0.0, z: 0.0 }, color: Vector{x: 0.999, y: 0.999, z: 0.999}}, // Glas
    Sphere{radius:600 as f64,  position: Vector{ x: 50 as f64, y: 681.6-0.27 as f64, z: 81.6}, emission: Vector{x: 12.0, y: 12.0, z: 12.0}, color: Vector{x: 1.0, y: 1.0, z: 1.0}}, //Lite
];

const WIDTH: usize = 1024;
const HEIGHT: usize = 768;
static PI: f64 = 3.14159265358979323846264338327950288_f64;

fn main() {

    let mut cam: Camera = Default::default();
    cam.eye.o = Vector {x: 50.0, y: 52.0, z: 295.6};
    cam.eye.d = Vector {x: 0.0, y: -0.042612, z: -1.0};
    cam.up = Vector{x: 1.0, y: 0.0, z: 0.0};

    let pool = TaskPool::new(std::os::num_cpus());
    let (tx, rx):  (Sender<(usize, usize, Vector)>, Receiver<(usize, usize, Vector)>) = channel();

    let samples: usize = 5000;
    let mut output =  box [[Vector{x: 0.0, y: 0.0, z: 0.0}; WIDTH]; HEIGHT];

    for i in range(0, HEIGHT) {
        for j in range(0, WIDTH) {
            let tx = tx.clone();
            let cam = cam.clone();
            pool.execute(move|| {
                let mut r: Vector = Default::default();
                for _ in range(0, samples) {
                    let ray: Ray = get_ray(&cam, i, j);
                    r = &r + &get_light(ray, 0).smul(1.0/samples as f64);
                }
                tx.send((i, j, Vector{ x: clamp(r.x), y: clamp(r.y), z: clamp(r.z) })).unwrap();
            });
        }
    }

    for p in range(0,WIDTH*HEIGHT-1) {
        print!("\rRaytracing... ({:.0}%)", (p as f64) / ((WIDTH*HEIGHT) as f64) * 100.0);
        let (i, j, color) = rx.recv().unwrap();
        output[i][j] = color;
    }

    println!("\nWriting Image...");
    let file = File::create(&Path::new("image.ppm"));
    let mut writer = BufferedWriter::new(file);

    writer.write(format!("P3\n{} {}\n{}\n", WIDTH, HEIGHT, 255).as_bytes()).ok();
    for i in range(0, HEIGHT) {
        for j in range(0, WIDTH) {
            writer.write(format!("{} {} {} ", to_int(output[i][j].x), to_int(output[i][j].y), to_int(output[i][j].z)).as_bytes()).ok();
        }
    }
}<|endoftext|>


<commit_before>//! Key algorithms used in Matrix spec.

#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};

use strum::{AsRefStr, Display, EnumString};

/// The basic key algorithms in the specification.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, AsRefStr, Display, EnumString)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[non_exhaustive]
#[strum(serialize_all = "snake_case")]
pub enum DeviceKeyAlgorithm {
    /// The Ed25519 signature algorithm.
    Ed25519,

    /// The Curve25519 ECDH algorithm.
    Curve25519,

    /// The Curve25519 ECDH algorithm, but the key also contains signatures
    SignedCurve25519,
}

/// The server key algorithms defined in the Matrix spec.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, AsRefStr, Display, EnumString)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[non_exhaustive]
#[strum(serialize_all = "snake_case")]
pub enum ServerKeyAlgorithm {
    /// The Ed25519 signature algorithm.
    Ed25519,
}
<commit_msg>Add tests for key_algorithm parsing<commit_after>//! Key algorithms used in Matrix spec.

use std::convert::TryFrom;

#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};

use strum::{AsRefStr, Display, EnumString};

/// The basic key algorithms in the specification.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, AsRefStr, Display, EnumString)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[non_exhaustive]
#[strum(serialize_all = "snake_case")]
pub enum DeviceKeyAlgorithm {
    /// The Ed25519 signature algorithm.
    Ed25519,

    /// The Curve25519 ECDH algorithm.
    Curve25519,

    /// The Curve25519 ECDH algorithm, but the key also contains signatures
    SignedCurve25519,
}

impl TryFrom<&'_ str> for DeviceKeyAlgorithm {
    type Error = strum::ParseError;

    fn try_from(value: &str) -> Result<Self, Self::Error> {
        value.parse()
    }
}

impl TryFrom<String> for DeviceKeyAlgorithm {
    type Error = strum::ParseError;

    fn try_from(value: String) -> Result<Self, Self::Error> {
        value.parse()
    }
}

/// The server key algorithms defined in the Matrix spec.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, AsRefStr, Display, EnumString)]
#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
#[non_exhaustive]
#[strum(serialize_all = "snake_case")]
pub enum ServerKeyAlgorithm {
    /// The Ed25519 signature algorithm.
    Ed25519,
}

impl TryFrom<&'_ str> for ServerKeyAlgorithm {
    type Error = strum::ParseError;

    fn try_from(value: &str) -> Result<Self, Self::Error> {
        value.parse()
    }
}

impl TryFrom<String> for ServerKeyAlgorithm {
    type Error = strum::ParseError;

    fn try_from(value: String) -> Result<Self, Self::Error> {
        value.parse()
    }
}

#[cfg(test)]
mod tests {
    use super::{DeviceKeyAlgorithm, ServerKeyAlgorithm};

    #[test]
    fn parse_device_key_algorithm() {
        assert_eq!("ed25519".parse(), Ok(DeviceKeyAlgorithm::Ed25519));
        assert_eq!("curve25519".parse(), Ok(DeviceKeyAlgorithm::Curve25519));
        assert_eq!("signed_curve25519".parse(), Ok(DeviceKeyAlgorithm::SignedCurve25519));
    }

    #[test]
    fn parse_server_key_algorithm() {
        assert_eq!("ed25519".parse(), Ok(ServerKeyAlgorithm::Ed25519));
    }
}
<|endoftext|>


<commit_before>use std::collections::hash_map::HashMap;
use std::collections::hash_set::HashSet;
use std::fmt::{Display, Formatter, Error};
use std::string::String;
use std::vec::Vec;

use alloc_box::AllocBox;
use super::binding::UniqueBinding;
use super::js_var::{JsVar, JsKey, JsType, JsPtrEnum};

macro_rules! ptr_type_mismatch {
    ($tag:expr, $ptr:expr) => {
        panic!("Pointer type mismatch: expected {:?} but received {:?}", $tag, $ptr);
    }
}

pub type JsProto = Option<Box<JsObjStruct>>;

#[derive(Clone, Debug)]
pub struct JsObjStruct {
    pub proto: JsProto,
    pub name: String,
    pub dict: HashMap<JsKey, JsVar>,
}

impl JsObjStruct {
    #[allow(unused_variables)]
    pub fn new(proto: JsProto, name: &str, kv_tuples: Vec<(JsKey, JsVar, Option<JsPtrEnum>)>,
               allocator: &mut AllocBox) -> JsObjStruct {
        let proto_vec : Vec<_> = match proto {
            Some(ref obj) => obj.clone().dict.into_iter().map(|(key, var)| {
                let ptr = allocator.find_id(&var.unique).map(|ptr| ptr.borrow().clone());
                (key, JsVar::new(var.t), ptr)
            }).collect(),
            None => Vec::new(),
        };

        JsObjStruct {
            proto: proto,
            name: String::from(name),
            dict: proto_vec.into_iter().chain(kv_tuples.into_iter()).map(|(k, v, ptr)| {
                match v.t {
                    JsType::JsPtr(ref tag) => match ptr {
                        Some(ptr) => {
                            if tag.eq_ptr_type(&ptr) {
                                allocator.alloc(v.unique.clone(), ptr)
                                    .expect("Unable to allocate!"); // TODO better error handling
                            } else {
                                ptr_type_mismatch!(tag, ptr);
                            }
                        },
                        None => ptr_type_mismatch!(tag, None::<JsPtrEnum>),
                    },
                    _ => match ptr {
                        Some(ptr) => ptr_type_mismatch!(None::<JsPtrEnum>, ptr),
                        None => {},
                    },
                }
                (k, v)
            }).collect()
        }
    }

    pub fn add_key(&mut self, obj_binding: &UniqueBinding, k: JsKey, v: JsVar,
                   ptr: Option<JsPtrEnum>, allocator: &mut AllocBox) {
        if let Some(var) = self.dict.get(&k) {
            match var.t {
                JsType::JsPtr(_) => { allocator.condemn(var.unique.clone()).expect("Unable to whiten!") },
                _ => {}
            }
        }
        /*if let Some(ptr) = ptr {
            allocator.alloc(v.unique.clone(), ptr).expect("Unable to allocate!"); // TODO better error handling
        }*/

        match allocator.find_id(obj_binding) {
            Some(ref ptr) => match &mut *(ptr.borrow_mut()) {
                &mut JsPtrEnum::JsObj(ref mut obj) => obj.dict.insert(k.clone(), v.clone()),
                _ => panic!("Binding does not belong to an object!"),
            },
            None => panic!("No pointer with matching binding found!"),
        };

        self.dict.insert(k, v);
    }

    pub fn remove_key(&mut self, obj_binding: &UniqueBinding, k: &JsKey, allocator: &mut AllocBox)
        -> Option<(JsVar, Option<JsPtrEnum>)>{
        let var_opt = match &mut *(allocator.find_id(obj_binding).expect("No pointer with matching binding found!").borrow_mut()) {
            &mut JsPtrEnum::JsObj(ref mut obj) => obj.dict.remove(k),
            _ => panic!("Binding does not belong to an object!"),
        };

        if let Some(var) = var_opt {
            let ptr = allocator.find_id(&var.unique).map(|s| s.borrow().clone());

            if ptr.is_some() {
                allocator.condemn(var.unique.clone()).expect("Unable to whiten!");
            }

            let _ = self.dict.remove(k);
            Some((var, ptr))
        } else {
            None
        }
    }

    pub fn get_children(&self) -> HashSet<UniqueBinding> {
        let mut bindings = HashSet::new();
        for v in self.dict.values() {
            match v.t {
                JsType::JsPtr(_) => { bindings.insert(v.unique.clone()); },
                _ => (),
            }
        }
        bindings
    }
}

impl Display for JsObjStruct {
    fn fmt(&self, mut fmt: &mut Formatter) -> Result<(), Error> {
        try!(write!(fmt, "{{ "));

        for (i, (ref key, ref val)) in self.dict.iter().enumerate() {
            if i != 0 {
                try!(write!(fmt, ", "));
            }

            try!(write!(fmt, "{}: {}", key, val));
        }

        write!(fmt, " }}")
    }
}
<commit_msg>Silence unused var warning<commit_after>use std::collections::hash_map::HashMap;
use std::collections::hash_set::HashSet;
use std::fmt::{Display, Formatter, Error};
use std::string::String;
use std::vec::Vec;

use alloc_box::AllocBox;
use super::binding::UniqueBinding;
use super::js_var::{JsVar, JsKey, JsType, JsPtrEnum};

macro_rules! ptr_type_mismatch {
    ($tag:expr, $ptr:expr) => {
        panic!("Pointer type mismatch: expected {:?} but received {:?}", $tag, $ptr);
    }
}

pub type JsProto = Option<Box<JsObjStruct>>;

#[derive(Clone, Debug)]
pub struct JsObjStruct {
    pub proto: JsProto,
    pub name: String,
    pub dict: HashMap<JsKey, JsVar>,
}

impl JsObjStruct {
    #[allow(unused_variables)]
    pub fn new(proto: JsProto, name: &str, kv_tuples: Vec<(JsKey, JsVar, Option<JsPtrEnum>)>,
               allocator: &mut AllocBox) -> JsObjStruct {
        let proto_vec : Vec<_> = match proto {
            Some(ref obj) => obj.clone().dict.into_iter().map(|(key, var)| {
                let ptr = allocator.find_id(&var.unique).map(|ptr| ptr.borrow().clone());
                (key, JsVar::new(var.t), ptr)
            }).collect(),
            None => Vec::new(),
        };

        JsObjStruct {
            proto: proto,
            name: String::from(name),
            dict: proto_vec.into_iter().chain(kv_tuples.into_iter()).map(|(k, v, ptr)| {
                match v.t {
                    JsType::JsPtr(ref tag) => match ptr {
                        Some(ptr) => {
                            if tag.eq_ptr_type(&ptr) {
                                println!("{:?}", ptr);
                                allocator.alloc(v.unique.clone(), ptr)
                                    .expect("Unable to allocate!"); // TODO better error handling
                            } else {
                                ptr_type_mismatch!(tag, ptr);
                            }
                        },
                        None => ptr_type_mismatch!(tag, None::<JsPtrEnum>),
                    },
                    _ => match ptr {
                        Some(ptr) => ptr_type_mismatch!(None::<JsPtrEnum>, ptr),
                        None => {},
                    },
                }
                (k, v)
            }).collect()
        }
    }

    pub fn add_key(&mut self, obj_binding: &UniqueBinding, k: JsKey, v: JsVar,
                   _ptr: Option<JsPtrEnum>, allocator: &mut AllocBox) {
        if let Some(var) = self.dict.get(&k) {
            match var.t {
                JsType::JsPtr(_) => { allocator.condemn(var.unique.clone()).expect("Unable to whiten!") },
                _ => {}
            }
        }
        /*if let Some(ptr) = ptr {
            allocator.alloc(v.unique.clone(), ptr).expect("Unable to allocate!"); // TODO better error handling
        }*/

        match allocator.find_id(obj_binding) {
            Some(ref ptr) => match &mut *(ptr.borrow_mut()) {
                &mut JsPtrEnum::JsObj(ref mut obj) => obj.dict.insert(k.clone(), v.clone()),
                _ => panic!("Binding does not belong to an object!"),
            },
            None => panic!("No pointer with matching binding found!"),
        };

        self.dict.insert(k, v);
    }

    pub fn remove_key(&mut self, obj_binding: &UniqueBinding, k: &JsKey, allocator: &mut AllocBox)
        -> Option<(JsVar, Option<JsPtrEnum>)>{
        let var_opt = match &mut *(allocator.find_id(obj_binding).expect("No pointer with matching binding found!").borrow_mut()) {
            &mut JsPtrEnum::JsObj(ref mut obj) => obj.dict.remove(k),
            _ => panic!("Binding does not belong to an object!"),
        };

        if let Some(var) = var_opt {
            let ptr = allocator.find_id(&var.unique).map(|s| s.borrow().clone());

            if ptr.is_some() {
                allocator.condemn(var.unique.clone()).expect("Unable to whiten!");
            }

            let _ = self.dict.remove(k);
            Some((var, ptr))
        } else {
            None
        }
    }

    pub fn get_children(&self) -> HashSet<UniqueBinding> {
        let mut bindings = HashSet::new();
        for v in self.dict.values() {
            match v.t {
                JsType::JsPtr(_) => { bindings.insert(v.unique.clone()); },
                _ => (),
            }
        }
        bindings
    }
}

impl Display for JsObjStruct {
    fn fmt(&self, mut fmt: &mut Formatter) -> Result<(), Error> {
        try!(write!(fmt, "{{ "));

        for (i, (ref key, ref val)) in self.dict.iter().enumerate() {
            if i != 0 {
                try!(write!(fmt, ", "));
            }

            try!(write!(fmt, "{}: {}", key, val));
        }

        write!(fmt, " }}")
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    backup_types::{
        epoch_ending::backup::{EpochEndingBackupController, EpochEndingBackupOpt},
        state_snapshot::backup::{StateSnapshotBackupController, StateSnapshotBackupOpt},
        transaction::backup::{TransactionBackupController, TransactionBackupOpt},
    },
    metadata,
    metadata::cache::MetadataCacheOpt,
    storage::BackupStorage,
    utils::{backup_service_client::BackupServiceClient, GlobalBackupOpt},
};
use anyhow::{anyhow, ensure, Result};
use futures::{stream, Future, StreamExt};
use libra_logger::prelude::*;
use libra_types::transaction::Version;
use libradb::backup::backup_handler::DbState;
use std::{fmt::Debug, sync::Arc};
use structopt::StructOpt;
use tokio::{
    sync::watch,
    time::{interval, Duration},
};

#[derive(StructOpt)]
pub struct BackupCoordinatorOpt {
    #[structopt(flatten)]
    pub metadata_cache_opt: MetadataCacheOpt,
    // We replay transactions on top of a state snapshot at about 2000 tps, having a state snapshot
    // every 2000 * 3600 * 2 = 14.4 Mil versions will guarantee that to achieve any state in
    // history, it won't take us more than two hours in transaction replaying. Defaulting to 10 Mil
    // here to make it less than two, and easier for eyes.
    #[structopt(long, default_value = "10000000")]
    pub state_snapshot_interval: usize,
    // Assuming the network runs at 100 tps, it's 100 * 3600 = 360k transactions per hour, we don't
    // want the backups to lag behind too much. Defaulting to 100k here in case the network is way
    // slower than expected.
    #[structopt(long, default_value = "100000")]
    pub transaction_batch_size: usize,
}

impl BackupCoordinatorOpt {
    fn validate(&self) -> Result<()> {
        ensure!(
            self.state_snapshot_interval > 0 && self.transaction_batch_size > 0,
            "Backup interval and batch size must be greater than 0."
        );
        Ok(())
    }
}

pub struct BackupCoordinator {
    client: Arc<BackupServiceClient>,
    storage: Arc<dyn BackupStorage>,
    global_opt: GlobalBackupOpt,
    metadata_cache_opt: MetadataCacheOpt,
    state_snapshot_interval: usize,
    transaction_batch_size: usize,
}

impl BackupCoordinator {
    pub fn new(
        opt: BackupCoordinatorOpt,
        global_opt: GlobalBackupOpt,
        client: Arc<BackupServiceClient>,
        storage: Arc<dyn BackupStorage>,
    ) -> Self {
        opt.validate().unwrap();
        Self {
            client,
            storage,
            global_opt,
            metadata_cache_opt: opt.metadata_cache_opt,
            state_snapshot_interval: opt.state_snapshot_interval,
            transaction_batch_size: opt.transaction_batch_size,
        }
    }
    pub async fn run(&self) -> Result<()> {
        // Connect to both the local Libra node and the backup storage.
        let backup_state =
            metadata::cache::sync_and_load(&self.metadata_cache_opt, Arc::clone(&self.storage))
                .await?
                .get_storage_state();
        let db_state = self
            .client
            .get_db_state()
            .await?
            .ok_or_else(|| anyhow!("DB not bootstrapped."))?;

        // On new DbState retrieved:
        // `watch_db_state` informs `backup_epoch_endings` via channel 1,
        // and the the latter informs the other backup type workers via channel 2, after epoch
        // ending is properly backed up, if necessary. This way, the epoch ending LedgerInfo needed
        // for proof verification is always available in the same backup storage.
        let (tx1, rx1) = watch::channel(db_state);
        let (tx2, rx2) = watch::channel(db_state);

        // Schedule work streams.
        let watch_db_state = interval(Duration::from_secs(1))
            .then(|_| self.try_refresh_db_state(&tx1))
            .boxed_local();

        let backup_epoch_endings = self
            .backup_work_stream(
                backup_state.latest_epoch_ending_epoch,
                &rx1,
                |slf, last_epoch, db_state| {
                    Self::backup_epoch_endings(slf, last_epoch, db_state, &tx2)
                },
            )
            .boxed_local();
        let backup_state_snapshots = self
            .backup_work_stream(
                backup_state.latest_state_snapshot_version,
                &rx2,
                Self::backup_state_snapshot,
            )
            .boxed_local();
        let backup_transactions = self
            .backup_work_stream(
                backup_state.latest_transaction_version,
                &rx2,
                Self::backup_transactions,
            )
            .boxed_local();

        info!("Backup coordinator started.");
        let mut all_work = stream::select_all(vec![
            watch_db_state,
            backup_epoch_endings,
            backup_state_snapshots,
            backup_transactions,
        ]);

        loop {
            all_work
                .next()
                .await
                .ok_or_else(|| anyhow!("Must be a bug: we never returned None."))?
        }
    }
}

impl BackupCoordinator {
    async fn try_refresh_db_state(&self, db_state_broadcast: &watch::Sender<DbState>) {
        match self.client.get_db_state().await {
            Ok(s) => db_state_broadcast
                .broadcast(s.expect("Db should have been bootstrapped."))
                .map_err(|e| anyhow!("Receivers should not be cancelled: {}", e))
                .unwrap(),
            Err(e) => warn!(
                "Failed pulling DbState from local Libra node: {}. Will keep trying.",
                e
            ),
        };
    }

    async fn backup_epoch_endings(
        &self,
        last_epoch_ending_epoch_in_backup: Option<u64>,
        db_state: DbState,
        downstream_db_state_broadcaster: &watch::Sender<DbState>,
    ) -> Result<Option<u64>> {
        let (first, last) = get_batch_range(last_epoch_ending_epoch_in_backup, 1);

        let next_state = if db_state.epoch <= last {
            // "<=" because `db_state.epoch` hasn't ended yet, wait for the next db_state update
            last_epoch_ending_epoch_in_backup
        } else {
            EpochEndingBackupController::new(
                EpochEndingBackupOpt {
                    start_epoch: first,
                    end_epoch: last + 1,
                },
                self.global_opt.clone(),
                Arc::clone(&self.client),
                Arc::clone(&self.storage),
            )
            .run()
            .await?;
            Some(last)
        };

        downstream_db_state_broadcaster
            .broadcast(db_state)
            .map_err(|e| anyhow!("Receivers should not be cancelled: {}", e))
            .unwrap();
        Ok(next_state)
    }

    async fn backup_state_snapshot(
        &self,
        last_snapshot_version_in_backup: Option<Version>,
        db_state: DbState,
    ) -> Result<Option<Version>> {
        let next_snapshot_version = get_next_snapshot(
            last_snapshot_version_in_backup,
            db_state,
            self.state_snapshot_interval,
        );

        if db_state.committed_version < next_snapshot_version {
            // wait for the next db_state update
            return Ok(last_snapshot_version_in_backup);
        }

        StateSnapshotBackupController::new(
            StateSnapshotBackupOpt {
                version: next_snapshot_version,
            },
            self.global_opt.clone(),
            Arc::clone(&self.client),
            Arc::clone(&self.storage),
        )
        .run()
        .await?;

        Ok(Some(next_snapshot_version))
    }

    async fn backup_transactions(
        &self,
        last_transaction_version_in_backup: Option<Version>,
        db_state: DbState,
    ) -> Result<Option<u64>> {
        let (first, last) = get_batch_range(
            last_transaction_version_in_backup,
            self.transaction_batch_size,
        );

        if db_state.committed_version < last {
            // wait for the next db_state update
            return Ok(last_transaction_version_in_backup);
        }

        TransactionBackupController::new(
            TransactionBackupOpt {
                start_version: first,
                num_transactions: (last + 1 - first) as usize,
            },
            self.global_opt.clone(),
            Arc::clone(&self.client),
            Arc::clone(&self.storage),
        )
        .run()
        .await?;

        Ok(Some(last))
    }

    fn backup_work_stream<'a, S, W, Fut>(
        &'a self,
        initial_state: S,
        db_state_rx: &'a watch::Receiver<DbState>,
        worker: W,
    ) -> impl StreamExt<Item = ()> + 'a
    where
        S: Copy + Debug + 'a,
        W: Worker<'a, S, Fut> + Copy + 'a,
        Fut: Future<Output = Result<S>> + 'a,
    {
        stream::unfold(
            (initial_state, db_state_rx.clone()),
            move |(s, mut rx)| async move {
                let db_state = rx
                    .recv()
                    .await
                    .ok_or_else(|| anyhow!("The broadcaster has been dropped."))
                    .unwrap();
                let next_state = worker(self, s, db_state).await.unwrap_or_else(|e| {
                    warn!("backup failed: {}. Keep trying with state {:?}.", e, s);
                    s
                });
                Some(((), (next_state, rx)))
            },
        )
    }
}

trait Worker<'a, S, Fut: Future<Output = Result<S>> + 'a>:
    Fn(&'a BackupCoordinator, S, DbState) -> Fut
{
}

impl<'a, T, S, Fut> Worker<'a, S, Fut> for T
where
    T: Fn(&'a BackupCoordinator, S, DbState) -> Fut,
    Fut: Future<Output = Result<S>> + 'a,
{
}

fn get_batch_range(last_in_backup: Option<u64>, batch_size: usize) -> (u64, u64) {
    // say, 7 is already in backup, and we target batches of size 10, we will return (8, 10) in this
    // case, so 8, 9, 10 will be in this batch, and next time the backup worker will pass in 10,
    // and we will return (11, 20). The transaction 0 will be in it's own batch.
    last_in_backup.map_or((0, 0), |n| {
        let first = n + 1;
        let batch = n / batch_size as u64 + 1;
        let last = batch * batch_size as u64;
        (first, last)
    })
}

fn get_next_snapshot(last_in_backup: Option<u64>, db_state: DbState, interval: usize) -> u64 {
    // We don't try to guarantee snapshots are taken at each applicable interval: when the backup
    // progress can't keep up with the ledger growth, we favor timeliness over completeness.
    // For example, with interval 100, when we finished taking a snapshot at version 700, if we
    // found the latest version is already 1250, the next snapshot we take will be at 1200, not 800.

    let next_for_storage = match last_in_backup {
        Some(last) => (last / interval as u64 + 1) * interval as u64,
        None => 0,
    };

    let last_for_db: u64 = db_state.committed_version / interval as u64 * interval as u64;

    std::cmp::max(next_for_storage, last_for_db)
}

#[cfg(test)]
mod tests {
    use crate::coordinators::backup::{get_batch_range, get_next_snapshot};
    use libradb::backup::backup_handler::DbState;

    #[test]
    fn test_get_batch_range() {
        assert_eq!(get_batch_range(None, 100), (0, 0));
        assert_eq!(get_batch_range(Some(0), 100), (1, 100));
        assert_eq!(get_batch_range(Some(100), 50), (101, 150));
        assert_eq!(get_batch_range(Some(150), 100), (151, 200));
        assert_eq!(get_batch_range(Some(200), 100), (201, 300));
    }

    #[test]
    fn test_get_next_snapshot() {
        let _state = |v| DbState {
            epoch: 0,
            committed_version: v,
            synced_version: v,
        };

        assert_eq!(get_next_snapshot(None, _state(90), 100), 0);
        assert_eq!(get_next_snapshot(Some(0), _state(90), 100), 100);
        assert_eq!(get_next_snapshot(Some(0), _state(190), 100), 100);
        assert_eq!(get_next_snapshot(Some(0), _state(200), 100), 200);
        assert_eq!(get_next_snapshot(Some(0), _state(250), 100), 200);
        assert_eq!(get_next_snapshot(Some(200), _state(250), 100), 300);
    }
}
<commit_msg>[backup] backup coordinator: skip 1 sec wait when there's still work to do<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    backup_types::{
        epoch_ending::backup::{EpochEndingBackupController, EpochEndingBackupOpt},
        state_snapshot::backup::{StateSnapshotBackupController, StateSnapshotBackupOpt},
        transaction::backup::{TransactionBackupController, TransactionBackupOpt},
    },
    metadata,
    metadata::cache::MetadataCacheOpt,
    storage::BackupStorage,
    utils::{backup_service_client::BackupServiceClient, GlobalBackupOpt},
};
use anyhow::{anyhow, ensure, Result};
use futures::{stream, Future, StreamExt};
use libra_logger::prelude::*;
use libra_types::transaction::Version;
use libradb::backup::backup_handler::DbState;
use std::{fmt::Debug, sync::Arc};
use structopt::StructOpt;
use tokio::{
    sync::watch,
    time::{interval, Duration},
};

#[derive(StructOpt)]
pub struct BackupCoordinatorOpt {
    #[structopt(flatten)]
    pub metadata_cache_opt: MetadataCacheOpt,
    // We replay transactions on top of a state snapshot at about 2000 tps, having a state snapshot
    // every 2000 * 3600 * 2 = 14.4 Mil versions will guarantee that to achieve any state in
    // history, it won't take us more than two hours in transaction replaying. Defaulting to 10 Mil
    // here to make it less than two, and easier for eyes.
    #[structopt(long, default_value = "10000000")]
    pub state_snapshot_interval: usize,
    // Assuming the network runs at 100 tps, it's 100 * 3600 = 360k transactions per hour, we don't
    // want the backups to lag behind too much. Defaulting to 100k here in case the network is way
    // slower than expected.
    #[structopt(long, default_value = "100000")]
    pub transaction_batch_size: usize,
}

impl BackupCoordinatorOpt {
    fn validate(&self) -> Result<()> {
        ensure!(
            self.state_snapshot_interval > 0 && self.transaction_batch_size > 0,
            "Backup interval and batch size must be greater than 0."
        );
        Ok(())
    }
}

pub struct BackupCoordinator {
    client: Arc<BackupServiceClient>,
    storage: Arc<dyn BackupStorage>,
    global_opt: GlobalBackupOpt,
    metadata_cache_opt: MetadataCacheOpt,
    state_snapshot_interval: usize,
    transaction_batch_size: usize,
}

impl BackupCoordinator {
    pub fn new(
        opt: BackupCoordinatorOpt,
        global_opt: GlobalBackupOpt,
        client: Arc<BackupServiceClient>,
        storage: Arc<dyn BackupStorage>,
    ) -> Self {
        opt.validate().unwrap();
        Self {
            client,
            storage,
            global_opt,
            metadata_cache_opt: opt.metadata_cache_opt,
            state_snapshot_interval: opt.state_snapshot_interval,
            transaction_batch_size: opt.transaction_batch_size,
        }
    }
    pub async fn run(&self) -> Result<()> {
        // Connect to both the local Libra node and the backup storage.
        let backup_state =
            metadata::cache::sync_and_load(&self.metadata_cache_opt, Arc::clone(&self.storage))
                .await?
                .get_storage_state();
        let db_state = self
            .client
            .get_db_state()
            .await?
            .ok_or_else(|| anyhow!("DB not bootstrapped."))?;

        // On new DbState retrieved:
        // `watch_db_state` informs `backup_epoch_endings` via channel 1,
        // and the the latter informs the other backup type workers via channel 2, after epoch
        // ending is properly backed up, if necessary. This way, the epoch ending LedgerInfo needed
        // for proof verification is always available in the same backup storage.
        let (tx1, rx1) = watch::channel(db_state);
        let (tx2, rx2) = watch::channel(db_state);

        // Schedule work streams.
        let watch_db_state = interval(Duration::from_secs(1))
            .then(|_| self.try_refresh_db_state(&tx1))
            .boxed_local();

        let backup_epoch_endings = self
            .backup_work_stream(
                backup_state.latest_epoch_ending_epoch,
                &rx1,
                |slf, last_epoch, db_state| {
                    Self::backup_epoch_endings(slf, last_epoch, db_state, &tx2)
                },
            )
            .boxed_local();
        let backup_state_snapshots = self
            .backup_work_stream(
                backup_state.latest_state_snapshot_version,
                &rx2,
                Self::backup_state_snapshot,
            )
            .boxed_local();
        let backup_transactions = self
            .backup_work_stream(
                backup_state.latest_transaction_version,
                &rx2,
                Self::backup_transactions,
            )
            .boxed_local();

        info!("Backup coordinator started.");
        let mut all_work = stream::select_all(vec![
            watch_db_state,
            backup_epoch_endings,
            backup_state_snapshots,
            backup_transactions,
        ]);

        loop {
            all_work
                .next()
                .await
                .ok_or_else(|| anyhow!("Must be a bug: we never returned None."))?
        }
    }
}

impl BackupCoordinator {
    async fn try_refresh_db_state(&self, db_state_broadcast: &watch::Sender<DbState>) {
        match self.client.get_db_state().await {
            Ok(s) => db_state_broadcast
                .broadcast(s.expect("Db should have been bootstrapped."))
                .map_err(|e| anyhow!("Receivers should not be cancelled: {}", e))
                .unwrap(),
            Err(e) => warn!(
                "Failed pulling DbState from local Libra node: {}. Will keep trying.",
                e
            ),
        };
    }

    async fn backup_epoch_endings(
        &self,
        mut last_epoch_ending_epoch_in_backup: Option<u64>,
        db_state: DbState,
        downstream_db_state_broadcaster: &watch::Sender<DbState>,
    ) -> Result<Option<u64>> {
        loop {
            let (first, last) = get_batch_range(last_epoch_ending_epoch_in_backup, 1);

            if db_state.epoch <= last {
                // "<=" because `db_state.epoch` hasn't ended yet, wait for the next db_state update
                break;
            }

            EpochEndingBackupController::new(
                EpochEndingBackupOpt {
                    start_epoch: first,
                    end_epoch: last + 1,
                },
                self.global_opt.clone(),
                Arc::clone(&self.client),
                Arc::clone(&self.storage),
            )
            .run()
            .await?;
            last_epoch_ending_epoch_in_backup = Some(last)
        }

        downstream_db_state_broadcaster
            .broadcast(db_state)
            .map_err(|e| anyhow!("Receivers should not be cancelled: {}", e))
            .unwrap();
        Ok(last_epoch_ending_epoch_in_backup)
    }

    async fn backup_state_snapshot(
        &self,
        last_snapshot_version_in_backup: Option<Version>,
        db_state: DbState,
    ) -> Result<Option<Version>> {
        let next_snapshot_version = get_next_snapshot(
            last_snapshot_version_in_backup,
            db_state,
            self.state_snapshot_interval,
        );

        if db_state.committed_version < next_snapshot_version {
            // wait for the next db_state update
            return Ok(last_snapshot_version_in_backup);
        }

        StateSnapshotBackupController::new(
            StateSnapshotBackupOpt {
                version: next_snapshot_version,
            },
            self.global_opt.clone(),
            Arc::clone(&self.client),
            Arc::clone(&self.storage),
        )
        .run()
        .await?;

        Ok(Some(next_snapshot_version))
    }

    async fn backup_transactions(
        &self,
        mut last_transaction_version_in_backup: Option<Version>,
        db_state: DbState,
    ) -> Result<Option<u64>> {
        loop {
            let (first, last) = get_batch_range(
                last_transaction_version_in_backup,
                self.transaction_batch_size,
            );

            if db_state.committed_version < last {
                // wait for the next db_state update
                return Ok(last_transaction_version_in_backup);
            }

            TransactionBackupController::new(
                TransactionBackupOpt {
                    start_version: first,
                    num_transactions: (last + 1 - first) as usize,
                },
                self.global_opt.clone(),
                Arc::clone(&self.client),
                Arc::clone(&self.storage),
            )
            .run()
            .await?;

            last_transaction_version_in_backup = Some(last);
        }
    }

    fn backup_work_stream<'a, S, W, Fut>(
        &'a self,
        initial_state: S,
        db_state_rx: &'a watch::Receiver<DbState>,
        worker: W,
    ) -> impl StreamExt<Item = ()> + 'a
    where
        S: Copy + Debug + 'a,
        W: Worker<'a, S, Fut> + Copy + 'a,
        Fut: Future<Output = Result<S>> + 'a,
    {
        stream::unfold(
            (initial_state, db_state_rx.clone()),
            move |(s, mut rx)| async move {
                let db_state = rx
                    .recv()
                    .await
                    .ok_or_else(|| anyhow!("The broadcaster has been dropped."))
                    .unwrap();
                let next_state = worker(self, s, db_state).await.unwrap_or_else(|e| {
                    warn!("backup failed: {}. Keep trying with state {:?}.", e, s);
                    s
                });
                Some(((), (next_state, rx)))
            },
        )
    }
}

trait Worker<'a, S, Fut: Future<Output = Result<S>> + 'a>:
    Fn(&'a BackupCoordinator, S, DbState) -> Fut
{
}

impl<'a, T, S, Fut> Worker<'a, S, Fut> for T
where
    T: Fn(&'a BackupCoordinator, S, DbState) -> Fut,
    Fut: Future<Output = Result<S>> + 'a,
{
}

fn get_batch_range(last_in_backup: Option<u64>, batch_size: usize) -> (u64, u64) {
    // say, 7 is already in backup, and we target batches of size 10, we will return (8, 10) in this
    // case, so 8, 9, 10 will be in this batch, and next time the backup worker will pass in 10,
    // and we will return (11, 20). The transaction 0 will be in it's own batch.
    last_in_backup.map_or((0, 0), |n| {
        let first = n + 1;
        let batch = n / batch_size as u64 + 1;
        let last = batch * batch_size as u64;
        (first, last)
    })
}

fn get_next_snapshot(last_in_backup: Option<u64>, db_state: DbState, interval: usize) -> u64 {
    // We don't try to guarantee snapshots are taken at each applicable interval: when the backup
    // progress can't keep up with the ledger growth, we favor timeliness over completeness.
    // For example, with interval 100, when we finished taking a snapshot at version 700, if we
    // found the latest version is already 1250, the next snapshot we take will be at 1200, not 800.

    let next_for_storage = match last_in_backup {
        Some(last) => (last / interval as u64 + 1) * interval as u64,
        None => 0,
    };

    let last_for_db: u64 = db_state.committed_version / interval as u64 * interval as u64;

    std::cmp::max(next_for_storage, last_for_db)
}

#[cfg(test)]
mod tests {
    use crate::coordinators::backup::{get_batch_range, get_next_snapshot};
    use libradb::backup::backup_handler::DbState;

    #[test]
    fn test_get_batch_range() {
        assert_eq!(get_batch_range(None, 100), (0, 0));
        assert_eq!(get_batch_range(Some(0), 100), (1, 100));
        assert_eq!(get_batch_range(Some(100), 50), (101, 150));
        assert_eq!(get_batch_range(Some(150), 100), (151, 200));
        assert_eq!(get_batch_range(Some(200), 100), (201, 300));
    }

    #[test]
    fn test_get_next_snapshot() {
        let _state = |v| DbState {
            epoch: 0,
            committed_version: v,
            synced_version: v,
        };

        assert_eq!(get_next_snapshot(None, _state(90), 100), 0);
        assert_eq!(get_next_snapshot(Some(0), _state(90), 100), 100);
        assert_eq!(get_next_snapshot(Some(0), _state(190), 100), 100);
        assert_eq!(get_next_snapshot(Some(0), _state(200), 100), 200);
        assert_eq!(get_next_snapshot(Some(0), _state(250), 100), 200);
        assert_eq!(get_next_snapshot(Some(200), _state(250), 100), 300);
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::path::PathBuf;

use devicemapper::{DM, DmFlags, DevId};

use nix::mount::umount;
use mnt::get_submounts;

/// Attempt to remove all device mapper devices which match the stratis naming convention.
/// FIXME: Current implementation complicated by https://bugzilla.redhat.com/show_bug.cgi?id=1506287
fn dm_stratis_devices_remove() {

    let dm = DM::new().unwrap();

    loop {
        let mut progress_made = false;
        for d in dm.list_devices()
                .unwrap()
                .iter()
                .filter(|d| format!("{}", d.0.as_ref()).starts_with("stratis-1")) {
            progress_made |= dm.device_remove(&DevId::Name(&d.0), DmFlags::empty())
                .is_ok();
        }

        if !progress_made {
            break;
        }
    }
}

/// Try and un-mount any filesystems that have the name stratis in the mount point.
fn stratis_filesystems_unmount() {
    for m in get_submounts(&PathBuf::from("/"))
            .unwrap()
            .iter()
            .filter(|m| m.file.to_str().map_or(false, |s| s.contains("stratis"))) {
        umount(&m.file).unwrap();
    }
}

/// When a unit test panics we can leave the system in an inconsistent state.  This function
/// tries to clean up by un-mounting any mounted file systems which contain the string
/// "stratis_testing" and then it tries to remove any device mapper tables which are also stratis
/// created.
pub fn clean_up() -> () {
    stratis_filesystems_unmount();
    dm_stratis_devices_remove();
}
<commit_msg>Use umount2 and MNT_DETACH to unmount filesystems<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::path::PathBuf;

use devicemapper::{DM, DmFlags, DevId};

use nix::mount::{MNT_DETACH, umount2};
use mnt::get_submounts;

/// Attempt to remove all device mapper devices which match the stratis naming convention.
/// FIXME: Current implementation complicated by https://bugzilla.redhat.com/show_bug.cgi?id=1506287
fn dm_stratis_devices_remove() {

    let dm = DM::new().unwrap();

    loop {
        let mut progress_made = false;
        for d in dm.list_devices()
                .unwrap()
                .iter()
                .filter(|d| format!("{}", d.0.as_ref()).starts_with("stratis-1")) {
            progress_made |= dm.device_remove(&DevId::Name(&d.0), DmFlags::empty())
                .is_ok();
        }

        if !progress_made {
            break;
        }
    }
}

/// Try and un-mount any filesystems that have the name stratis in the mount point.
fn stratis_filesystems_unmount() {
    for m in get_submounts(&PathBuf::from("/"))
            .unwrap()
            .iter()
            .filter(|m| m.file.to_str().map_or(false, |s| s.contains("stratis"))) {
        umount2(&m.file, MNT_DETACH).unwrap();
    }
}

/// When a unit test panics we can leave the system in an inconsistent state.  This function
/// tries to clean up by un-mounting any mounted file systems which contain the string
/// "stratis_testing" and then it tries to remove any device mapper tables which are also stratis
/// created.
pub fn clean_up() -> () {
    stratis_filesystems_unmount();
    dm_stratis_devices_remove();
}
<|endoftext|>


<commit_before>extern crate disqrust;
extern crate disque;

use std::time::Duration;
use std::sync::mpsc::{channel, Sender};

use disque::Disque;
use disqrust::{EventLoop, Handler, JobStatus};

enum HandlerCall {
    Job(Vec<u8>, String, Vec<u8>),
    Error(Vec<u8>, String, u32, u32),
}

impl HandlerCall {
    fn body(&self) -> Vec<u8> {
        match *self {
            HandlerCall::Job(_, _, ref body) => body.clone(),
            HandlerCall::Error(_, _, _, _) => panic!("getting body for error"),
        }
    }

    fn nack_additional_deliveries(&self) -> (u32, u32) {
        match *self {
            HandlerCall::Job(_, _, _) => panic!("getting nack for job"),
            HandlerCall::Error(_, _, a, b) => (a, b),
        }
    }
}

#[derive(Clone)]
struct MyHandler {
    sender: Sender<HandlerCall>,
    process_job_ret: JobStatus,
    process_error_ret: bool,
}

impl MyHandler {
    fn new(sender: Sender<HandlerCall>, process_job_ret: JobStatus,
            process_error_ret: bool) -> MyHandler {
        MyHandler {
            sender: sender,
            process_job_ret: process_job_ret,
            process_error_ret: process_error_ret,
        }
    }
}

impl Handler for MyHandler {
    fn process_job(&self, queue_name: &[u8], jobid: &String, body: Vec<u8>
            ) -> JobStatus {
        self.sender.send(HandlerCall::Job(queue_name.to_vec(), jobid.clone(),
                    body)).unwrap();
        self.process_job_ret.clone()
    }

    fn process_error(&self, queue_name: &[u8], jobid: &String, nack: u32,
            additional_deliveries: u32) -> bool {
        self.sender.send(HandlerCall::Error(queue_name.to_vec(), jobid.clone(),
                    nack, additional_deliveries)).unwrap();
        self.process_error_ret
    }
}

#[test]
fn basic() {
    let queue = b"basicqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    disque.addjob(queue, b"my job", Duration::from_secs(10), None, None, None,
            None, None, false).unwrap();
    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx,
                JobStatus::AckJob, true));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().body(), b"my job".to_vec());
    assert!(rx.try_recv().is_err());
}

#[test]
fn error() {
    let queue = b"errorqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(queue, b"my job", Duration::from_secs(10), None,
            None, None, None, None, false).unwrap();
    disque.getjob(true, None, &[queue]).unwrap();
    disque.nack(&[jobid.as_bytes()]).unwrap();

    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx,
                JobStatus::AckJob, false));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().nack_additional_deliveries(), (1, 0));
    assert!(rx.try_recv().is_err());
}

#[test]
fn error_and_job() {
    let queue = b"errorjobqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(queue, b"my job", Duration::from_secs(10), None,
            None, None, None, None, false).unwrap();
    disque.getjob(true, None, &[queue]).unwrap();
    disque.nack(&[jobid.as_bytes()]).unwrap();

    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx,
                JobStatus::AckJob, true));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().nack_additional_deliveries(), (1, 0));
    assert_eq!(rx.try_recv().unwrap().body(), b"my job");
    assert!(rx.try_recv().is_err());
}
<commit_msg>Extract tests common initialization<commit_after>extern crate disqrust;
extern crate disque;

use std::time::Duration;
use std::sync::mpsc::{channel, Sender};

use disque::Disque;
use disqrust::{EventLoop, Handler, JobStatus};

enum HandlerCall {
    Job(Vec<u8>, String, Vec<u8>),
    Error(Vec<u8>, String, u32, u32),
}

impl HandlerCall {
    fn body(&self) -> Vec<u8> {
        match *self {
            HandlerCall::Job(_, _, ref body) => body.clone(),
            HandlerCall::Error(_, _, _, _) => panic!("getting body for error"),
        }
    }

    fn nack_additional_deliveries(&self) -> (u32, u32) {
        match *self {
            HandlerCall::Job(_, _, _) => panic!("getting nack for job"),
            HandlerCall::Error(_, _, a, b) => (a, b),
        }
    }
}

#[derive(Clone)]
struct MyHandler {
    sender: Sender<HandlerCall>,
    process_job_ret: JobStatus,
    process_error_ret: bool,
}

impl MyHandler {
    fn new(sender: Sender<HandlerCall>, process_job_ret: JobStatus,
            process_error_ret: bool) -> MyHandler {
        MyHandler {
            sender: sender,
            process_job_ret: process_job_ret,
            process_error_ret: process_error_ret,
        }
    }
}

impl Handler for MyHandler {
    fn process_job(&self, queue_name: &[u8], jobid: &String, body: Vec<u8>
            ) -> JobStatus {
        self.sender.send(HandlerCall::Job(queue_name.to_vec(), jobid.clone(),
                    body)).unwrap();
        self.process_job_ret.clone()
    }

    fn process_error(&self, queue_name: &[u8], jobid: &String, nack: u32,
            additional_deliveries: u32) -> bool {
        self.sender.send(HandlerCall::Error(queue_name.to_vec(), jobid.clone(),
                    nack, additional_deliveries)).unwrap();
        self.process_error_ret
    }
}

#[cfg(test)]
fn create_job(queue: &[u8], job: &[u8], nack: bool
        ) -> (Disque, Vec<u8>, Vec<u8>, String) {
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(queue, job, Duration::from_secs(10), None,
            None, None, None, None, false).unwrap();
    if nack {
        disque.getjob(true, None, &[queue]).unwrap();
        disque.nack(&[jobid.as_bytes()]).unwrap();
    }
    (disque, queue.to_vec(), job.to_vec(), jobid)
}

#[test]
fn basic() {
    let (disque, queue, job, _) = create_job(b"basic", b"job67", false);

    let (tx, rx) = channel();
    let handler = MyHandler::new(tx, JobStatus::AckJob, true);
    let mut el = EventLoop::new(disque, 1, handler);
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().body(), job);
    assert!(rx.try_recv().is_err());
}

#[test]
fn error() {
    let (disque, queue, _, _) = create_job(b"error", b"job456", true);

    let (tx, rx) = channel();
    let handler = MyHandler::new(tx, JobStatus::AckJob, false);
    let mut el = EventLoop::new(disque, 1, handler);
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().nack_additional_deliveries(), (1, 0));
    assert!(rx.try_recv().is_err());
}

#[test]
fn error_and_job() {
    let (disque, queue, job, _) = create_job(b"errorandjob", b"job123", true);

    let (tx, rx) = channel();
    let handler = MyHandler::new(tx, JobStatus::AckJob, true);
    let mut el = EventLoop::new(disque, 1, handler);
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    el.stop();
    assert_eq!(rx.try_recv().unwrap().nack_additional_deliveries(), (1, 0));
    assert_eq!(rx.try_recv().unwrap().body(), job);
    assert!(rx.try_recv().is_err());
}
<|endoftext|>


<commit_before>use std::io::fs;
use std::io;
use std::os;
use std::str;
use cargo::util::{process, ProcessBuilder};

use support::paths;
use support::{project, cargo_dir, mkdir_recursive, ProjectBuilder, ResultTest};

fn setup() {
}

/// Add an empty file with executable flags (and platform-dependent suffix).
/// TODO: move this to `ProjectBuilder` if other cases using this emerge.
fn fake_executable(proj: ProjectBuilder, dir: &Path, name: &str) -> ProjectBuilder {
    let path = proj.root().join(dir).join(format!("{}{}", name, os::consts::EXE_SUFFIX));
    mkdir_recursive(&Path::new(path.dirname())).assert();
    fs::File::create(&path).assert();
    let io::FileStat{perm, ..} = fs::stat(&path).assert();
    fs::chmod(&path, io::OtherExecute | perm).assert();
    proj
}

/// Copy real cargo exeutable just built to specified location, and
/// prepare to run it.
fn copied_executable_process(proj: &ProjectBuilder, name: &str, dir: &Path) -> ProcessBuilder {
    let name = format!("{}{}", name, os::consts::EXE_SUFFIX);
    let path_src = cargo_dir().join(name.clone());
    let path_dst = proj.root().join(dir).join(name);
    mkdir_recursive(&Path::new(path_dst.dirname())).assert();
    fs::copy(&path_src, &path_dst).assert();
    process(path_dst)
        .cwd(proj.root())
        .env("HOME", Some(paths::home().as_vec()))
}

// We can't entirely obliterate PATH because windows needs it for paths to
// things like libgcc, but we want to filter out everything which has a `cargo`
// installation as we don't want it to muck with the --list tests
fn new_path() -> Vec<Path> {
    let path = os::getenv_as_bytes("PATH").unwrap_or(Vec::new());
    os::split_paths(path).move_iter().filter(|p| {
        !p.join(format!("cargo{}", os::consts::EXE_SUFFIX)).exists()
    }).collect()
}
test!(list_commands_non_overlapping {
    // lib/cargo | cargo-3
    // bin/       | cargo-2
    // PATH       | cargo-1
    // Check if --list searches all 3 targets.
    // Also checks that results are in lexicographic order.
    let proj = project("list-non-overlapping");
    let proj = fake_executable(proj, &Path::new("lib/cargo"), "cargo-3");
    let proj = fake_executable(proj, &Path::new("bin"), "cargo-2");
    let proj = fake_executable(proj, &Path::new("path-test"), "cargo-1");
    let pr = copied_executable_process(&proj, "cargo", &Path::new("bin"));

    let mut path = new_path();
    path.push(proj.root().join("path-test"));
    let path = os::join_paths(path.as_slice()).unwrap();
    let output = pr.arg("-v").arg("--list").env("PATH", Some(path.as_slice()));
    let output = output.exec_with_output().assert();
    let output = str::from_utf8(output.output.as_slice()).assert();
    assert!(output.contains("\n    1\n"), "missing 1: {}", output);
    assert!(output.contains("\n    2\n"), "missing 2: {}", output);
    assert!(output.contains("\n    3\n"), "missing 3: {}", output);
})
<commit_msg>Slim down the `cargo --list` test<commit_after>use std::io::fs;
use std::io;
use std::os;
use std::str;
use cargo::util::process;

use support::paths;
use support::{project, cargo_dir, mkdir_recursive, ProjectBuilder, ResultTest};

fn setup() {
}

/// Add an empty file with executable flags (and platform-dependent suffix).
/// TODO: move this to `ProjectBuilder` if other cases using this emerge.
fn fake_executable(proj: ProjectBuilder, dir: &Path, name: &str) -> ProjectBuilder {
    let path = proj.root().join(dir).join(format!("{}{}", name, os::consts::EXE_SUFFIX));
    mkdir_recursive(&Path::new(path.dirname())).assert();
    fs::File::create(&path).assert();
    let io::FileStat{perm, ..} = fs::stat(&path).assert();
    fs::chmod(&path, io::OtherExecute | perm).assert();
    proj
}

// We can't entirely obliterate PATH because windows needs it for paths to
// things like libgcc, but we want to filter out everything which has a `cargo`
// installation as we don't want it to muck with the --list tests
fn new_path() -> Vec<Path> {
    let path = os::getenv_as_bytes("PATH").unwrap_or(Vec::new());
    os::split_paths(path).move_iter().filter(|p| {
        !p.join(format!("cargo{}", os::consts::EXE_SUFFIX)).exists()
    }).collect()
}
test!(list_commands_looks_at_path {
    let proj = project("list-non-overlapping");
    let proj = fake_executable(proj, &Path::new("path-test"), "cargo-1");
    let pr = process(cargo_dir().join("cargo")).cwd(proj.root())
                    .env("HOME", Some(paths::home()));

    let mut path = new_path();
    path.push(proj.root().join("path-test"));
    let path = os::join_paths(path.as_slice()).unwrap();
    let output = pr.arg("-v").arg("--list").env("PATH", Some(path.as_slice()));
    let output = output.exec_with_output().assert();
    let output = str::from_utf8(output.output.as_slice()).assert();
    assert!(output.contains("\n    1\n"), "missing 1: {}", output);
})
<|endoftext|>