<commit_before>// This file is part of the uutils coreutils package.
//
// (c) Joseph Crail <jbcrail@gmail.com>
// (c) Jian Zeng <anonymousknight96 AT gmail.com>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

#[cfg(windows)]
extern crate dunce;
#[cfg(target_os = "redox")]
extern crate termion;

#[cfg(unix)]
use libc::{
    mode_t, S_IRGRP, S_IROTH, S_IRUSR, S_ISGID, S_ISUID, S_ISVTX, S_IWGRP, S_IWOTH, S_IWUSR,
    S_IXGRP, S_IXOTH, S_IXUSR,
};
use std::borrow::Cow;
use std::env;
use std::fs;
#[cfg(target_os = "redox")]
use std::io;
use std::io::Result as IOResult;
use std::io::{Error, ErrorKind};
#[cfg(any(unix, target_os = "redox"))]
use std::os::unix::fs::MetadataExt;
use std::path::{Component, Path, PathBuf};

#[cfg(unix)]
macro_rules! has {
    ($mode:expr, $perm:expr) => {
        $mode & ($perm as u32) != 0
    };
}

pub fn resolve_relative_path(path: &Path) -> Cow<Path> {
    if path.components().all(|e| e != Component::ParentDir) {
        return path.into();
    }
    let root = Component::RootDir.as_os_str();
    let mut result = env::current_dir().unwrap_or(PathBuf::from(root));
    for comp in path.components() {
        match comp {
            Component::ParentDir => {
                if let Ok(p) = result.read_link() {
                    result = p;
                }
                result.pop();
            }
            Component::CurDir => (),
            Component::RootDir | Component::Normal(_) | Component::Prefix(_) => {
                result.push(comp.as_os_str())
            }
        }
    }
    result.into()
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CanonicalizeMode {
    None,
    Normal,
    Existing,
    Missing,
}

fn resolve<P: AsRef<Path>>(original: P) -> IOResult<PathBuf> {
    const MAX_LINKS_FOLLOWED: u32 = 255;
    let mut followed = 0;
    let mut result = original.as_ref().to_path_buf();
    loop {
        if followed == MAX_LINKS_FOLLOWED {
            return Err(Error::new(
                ErrorKind::InvalidInput,
                "maximum links followed",
            ));
        }

        match fs::symlink_metadata(&result) {
            Err(e) => return Err(e),
            Ok(ref m) if !m.file_type().is_symlink() => break,
            Ok(..) => {
                followed += 1;
                match fs::read_link(&result) {
                    Ok(path) => {
                        result.pop();
                        result.push(path);
                    }
                    Err(e) => {
                        return Err(e);
                    }
                }
            }
        }
    }
    Ok(result)
}

pub fn canonicalize<P: AsRef<Path>>(original: P, can_mode: CanonicalizeMode) -> IOResult<PathBuf> {
    // Create an absolute path
    let original = original.as_ref();
    let original = if original.is_absolute() {
        original.to_path_buf()
    } else {
        dunce::canonicalize(env::current_dir().unwrap())
            .unwrap()
            .join(original)
    };

    let mut result = PathBuf::new();
    let mut parts = vec![];

    // Split path by directory separator; add prefix (Windows-only) and root
    // directory to final path buffer; add remaining parts to temporary
    // vector for canonicalization.
    for part in original.components() {
        match part {
            Component::Prefix(_) | Component::RootDir => {
                result.push(part.as_os_str());
            }
            Component::CurDir => (),
            Component::ParentDir => {
                parts.pop();
            }
            Component::Normal(_) => {
                parts.push(part.as_os_str());
            }
        }
    }

    // Resolve the symlinks where possible
    if !parts.is_empty() {
        for part in parts[..parts.len() - 1].iter() {
            result.push(part);

            if can_mode == CanonicalizeMode::None {
                continue;
            }

            match resolve(&result) {
                Err(e) => match can_mode {
                    CanonicalizeMode::Missing => continue,
                    _ => return Err(e),
                },
                Ok(path) => {
                    result.pop();
                    result.push(path);
                }
            }
        }

        result.push(parts.last().unwrap());

        match resolve(&result) {
            Err(e) => {
                if can_mode == CanonicalizeMode::Existing {
                    return Err(e);
                }
            }
            Ok(path) => {
                result.pop();
                result.push(path);
            }
        }
    }
    Ok(result)
}

#[cfg(unix)]
pub fn is_stdin_interactive() -> bool {
    unsafe { libc::isatty(libc::STDIN_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stdin_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stdin_interactive() -> bool {
    termion::is_tty(&io::stdin())
}

#[cfg(unix)]
pub fn is_stdout_interactive() -> bool {
    unsafe { libc::isatty(libc::STDOUT_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stdout_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stdout_interactive() -> bool {
    termion::is_tty(&io::stdout())
}

#[cfg(unix)]
pub fn is_stderr_interactive() -> bool {
    unsafe { libc::isatty(libc::STDERR_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stderr_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stderr_interactive() -> bool {
    termion::is_tty(&io::stderr())
}

#[cfg(not(unix))]
#[allow(unused_variables)]
pub fn display_permissions(metadata: &fs::Metadata) -> String {
    String::from("---------")
}

#[cfg(unix)]
pub fn display_permissions(metadata: &fs::Metadata) -> String {
    let mode: mode_t = metadata.mode() as mode_t;
    display_permissions_unix(mode as u32)
}

#[cfg(unix)]
pub fn display_permissions_unix(mode: u32) -> String {
    let mut result = String::with_capacity(9);
    result.push(if has!(mode, S_IRUSR) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWUSR) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISUID) {
        if has!(mode, S_IXUSR) {
            's'
        } else {
            'S'
        }
    } else if has!(mode, S_IXUSR) {
        'x'
    } else {
        '-'
    });

    result.push(if has!(mode, S_IRGRP) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWGRP) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISGID) {
        if has!(mode, S_IXGRP) {
            's'
        } else {
            'S'
        }
    } else if has!(mode, S_IXGRP) {
        'x'
    } else {
        '-'
    });

    result.push(if has!(mode, S_IROTH) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWOTH) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISVTX) {
        if has!(mode, S_IXOTH) {
            't'
        } else {
            'T'
        }
    } else if has!(mode, S_IXOTH) {
        'x'
    } else {
        '-'
    });

    result
}
<commit_msg>uucore/refactor ~ fix `cargo clippy` complaints (or_fun_call)<commit_after>// This file is part of the uutils coreutils package.
//
// (c) Joseph Crail <jbcrail@gmail.com>
// (c) Jian Zeng <anonymousknight96 AT gmail.com>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

#[cfg(windows)]
extern crate dunce;
#[cfg(target_os = "redox")]
extern crate termion;

#[cfg(unix)]
use libc::{
    mode_t, S_IRGRP, S_IROTH, S_IRUSR, S_ISGID, S_ISUID, S_ISVTX, S_IWGRP, S_IWOTH, S_IWUSR,
    S_IXGRP, S_IXOTH, S_IXUSR,
};
use std::borrow::Cow;
use std::env;
use std::fs;
#[cfg(target_os = "redox")]
use std::io;
use std::io::Result as IOResult;
use std::io::{Error, ErrorKind};
#[cfg(any(unix, target_os = "redox"))]
use std::os::unix::fs::MetadataExt;
use std::path::{Component, Path, PathBuf};

#[cfg(unix)]
macro_rules! has {
    ($mode:expr, $perm:expr) => {
        $mode & ($perm as u32) != 0
    };
}

pub fn resolve_relative_path(path: &Path) -> Cow<Path> {
    if path.components().all(|e| e != Component::ParentDir) {
        return path.into();
    }
    let root = Component::RootDir.as_os_str();
    let mut result = env::current_dir().unwrap_or_else(|_| PathBuf::from(root));
    for comp in path.components() {
        match comp {
            Component::ParentDir => {
                if let Ok(p) = result.read_link() {
                    result = p;
                }
                result.pop();
            }
            Component::CurDir => (),
            Component::RootDir | Component::Normal(_) | Component::Prefix(_) => {
                result.push(comp.as_os_str())
            }
        }
    }
    result.into()
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CanonicalizeMode {
    None,
    Normal,
    Existing,
    Missing,
}

fn resolve<P: AsRef<Path>>(original: P) -> IOResult<PathBuf> {
    const MAX_LINKS_FOLLOWED: u32 = 255;
    let mut followed = 0;
    let mut result = original.as_ref().to_path_buf();
    loop {
        if followed == MAX_LINKS_FOLLOWED {
            return Err(Error::new(
                ErrorKind::InvalidInput,
                "maximum links followed",
            ));
        }

        match fs::symlink_metadata(&result) {
            Err(e) => return Err(e),
            Ok(ref m) if !m.file_type().is_symlink() => break,
            Ok(..) => {
                followed += 1;
                match fs::read_link(&result) {
                    Ok(path) => {
                        result.pop();
                        result.push(path);
                    }
                    Err(e) => {
                        return Err(e);
                    }
                }
            }
        }
    }
    Ok(result)
}

pub fn canonicalize<P: AsRef<Path>>(original: P, can_mode: CanonicalizeMode) -> IOResult<PathBuf> {
    // Create an absolute path
    let original = original.as_ref();
    let original = if original.is_absolute() {
        original.to_path_buf()
    } else {
        dunce::canonicalize(env::current_dir().unwrap())
            .unwrap()
            .join(original)
    };

    let mut result = PathBuf::new();
    let mut parts = vec![];

    // Split path by directory separator; add prefix (Windows-only) and root
    // directory to final path buffer; add remaining parts to temporary
    // vector for canonicalization.
    for part in original.components() {
        match part {
            Component::Prefix(_) | Component::RootDir => {
                result.push(part.as_os_str());
            }
            Component::CurDir => (),
            Component::ParentDir => {
                parts.pop();
            }
            Component::Normal(_) => {
                parts.push(part.as_os_str());
            }
        }
    }

    // Resolve the symlinks where possible
    if !parts.is_empty() {
        for part in parts[..parts.len() - 1].iter() {
            result.push(part);

            if can_mode == CanonicalizeMode::None {
                continue;
            }

            match resolve(&result) {
                Err(e) => match can_mode {
                    CanonicalizeMode::Missing => continue,
                    _ => return Err(e),
                },
                Ok(path) => {
                    result.pop();
                    result.push(path);
                }
            }
        }

        result.push(parts.last().unwrap());

        match resolve(&result) {
            Err(e) => {
                if can_mode == CanonicalizeMode::Existing {
                    return Err(e);
                }
            }
            Ok(path) => {
                result.pop();
                result.push(path);
            }
        }
    }
    Ok(result)
}

#[cfg(unix)]
pub fn is_stdin_interactive() -> bool {
    unsafe { libc::isatty(libc::STDIN_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stdin_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stdin_interactive() -> bool {
    termion::is_tty(&io::stdin())
}

#[cfg(unix)]
pub fn is_stdout_interactive() -> bool {
    unsafe { libc::isatty(libc::STDOUT_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stdout_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stdout_interactive() -> bool {
    termion::is_tty(&io::stdout())
}

#[cfg(unix)]
pub fn is_stderr_interactive() -> bool {
    unsafe { libc::isatty(libc::STDERR_FILENO) == 1 }
}

#[cfg(windows)]
pub fn is_stderr_interactive() -> bool {
    false
}

#[cfg(target_os = "redox")]
pub fn is_stderr_interactive() -> bool {
    termion::is_tty(&io::stderr())
}

#[cfg(not(unix))]
#[allow(unused_variables)]
pub fn display_permissions(metadata: &fs::Metadata) -> String {
    String::from("---------")
}

#[cfg(unix)]
pub fn display_permissions(metadata: &fs::Metadata) -> String {
    let mode: mode_t = metadata.mode() as mode_t;
    display_permissions_unix(mode as u32)
}

#[cfg(unix)]
pub fn display_permissions_unix(mode: u32) -> String {
    let mut result = String::with_capacity(9);
    result.push(if has!(mode, S_IRUSR) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWUSR) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISUID) {
        if has!(mode, S_IXUSR) {
            's'
        } else {
            'S'
        }
    } else if has!(mode, S_IXUSR) {
        'x'
    } else {
        '-'
    });

    result.push(if has!(mode, S_IRGRP) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWGRP) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISGID) {
        if has!(mode, S_IXGRP) {
            's'
        } else {
            'S'
        }
    } else if has!(mode, S_IXGRP) {
        'x'
    } else {
        '-'
    });

    result.push(if has!(mode, S_IROTH) { 'r' } else { '-' });
    result.push(if has!(mode, S_IWOTH) { 'w' } else { '-' });
    result.push(if has!(mode, S_ISVTX) {
        if has!(mode, S_IXOTH) {
            't'
        } else {
            'T'
        }
    } else if has!(mode, S_IXOTH) {
        'x'
    } else {
        '-'
    });

    result
}
<|endoftext|>


<commit_before>#[cfg(test)]
use ::{card, deck, hand};

#[test]
fn create_hand() {
    let mut deck = deck::Deck::new();
    let mut cards: Vec<card::Card> = vec![];
    for _i in 0..5 {
        cards.push(deck.draw().unwrap());
    }
    let hand = hand::Hand::new(cards);

    assert_eq!(hand.cards.len(), 5);
}

#[test]
fn has_flush() {
    let a = card::Card::new(card::Value::Three, card::Suit::Hearts);
    let b = card::Card::new(card::Value::Six, card::Suit::Hearts);
    let c = card::Card::new(card::Value::Ten, card::Suit::Hearts);
    let d = card::Card::new(card::Value::Queen, card::Suit::Hearts);
    let e = card::Card::new(card::Value::Two, card::Suit::Hearts);
    let cards: Vec<card::Card> = vec![a, b, c, d, e];
    let hand: hand::Hand = hand::Hand::new(cards);

    assert_eq!(hand.has_flush(), true);
}

#[test]
fn no_flush() {
    let a = card::Card::new(card::Value::Three, card::Suit::Hearts);
    let b = card::Card::new(card::Value::Six, card::Suit::Spades);
    let c = card::Card::new(card::Value::Ten, card::Suit::Hearts);
    let d = card::Card::new(card::Value::Queen, card::Suit::Hearts);
    let e = card::Card::new(card::Value::Two, card::Suit::Hearts);
    let cards: Vec<card::Card> = vec![a, b, c, d, e];
    let hand: hand::Hand = hand::Hand::new(cards);

    assert_eq!(hand.has_flush(), false);
}

#[test]
fn compare_lesser_poker_hand() {
    assert_eq!(hand::PokerRanking::HighCard < hand::PokerRanking::OnePair, true);
    assert_eq!(hand::PokerRanking::HighCard == hand::PokerRanking::OnePair, false);
    assert_eq!(hand::PokerRanking::HighCard > hand::PokerRanking::OnePair, false);
}

#[test]
fn compare_greater_poker_hand() {
    assert_eq!(hand::PokerRanking::Flush < hand::PokerRanking::TwoPair, false);
    assert_eq!(hand::PokerRanking::Flush == hand::PokerRanking::TwoPair, false);
    assert_eq!(hand::PokerRanking::Flush > hand::PokerRanking::TwoPair, true);
}

#[test]
fn compare_equal_poker_hands() {
    assert_eq!(hand::PokerRanking::Trips < hand::PokerRanking::Trips, false);
    assert_eq!(hand::PokerRanking::Trips == hand::PokerRanking::Trips, true);
    assert_eq!(hand::PokerRanking::Trips > hand::PokerRanking::Trips, false);
}
<commit_msg>Changed ranking tests to evaluate raw value of PokerRanking enums<commit_after>#[cfg(test)]
use ::{card, deck, hand};

#[test]
fn create_hand() {
    let mut deck = deck::Deck::new();
    let mut cards: Vec<card::Card> = vec![];
    for _i in 0..5 {
        cards.push(deck.draw().unwrap());
    }
    let hand = hand::Hand::new(cards);

    assert_eq!(hand.cards.len(), 5);
}

#[test]
fn has_flush() {
    let a = card::Card::new(card::Value::Three, card::Suit::Hearts);
    let b = card::Card::new(card::Value::Six, card::Suit::Hearts);
    let c = card::Card::new(card::Value::Ten, card::Suit::Hearts);
    let d = card::Card::new(card::Value::Queen, card::Suit::Hearts);
    let e = card::Card::new(card::Value::Two, card::Suit::Hearts);
    let cards: Vec<card::Card> = vec![a, b, c, d, e];
    let hand: hand::Hand = hand::Hand::new(cards);

    assert_eq!(hand.has_flush(), true);
}

#[test]
fn no_flush() {
    let a = card::Card::new(card::Value::Three, card::Suit::Hearts);
    let b = card::Card::new(card::Value::Six, card::Suit::Spades);
    let c = card::Card::new(card::Value::Ten, card::Suit::Hearts);
    let d = card::Card::new(card::Value::Queen, card::Suit::Hearts);
    let e = card::Card::new(card::Value::Two, card::Suit::Hearts);
    let cards: Vec<card::Card> = vec![a, b, c, d, e];
    let hand: hand::Hand = hand::Hand::new(cards);

    assert_eq!(hand.has_flush(), false);
}

#[test]
fn high_card_raw_value() {
    assert_eq!(hand::PokerRanking::HighCard as u32, 0);
}

#[test]
fn one_pair_raw_value() {
    assert_eq!(hand::PokerRanking::OnePair as u32, 1);
}

#[test]
fn two_pair_raw_value() {
    assert_eq!(hand::PokerRanking::TwoPair as u32, 2);
}

#[test]
fn trips_raw_value() {
    assert_eq!(hand::PokerRanking::Trips as u32, 3);
}

#[test]
fn straight_raw_value() {
    assert_eq!(hand::PokerRanking::Straight as u32, 4);
}

#[test]
fn flush_raw_value() {
    assert_eq!(hand::PokerRanking::Flush as u32, 5);
}

#[test]
fn full_house_raw_value() {
    assert_eq!(hand::PokerRanking::FullHouse as u32, 6);
}

#[test]
fn quads_raw_value() {
    assert_eq!(hand::PokerRanking::Quads as u32, 7);
}

#[test]
fn straight_flush_raw_value() {
    assert_eq!(hand::PokerRanking::StraightFlush as u32, 8);
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate log;
extern crate env_logger;

use std::error::Error;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

fn compare_and_cast (primary: char, secondary: char) -> u32 {
    let mut val: u32 = 0;

    debug!("{} -> {}", primary, secondary);
    if primary == secondary {
        debug!(" are equal");
        val = primary.to_string().parse::<u32>().unwrap();
    } else {
        debug!(" are not equal");
    }

    return val;
}

fn main() {
    let _logger = env_logger::init();

    let path = Path::new("data/input");
    let display = path.display();

    // Open the path in read-only mode, returns `io::Result<File>`
    let mut file = match File::open(&path) {
        // The `description` method of `io::Error` returns a string that
        // describes the error
        Err(why) => panic!("couldn't open {}: {}", display,
                                                   why.description()),
        Ok(file) => file,
    };

    // Read the file contents into a string, returns `io::Result<usize>`
    let mut captcha = String::new();
    match file.read_to_string(&mut captcha) {
        Err(why) => panic!("couldn't read {}: {}", display,
                                                   why.description()),
        Ok(_) => debug!("{}", captcha),
    }
    let captcha = captcha.trim();

    let mut sum: u32 = 0;
    for (primary_char, secondary_char) in captcha.chars().zip(captcha.chars().cycle().skip(1)) {
        sum = sum + compare_and_cast (primary_char, secondary_char);
        debug!("sum now == {}", sum);
    }

    print!("The answer to part1 is {}\n", sum);

    // Problem states that our captcha should be of even length, double check
    assert_eq! (captcha.len() % 2, 0);

    // Figure out the starting position of the 2nd iterator
    let half_captcha_len = captcha.len() / 2;
    let half_way_iter = captcha.chars().cycle().skip(half_captcha_len);

    let mut sum: u32 = 0;
    for (primary_char, secondary_char) in captcha.chars().take(half_captcha_len).zip(half_way_iter) {
        // Since the comparison is symmetric (we will compare index 0
        // with index n/2 and then later on index n/2 with index 0),
        // we can perform half of the comparisons and just double any
        // valid summations
        sum = sum + 2 * compare_and_cast (primary_char, secondary_char);
        debug!("sum now == {}", sum);
    }

    print!("The answer to part2 is {}\n", sum);
}
<commit_msg>day01: Clean up logging<commit_after>#[macro_use]
extern crate log;
extern crate env_logger;

use std::error::Error;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

fn compare_and_cast (primary: char, secondary: char) -> u32 {
    debug!("{} -> {}", primary, secondary);
    if primary == secondary {
        return primary.to_string().parse::<u32>().unwrap();
    } else {
        return 0;
    }
}

fn main() {
    let _logger = env_logger::init();

    let path = Path::new("data/input");
    let display = path.display();

    // Open the path in read-only mode, returns `io::Result<File>`
    let mut file = match File::open(&path) {
        // The `description` method of `io::Error` returns a string that
        // describes the error
        Err(why) => panic!("couldn't open {}: {}", display,
                                                   why.description()),
        Ok(file) => file,
    };

    // Read the file contents into a string, returns `io::Result<usize>`
    let mut captcha = String::new();
    match file.read_to_string(&mut captcha) {
        Err(why) => panic!("couldn't read {}: {}", display,
                                                   why.description()),
        Ok(_) => debug!("{}", captcha),
    }
    let captcha = captcha.trim(); // remove the trailing whitespace

    let mut sum: u32 = 0;
    for (primary_char, secondary_char) in captcha.chars().zip(captcha.chars().cycle().skip(1)) {
        sum = sum + compare_and_cast (primary_char, secondary_char);
        debug!("sum now == {}", sum);
    }

    print!("The answer to part1 is {}\n", sum);

    // Problem states that our captcha should be of even length, double check
    assert_eq! (captcha.len() % 2, 0);

    // Figure out the starting position of the 2nd iterator
    let half_captcha_len = captcha.len() / 2;
    let half_way_iter = captcha.chars().cycle().skip(half_captcha_len);

    let mut sum: u32 = 0;
    for (primary_char, secondary_char) in captcha.chars().take(half_captcha_len).zip(half_way_iter) {
        // Since the comparison is symmetric (we will compare index 0
        // with index n/2 and then later on index n/2 with index 0),
        // we can perform half of the comparisons and just double any
        // valid summations
        sum = sum + 2 * compare_and_cast (primary_char, secondary_char);
        debug!("sum now == {}", sum);
    }

    print!("The answer to part2 is {}\n", sum);
}
<|endoftext|>


<commit_before>pub mod test_common;
pub mod distance;

mod partition;
mod bounds;

use self::bounds::*;
use self::distance::*;

use std::cmp;

pub trait KdtreePointTrait: Copy + PartialEq {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<KdtreePoint> {
    nodes: Vec<KdtreeNode<KdtreePoint>>,

    node_adding_dimension: usize,
    node_depth_during_last_rebuild: usize,
    current_node_depth: usize,
}

impl<KdtreePoint: KdtreePointTrait> Kdtree<KdtreePoint> {
    pub fn new(mut points: &mut [KdtreePoint]) -> Kdtree<KdtreePoint> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let mut tree = Kdtree {
            nodes: vec![],
            node_adding_dimension: 0,
            node_depth_during_last_rebuild: 0,
            current_node_depth: 0,
        };

        tree.rebuild_tree(&mut points);

        tree
    }

    pub fn rebuild_tree(&mut self, points : &mut [KdtreePoint]) {
        self.nodes.clear();

        self.node_depth_during_last_rebuild = 0;
        self.current_node_depth = 0;

        let rect = Bounds::new_from_points(points);
        self.build_tree(points, &rect, 1);
    }

    /// Can be used if you are sure that the tree is degenerated or if you will never again insert the nodes into the tree.
    pub fn gather_points_and_rebuild(&mut self) {
        let mut points : Vec<KdtreePoint> = vec![];
        self.gather_points(0,&mut points);

        self.rebuild_tree(&mut points);
    }

    pub fn nearest_search(&self, node: &KdtreePoint) -> KdtreePoint
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance, &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    pub fn has_neighbor_in_range(&self, node: &KdtreePoint, range: f64) -> bool {
        let squared_range = range * range;

        self.distance_squared_to_nearest(node) <= squared_range
    }

    pub fn distance_squared_to_nearest(&self, node: &KdtreePoint) -> f64 {
        squared_euclidean(&self.nearest_search(node).dims(), node.dims())
    }

    pub fn insert_nodes_and_rebuild(&mut self, nodes_to_add : &mut [KdtreePoint]) {
        let mut pts : Vec<KdtreePoint> = vec![];
        self.gather_points(0, &mut pts);
        pts.extend(nodes_to_add.iter());

        self.rebuild_tree(&mut pts);
    }

    pub fn insert_node(&mut self, node_to_add : KdtreePoint) {

        let mut current_index = 0;
        let dimension = self.node_adding_dimension;
        let index_of_new_node = self.add_node(node_to_add,dimension,node_to_add.dims()[dimension]);
        self.node_adding_dimension = ( dimension + 1) % node_to_add.dims().len();
        let mut should_pop_node = false;

        let mut depth = 0;
        loop {

            depth +=1 ;
            let current_node = &mut self.nodes[current_index];

            if node_to_add.dims()[current_node.dimension] <= current_node.split_on {
                if let Some(left_node_index) = current_node.left_node {
                    current_index = left_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.left_node = Some(index_of_new_node);
                    }
                    break;
                }
            } else {
                if let Some(right_node_index) = current_node.right_node {
                    current_index = right_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.right_node = Some(index_of_new_node);
                    }
                    break;
                }
            }
        }

        if should_pop_node {
            self.nodes.pop();
        }

        if self.node_depth_during_last_rebuild as f64 * 8.0 < depth as f64  {
            self.gather_points_and_rebuild();
        }
    }

    fn nearest_search_impl(&self, p: &KdtreePoint, searched_index: usize, best_distance_squared: &mut f64, best_leaf_found: &mut usize) {
        let node = &self.nodes[searched_index];

        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[node.dimension];

        let (closer_node, farther_node) = if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value], &[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: KdtreePoint, dimension: usize, split_on: f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on);

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [KdtreePoint], bounds: &Bounds, depth : usize) -> usize {
        let splitting_index = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());
        let pivot_value = nodes[splitting_index].dims()[bounds.get_widest_dim()];

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), pivot_value);
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(pivot_value, bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect, depth+1);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(pivot_value, bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect, depth+1);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        self.node_depth_during_last_rebuild =  cmp::max(self.node_depth_during_last_rebuild,depth);

        node_id
    }

    fn gather_points(&self, current_index: usize, points : &mut Vec<KdtreePoint>){
        points.push(self.nodes[current_index].point);
        if let Some(left_index) = self.nodes[current_index].left_node {
            self.gather_points(left_index, points);
        }

        if let Some(right_index) = self.nodes[current_index].right_node {
            self.gather_points(right_index, points);
        }
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value: f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension: splitting_dimension,
            split_on: split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let mut empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(&mut empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(&mut qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(&mut point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    #[test]
    fn has_neighbor_in_range() {
        let mut vec: Vec<Point2WithId> = vec![Point2WithId::new(0,2.,0.)];

        let tree = Kdtree::new(&mut vec);

        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 0.));
        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 1.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 2.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 300.));
    }

    #[test]
    fn incremental_add_adds_as_expected() {
        //this test is tricky because it can have problems with the automatic tree rebuild.

        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        tree.insert_node(Point2WithId::new(0,1.,0.));
        tree.insert_node(Point2WithId::new(0,-1.,0.));

        assert_eq!(tree.nodes.len(), 3);
        assert_eq!(tree.nodes[0].dimension, 0);

        assert_eq!(tree.nodes[0].left_node.is_some(), true);
        assert_eq!(tree.nodes[1].point.dims()[0], 1.);
        assert_eq!(tree.nodes[2].point.dims()[0], -1.);

        assert_eq!(tree.nodes[0].right_node.is_some(), true);
    }

    #[test]
    fn incremental_add_filters_duplicates() {
        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        let node = Point2WithId::new(0,1.,0.);
        tree.insert_node(node);
        tree.insert_node(node);

        assert_eq!(tree.nodes.len(), 2);
    }

    fn qc_value_vec_to_2d_points_vec(xs: &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec: Vec<Point2WithId> = vec![];
        for i in 0..xs.len() {
            let mut is_duplicated_value = false;
            for j in 0..i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<commit_msg>tweaking<commit_after>pub mod test_common;
pub mod distance;

mod partition;
mod bounds;

use self::bounds::*;
use self::distance::*;

use std::cmp;

pub trait KdtreePointTrait: Copy + PartialEq {
    fn dims(&self) -> &[f64];
}

pub struct Kdtree<KdtreePoint> {
    nodes: Vec<KdtreeNode<KdtreePoint>>,

    node_adding_dimension: usize,
    node_depth_during_last_rebuild: usize,
    current_node_depth: usize,
}

impl<KdtreePoint: KdtreePointTrait> Kdtree<KdtreePoint> {
    pub fn new(mut points: &mut [KdtreePoint]) -> Kdtree<KdtreePoint> {
        if points.len() == 0 {
            panic!("empty vector point not allowed");
        }

        let mut tree = Kdtree {
            nodes: vec![],
            node_adding_dimension: 0,
            node_depth_during_last_rebuild: 0,
            current_node_depth: 0,
        };

        tree.rebuild_tree(&mut points);

        tree
    }

    pub fn rebuild_tree(&mut self, points : &mut [KdtreePoint]) {
        self.nodes.clear();

        self.node_depth_during_last_rebuild = 0;
        self.current_node_depth = 0;

        let rect = Bounds::new_from_points(points);
        self.build_tree(points, &rect, 1);
    }

    /// Can be used if you are sure that the tree is degenerated or if you will never again insert the nodes into the tree.
    pub fn gather_points_and_rebuild(&mut self) {
        let mut points : Vec<KdtreePoint> = vec![];
        self.gather_points(0,&mut points);

        self.rebuild_tree(&mut points);
    }

    pub fn nearest_search(&self, node: &KdtreePoint) -> KdtreePoint
    {
        let mut nearest_neighbor = 0usize;
        let mut best_distance = squared_euclidean(node.dims(), &self.nodes[0].point.dims());
        self.nearest_search_impl(node, 0usize, &mut best_distance, &mut nearest_neighbor);

        self.nodes[nearest_neighbor].point
    }

    pub fn has_neighbor_in_range(&self, node: &KdtreePoint, range: f64) -> bool {
        let squared_range = range * range;

        self.distance_squared_to_nearest(node) <= squared_range
    }

    pub fn distance_squared_to_nearest(&self, node: &KdtreePoint) -> f64 {
        squared_euclidean(&self.nearest_search(node).dims(), node.dims())
    }

    pub fn insert_nodes_and_rebuild(&mut self, nodes_to_add : &mut [KdtreePoint]) {
        let mut pts : Vec<KdtreePoint> = vec![];
        self.gather_points(0, &mut pts);
        pts.extend(nodes_to_add.iter());

        self.rebuild_tree(&mut pts);
    }

    pub fn insert_node(&mut self, node_to_add : KdtreePoint) {

        let mut current_index = 0;
        let dimension = self.node_adding_dimension;
        let index_of_new_node = self.add_node(node_to_add,dimension,node_to_add.dims()[dimension]);
        self.node_adding_dimension = ( dimension + 1) % node_to_add.dims().len();
        let mut should_pop_node = false;

        let mut depth = 0;
        loop {

            depth +=1 ;
            let current_node = &mut self.nodes[current_index];

            if node_to_add.dims()[current_node.dimension] <= current_node.split_on {
                if let Some(left_node_index) = current_node.left_node {
                    current_index = left_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.left_node = Some(index_of_new_node);
                    }
                    break;
                }
            } else {
                if let Some(right_node_index) = current_node.right_node {
                    current_index = right_node_index
                } else {
                    if current_node.point.eq(&node_to_add) {
                        should_pop_node = true;
                    } else {
                        current_node.right_node = Some(index_of_new_node);
                    }
                    break;
                }
            }
        }

        if should_pop_node {
            self.nodes.pop();
        }

        if self.node_depth_during_last_rebuild as f64 * 2.0 < depth as f64  {
            self.gather_points_and_rebuild();
        }
    }

    fn nearest_search_impl(&self, p: &KdtreePoint, searched_index: usize, best_distance_squared: &mut f64, best_leaf_found: &mut usize) {
        let node = &self.nodes[searched_index];

        let splitting_value = node.split_on;
        let point_splitting_dim_value = p.dims()[node.dimension];

        let (closer_node, farther_node) = if point_splitting_dim_value <= splitting_value {
            (node.left_node, node.right_node)
        } else {
            (node.right_node, node.left_node)
        };

        if let Some(closer_node) = closer_node {
            self.nearest_search_impl(p, closer_node, best_distance_squared, best_leaf_found);
        }

        let distance = squared_euclidean(p.dims(), node.point.dims());
        if distance < *best_distance_squared {
            *best_distance_squared = distance;
            *best_leaf_found = searched_index;
        }

        if let Some(farther_node) = farther_node {
            let distance_on_single_dimension = squared_euclidean(&[splitting_value], &[point_splitting_dim_value]);

            if distance_on_single_dimension <= *best_distance_squared {
                self.nearest_search_impl(p, farther_node, best_distance_squared, best_leaf_found);
            }
        }
    }

    fn add_node(&mut self, p: KdtreePoint, dimension: usize, split_on: f64) -> usize {
        let node = KdtreeNode::new(p, dimension, split_on);

        self.nodes.push(node);
        self.nodes.len() - 1
    }

    fn build_tree(&mut self, nodes: &mut [KdtreePoint], bounds: &Bounds, depth : usize) -> usize {
        let splitting_index = partition::partition_sliding_midpoint(nodes, bounds.get_midvalue_of_widest_dim(), bounds.get_widest_dim());
        let pivot_value = nodes[splitting_index].dims()[bounds.get_widest_dim()];

        let node_id = self.add_node(nodes[splitting_index], bounds.get_widest_dim(), pivot_value);
        let nodes_len = nodes.len();

        if splitting_index > 0 {
            let left_rect = bounds.clone_moving_max(pivot_value, bounds.get_widest_dim());
            let left_child_id = self.build_tree(&mut nodes[0..splitting_index], &left_rect, depth+1);
            self.nodes[node_id].left_node = Some(left_child_id);
        }

        if splitting_index < nodes.len() - 1 {
            let right_rect = bounds.clone_moving_min(pivot_value, bounds.get_widest_dim());

            let right_child_id = self.build_tree(&mut nodes[splitting_index + 1..nodes_len], &right_rect, depth+1);
            self.nodes[node_id].right_node = Some(right_child_id);
        }

        self.node_depth_during_last_rebuild =  cmp::max(self.node_depth_during_last_rebuild,depth);

        node_id
    }

    fn gather_points(&self, current_index: usize, points : &mut Vec<KdtreePoint>){
        points.push(self.nodes[current_index].point);
        if let Some(left_index) = self.nodes[current_index].left_node {
            self.gather_points(left_index, points);
        }

        if let Some(right_index) = self.nodes[current_index].right_node {
            self.gather_points(right_index, points);
        }
    }
}

pub struct KdtreeNode<T> {
    left_node: Option<usize>,
    right_node: Option<usize>,

    point: T,
    dimension: usize,
    split_on: f64
}

impl<T: KdtreePointTrait> KdtreeNode<T> {
    fn new(p: T, splitting_dimension: usize, split_on_value: f64) -> KdtreeNode<T> {
        KdtreeNode {
            left_node: None,
            right_node: None,

            point: p,
            dimension: splitting_dimension,
            split_on: split_on_value
        }
    }
}

#[cfg(test)]
mod tests {
    use ::kdtree::test_common::Point2WithId;

    use super::*;

    #[test]
    #[should_panic(expected = "empty vector point not allowed")]
    fn should_panic_given_empty_vector() {
        let mut empty_vec: Vec<Point2WithId> = vec![];

        Kdtree::new(&mut empty_vec);
    }

    quickcheck! {
        fn tree_build_creates_tree_with_as_many_leafs_as_there_is_points(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }
            let mut vec : Vec<Point2WithId> = vec![];
            for i in 0 .. xs.len() {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);

                vec.push(p);
            }

            let tree = Kdtree::new(&mut qc_value_vec_to_2d_points_vec(&xs));

            let mut to_iterate : Vec<usize> = vec![];
            to_iterate.push(0);

            while to_iterate.len() > 0 {
                let last_index = to_iterate.last().unwrap().clone();
                let ref x = tree.nodes.get(last_index).unwrap();
                to_iterate.pop();
                if x.left_node.is_some() {
                    to_iterate.push(x.left_node.unwrap());
                }
                if x.right_node.is_some() {
                    to_iterate.push(x.right_node.unwrap());
                }
            }
            xs.len() == tree.nodes.len()
        }
    }

    quickcheck! {
        fn nearest_neighbor_search_using_qc(xs : Vec<f64>) -> bool {
            if xs.len() == 0 {
                return true;
            }

            let point_vec = qc_value_vec_to_2d_points_vec(&xs);
            let tree = Kdtree::new(&mut point_vec.clone());

            for p in &point_vec {
                let found_nn = tree.nearest_search(p);

                assert_eq!(p.id,found_nn.id);
            }

            true
        }
    }

    #[test]
    fn has_neighbor_in_range() {
        let mut vec: Vec<Point2WithId> = vec![Point2WithId::new(0,2.,0.)];

        let tree = Kdtree::new(&mut vec);

        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 0.));
        assert_eq!(false,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 1.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 2.));
        assert_eq!(true,tree.has_neighbor_in_range(&Point2WithId::new(0,0.,0.), 300.));
    }

    #[test]
    fn incremental_add_adds_as_expected() {
        //this test is tricky because it can have problems with the automatic tree rebuild.

        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        tree.insert_node(Point2WithId::new(0,1.,0.));
        tree.insert_node(Point2WithId::new(0,-1.,0.));

        assert_eq!(tree.nodes.len(), 3);
        assert_eq!(tree.nodes[0].dimension, 0);

        assert_eq!(tree.nodes[0].left_node.is_some(), true);
        assert_eq!(tree.nodes[1].point.dims()[0], 1.);
        assert_eq!(tree.nodes[2].point.dims()[0], -1.);

        assert_eq!(tree.nodes[0].right_node.is_some(), true);
    }

    #[test]
    fn incremental_add_filters_duplicates() {
        let mut vec = vec![Point2WithId::new(0,0.,0.)];

        let mut tree = Kdtree::new(&mut vec);

        let node = Point2WithId::new(0,1.,0.);
        tree.insert_node(node);
        tree.insert_node(node);

        assert_eq!(tree.nodes.len(), 2);
    }

    fn qc_value_vec_to_2d_points_vec(xs: &Vec<f64>) -> Vec<Point2WithId> {
        let mut vec: Vec<Point2WithId> = vec![];
        for i in 0..xs.len() {
            let mut is_duplicated_value = false;
            for j in 0..i {
                if xs[i] == xs[j] {
                    is_duplicated_value = true;
                    break;
                }
            }
            if !is_duplicated_value {
                let p = Point2WithId::new(i as i32, xs[i], xs[i]);
                vec.push(p);
            }
        }

        vec
    }
}<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # The Rust Standard Library
//!
//! The Rust Standard Library provides the essential runtime
//! functionality for building portable Rust software.
//! It is linked to all Rust crates by default.
//!
//! ## Intrinsic types and operations
//!
//! The [`ptr`](ptr/index.html) and [`mem`](mem/index.html)
//! modules deal with unsafe pointers and memory manipulation.
//! [`marker`](marker/index.html) defines the special built-in traits,
//! and [`raw`](raw/index.html) the runtime representation of Rust types.
//! These are some of the lowest-level building blocks in Rust.
//!
//! ## Math on primitive types and math traits
//!
//! Although basic operations on primitive types are implemented
//! directly by the compiler, the standard library additionally
//! defines many common operations through traits defined in
//! mod [`num`](num/index.html).
//!
//! ## Pervasive types
//!
//! The [`option`](option/index.html) and [`result`](result/index.html)
//! modules define optional and error-handling types, `Option` and `Result`.
//! [`iter`](iter/index.html) defines Rust's iterator protocol
//! along with a wide variety of iterators.
//! [`Cell` and `RefCell`](cell/index.html) are for creating types that
//! manage their own mutability.
//!
//! ## Vectors, slices and strings
//!
//! The common container type, `Vec`, a growable vector backed by an array,
//! lives in the [`vec`](vec/index.html) module. Contiguous, unsized regions
//! of memory, `[T]`, commonly called "slices", and their borrowed versions,
//! `&[T]`, commonly called "borrowed slices", are built-in types for which the
//! [`slice`](slice/index.html) module defines many methods.
//!
//! `&str`, a UTF-8 string, is a built-in type, and the standard library
//! defines methods for it on a variety of traits in the
//! [`str`](str/index.html) module. Rust strings are immutable;
//! use the `String` type defined in [`string`](string/index.html)
//! for a mutable string builder.
//!
//! For converting to strings use the [`format!`](fmt/index.html)
//! macro, and for converting from strings use the
//! [`FromStr`](str/trait.FromStr.html) trait.
//!
//! ## Platform abstractions
//!
//! Besides basic data types, the standard library is largely concerned
//! with abstracting over differences in common platforms, most notably
//! Windows and Unix derivatives. The [`os`](os/index.html) module
//! provides a number of basic functions for interacting with the
//! operating environment, including program arguments, environment
//! variables, and directory navigation. The [`path`](path/index.html)
//! module encapsulates the platform-specific rules for dealing
//! with file paths.
//!
//! `std` also includes the [`ffi`](ffi/index.html) module for interoperating
//! with the C language.
//!
//! ## Concurrency, I/O, and the runtime
//!
//! The [`thread`](thread/index.html) module contains Rust's threading abstractions.
//! [`sync`](sync/index.html) contains further, primitive, shared memory types,
//! including [`atomic`](sync/atomic/index.html), and [`mpsc`](sync/mpsc/index.html),
//! which contains the channel types for message passing.
//!
//! Common types of I/O, including files, TCP, UDP, pipes, Unix domain sockets, and
//! process spawning, are defined in the [`io`](io/index.html) module.
//!
//! Rust's I/O and concurrency depends on a small runtime interface
//! that lives, along with its support code, in mod [`rt`](rt/index.html).
//! While a notable part of the standard library's architecture, this
//! module is not intended for public use.
//!
//! ## The Rust prelude and macros
//!
//! Finally, the [`prelude`](prelude/index.html) defines a
//! common set of traits, types, and functions that are made available
//! to all code by default. [`macros`](macros/index.html) contains
//! all the standard macros, such as `assert!`, `panic!`, `println!`,
//! and `format!`, also available to all Rust code.
// Do not remove on snapshot creation. Needed for bootstrap. (Issue #22364)
#![cfg_attr(stage0, feature(custom_attribute))]
#![crate_name = "std"]
#![stable(feature = "rust1", since = "1.0.0")]
#![staged_api]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/nightly/",
       html_playground_url = "http://play.rust-lang.org/")]
#![doc(test(no_crate_inject, attr(deny(warnings))))]
#![doc(test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))))]

#![feature(alloc)]
#![feature(box_syntax)]
#![feature(collections)]
#![feature(core)]
#![feature(lang_items)]
#![feature(libc)]
#![feature(linkage, thread_local, asm)]
#![feature(optin_builtin_traits)]
#![feature(rand)]
#![feature(staged_api)]
#![feature(unboxed_closures)]
#![feature(unicode)]
#![feature(unsafe_no_drop_flag, filling_drop)]
#![feature(macro_reexport)]
#![feature(unique)]
#![feature(allow_internal_unstable)]
#![feature(str_char)]
#![feature(into_cow)]
#![feature(std_misc)]
#![feature(slice_patterns)]
#![feature(debug_builders)]
#![feature(zero_one)]
#![cfg_attr(test, feature(float_from_str_radix))]
#![cfg_attr(test, feature(test, rustc_private, std_misc))]

// Don't link to std. We are std.
#![feature(no_std)]
#![no_std]

#![allow(trivial_casts)]
#![deny(missing_docs)]

#[cfg(test)] extern crate test;
#[cfg(test)] #[macro_use] extern crate log;

#[macro_use]
#[macro_reexport(assert, assert_eq, debug_assert, debug_assert_eq,
    unreachable, unimplemented, write, writeln)]
extern crate core;

#[macro_use]
#[macro_reexport(vec, format)]
extern crate collections as core_collections;

#[allow(deprecated)] extern crate rand as core_rand;
extern crate alloc;
extern crate rustc_unicode;
extern crate libc;

#[macro_use] #[no_link] extern crate rustc_bitflags;

// Make std testable by not duplicating lang items. See #2912
#[cfg(test)] extern crate std as realstd;
#[cfg(test)] pub use realstd::marker;
#[cfg(test)] pub use realstd::ops;
#[cfg(test)] pub use realstd::cmp;
#[cfg(test)] pub use realstd::boxed;

// NB: These reexports are in the order they should be listed in rustdoc

pub use core::any;
pub use core::cell;
pub use core::clone;
#[cfg(not(test))] pub use core::cmp;
pub use core::convert;
pub use core::default;
pub use core::hash;
pub use core::intrinsics;
pub use core::iter;
#[cfg(not(test))] pub use core::marker;
pub use core::mem;
#[cfg(not(test))] pub use core::ops;
pub use core::ptr;
pub use core::raw;
pub use core::simd;
pub use core::result;
pub use core::option;
pub mod error;

#[cfg(not(test))] pub use alloc::boxed;
pub use alloc::rc;

pub use core_collections::borrow;
pub use core_collections::fmt;
pub use core_collections::slice;
pub use core_collections::str;
pub use core_collections::string;
#[stable(feature = "rust1", since = "1.0.0")]
pub use core_collections::vec;

pub use rustc_unicode::char;

/* Exported macros */

#[macro_use]
mod macros;

mod rtdeps;

/* The Prelude. */

pub mod prelude;

/* Primitive types */

// NB: slice and str are primitive types too, but their module docs + primitive doc pages
// are inlined from the public re-exports of core_collections::{slice, str} above.

#[path = "num/float_macros.rs"]
#[macro_use]
mod float_macros;

#[path = "num/int_macros.rs"]
#[macro_use]
mod int_macros;

#[path = "num/uint_macros.rs"]
#[macro_use]
mod uint_macros;

#[path = "num/isize.rs"]  pub mod isize;
#[path = "num/i8.rs"]   pub mod i8;
#[path = "num/i16.rs"]  pub mod i16;
#[path = "num/i32.rs"]  pub mod i32;
#[path = "num/i64.rs"]  pub mod i64;

#[path = "num/usize.rs"] pub mod usize;
#[path = "num/u8.rs"]   pub mod u8;
#[path = "num/u16.rs"]  pub mod u16;
#[path = "num/u32.rs"]  pub mod u32;
#[path = "num/u64.rs"]  pub mod u64;

#[path = "num/f32.rs"]   pub mod f32;
#[path = "num/f64.rs"]   pub mod f64;

pub mod ascii;

pub mod thunk;

/* Common traits */

pub mod num;

/* Runtime and platform support */

#[macro_use]
pub mod thread;

pub mod collections;
pub mod dynamic_lib;
pub mod env;
pub mod ffi;
pub mod fs;
pub mod io;
pub mod net;
pub mod os;
pub mod path;
pub mod process;
pub mod sync;
pub mod time;

#[macro_use]
#[path = "sys/common/mod.rs"] mod sys_common;

#[cfg(unix)]
#[path = "sys/unix/mod.rs"] mod sys;
#[cfg(windows)]
#[path = "sys/windows/mod.rs"] mod sys;

pub mod rt;
mod panicking;
mod rand;

// Some external utilities of the standard library rely on randomness (aka
// rustc_back::TempDir and tests) and need a way to get at the OS rng we've got
// here. This module is not at all intended for stabilization as-is, however,
// but it may be stabilized long-term. As a result we're exposing a hidden,
// unstable module so we can get our build working.
#[doc(hidden)]
#[unstable(feature = "rand")]
pub mod __rand {
    pub use rand::{thread_rng, ThreadRng, Rng};
}

// Modules that exist purely to document + host impl docs for primitive types

mod array;
mod bool;
mod unit;
mod tuple;

// A curious inner-module that's not exported that contains the binding
// 'std' so that macro-expanded references to std::error and such
// can be resolved within libstd.
#[doc(hidden)]
mod std {
    pub use sync; // used for select!()
    pub use error; // used for try!()
    pub use fmt; // used for any formatting strings
    pub use option; // used for bitflags!{}
    pub use rt; // used for panic!()
    pub use vec; // used for vec![]
    pub use cell; // used for tls!
    pub use thread; // used for thread_local!
    pub use marker;  // used for tls!
    pub use ops; // used for bitflags!

    // The test runner calls ::std::env::args() but really wants realstd
    #[cfg(test)] pub use realstd::env as env;
    // The test runner requires std::slice::Vector, so re-export std::slice just for it.
    //
    // It is also used in vec![]
    pub use slice;

    pub use boxed; // used for vec![]
}
<commit_msg>`bitflags!` is no longer used in `std`<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! # The Rust Standard Library
//!
//! The Rust Standard Library provides the essential runtime
//! functionality for building portable Rust software.
//! It is linked to all Rust crates by default.
//!
//! ## Intrinsic types and operations
//!
//! The [`ptr`](ptr/index.html) and [`mem`](mem/index.html)
//! modules deal with unsafe pointers and memory manipulation.
//! [`marker`](marker/index.html) defines the special built-in traits,
//! and [`raw`](raw/index.html) the runtime representation of Rust types.
//! These are some of the lowest-level building blocks in Rust.
//!
//! ## Math on primitive types and math traits
//!
//! Although basic operations on primitive types are implemented
//! directly by the compiler, the standard library additionally
//! defines many common operations through traits defined in
//! mod [`num`](num/index.html).
//!
//! ## Pervasive types
//!
//! The [`option`](option/index.html) and [`result`](result/index.html)
//! modules define optional and error-handling types, `Option` and `Result`.
//! [`iter`](iter/index.html) defines Rust's iterator protocol
//! along with a wide variety of iterators.
//! [`Cell` and `RefCell`](cell/index.html) are for creating types that
//! manage their own mutability.
//!
//! ## Vectors, slices and strings
//!
//! The common container type, `Vec`, a growable vector backed by an array,
//! lives in the [`vec`](vec/index.html) module. Contiguous, unsized regions
//! of memory, `[T]`, commonly called "slices", and their borrowed versions,
//! `&[T]`, commonly called "borrowed slices", are built-in types for which the
//! [`slice`](slice/index.html) module defines many methods.
//!
//! `&str`, a UTF-8 string, is a built-in type, and the standard library
//! defines methods for it on a variety of traits in the
//! [`str`](str/index.html) module. Rust strings are immutable;
//! use the `String` type defined in [`string`](string/index.html)
//! for a mutable string builder.
//!
//! For converting to strings use the [`format!`](fmt/index.html)
//! macro, and for converting from strings use the
//! [`FromStr`](str/trait.FromStr.html) trait.
//!
//! ## Platform abstractions
//!
//! Besides basic data types, the standard library is largely concerned
//! with abstracting over differences in common platforms, most notably
//! Windows and Unix derivatives. The [`os`](os/index.html) module
//! provides a number of basic functions for interacting with the
//! operating environment, including program arguments, environment
//! variables, and directory navigation. The [`path`](path/index.html)
//! module encapsulates the platform-specific rules for dealing
//! with file paths.
//!
//! `std` also includes the [`ffi`](ffi/index.html) module for interoperating
//! with the C language.
//!
//! ## Concurrency, I/O, and the runtime
//!
//! The [`thread`](thread/index.html) module contains Rust's threading abstractions.
//! [`sync`](sync/index.html) contains further, primitive, shared memory types,
//! including [`atomic`](sync/atomic/index.html), and [`mpsc`](sync/mpsc/index.html),
//! which contains the channel types for message passing.
//!
//! Common types of I/O, including files, TCP, UDP, pipes, Unix domain sockets, and
//! process spawning, are defined in the [`io`](io/index.html) module.
//!
//! Rust's I/O and concurrency depends on a small runtime interface
//! that lives, along with its support code, in mod [`rt`](rt/index.html).
//! While a notable part of the standard library's architecture, this
//! module is not intended for public use.
//!
//! ## The Rust prelude and macros
//!
//! Finally, the [`prelude`](prelude/index.html) defines a
//! common set of traits, types, and functions that are made available
//! to all code by default. [`macros`](macros/index.html) contains
//! all the standard macros, such as `assert!`, `panic!`, `println!`,
//! and `format!`, also available to all Rust code.
// Do not remove on snapshot creation. Needed for bootstrap. (Issue #22364)
#![cfg_attr(stage0, feature(custom_attribute))]
#![crate_name = "std"]
#![stable(feature = "rust1", since = "1.0.0")]
#![staged_api]
#![crate_type = "rlib"]
#![crate_type = "dylib"]
#![doc(html_logo_url = "http://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "http://www.rust-lang.org/favicon.ico",
       html_root_url = "http://doc.rust-lang.org/nightly/",
       html_playground_url = "http://play.rust-lang.org/")]
#![doc(test(no_crate_inject, attr(deny(warnings))))]
#![doc(test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))))]

#![feature(alloc)]
#![feature(box_syntax)]
#![feature(collections)]
#![feature(core)]
#![feature(lang_items)]
#![feature(libc)]
#![feature(linkage, thread_local, asm)]
#![feature(optin_builtin_traits)]
#![feature(rand)]
#![feature(staged_api)]
#![feature(unboxed_closures)]
#![feature(unicode)]
#![feature(unsafe_no_drop_flag, filling_drop)]
#![feature(macro_reexport)]
#![feature(unique)]
#![feature(allow_internal_unstable)]
#![feature(str_char)]
#![feature(into_cow)]
#![feature(std_misc)]
#![feature(slice_patterns)]
#![feature(debug_builders)]
#![feature(zero_one)]
#![cfg_attr(test, feature(float_from_str_radix))]
#![cfg_attr(test, feature(test, rustc_private, std_misc))]

// Don't link to std. We are std.
#![feature(no_std)]
#![no_std]

#![allow(trivial_casts)]
#![deny(missing_docs)]

#[cfg(test)] extern crate test;
#[cfg(test)] #[macro_use] extern crate log;

#[macro_use]
#[macro_reexport(assert, assert_eq, debug_assert, debug_assert_eq,
    unreachable, unimplemented, write, writeln)]
extern crate core;

#[macro_use]
#[macro_reexport(vec, format)]
extern crate collections as core_collections;

#[allow(deprecated)] extern crate rand as core_rand;
extern crate alloc;
extern crate rustc_unicode;
extern crate libc;

#[macro_use] #[no_link] extern crate rustc_bitflags;

// Make std testable by not duplicating lang items. See #2912
#[cfg(test)] extern crate std as realstd;
#[cfg(test)] pub use realstd::marker;
#[cfg(test)] pub use realstd::ops;
#[cfg(test)] pub use realstd::cmp;
#[cfg(test)] pub use realstd::boxed;

// NB: These reexports are in the order they should be listed in rustdoc

pub use core::any;
pub use core::cell;
pub use core::clone;
#[cfg(not(test))] pub use core::cmp;
pub use core::convert;
pub use core::default;
pub use core::hash;
pub use core::intrinsics;
pub use core::iter;
#[cfg(not(test))] pub use core::marker;
pub use core::mem;
#[cfg(not(test))] pub use core::ops;
pub use core::ptr;
pub use core::raw;
pub use core::simd;
pub use core::result;
pub use core::option;
pub mod error;

#[cfg(not(test))] pub use alloc::boxed;
pub use alloc::rc;

pub use core_collections::borrow;
pub use core_collections::fmt;
pub use core_collections::slice;
pub use core_collections::str;
pub use core_collections::string;
#[stable(feature = "rust1", since = "1.0.0")]
pub use core_collections::vec;

pub use rustc_unicode::char;

/* Exported macros */

#[macro_use]
mod macros;

mod rtdeps;

/* The Prelude. */

pub mod prelude;

/* Primitive types */

// NB: slice and str are primitive types too, but their module docs + primitive doc pages
// are inlined from the public re-exports of core_collections::{slice, str} above.

#[path = "num/float_macros.rs"]
#[macro_use]
mod float_macros;

#[path = "num/int_macros.rs"]
#[macro_use]
mod int_macros;

#[path = "num/uint_macros.rs"]
#[macro_use]
mod uint_macros;

#[path = "num/isize.rs"]  pub mod isize;
#[path = "num/i8.rs"]   pub mod i8;
#[path = "num/i16.rs"]  pub mod i16;
#[path = "num/i32.rs"]  pub mod i32;
#[path = "num/i64.rs"]  pub mod i64;

#[path = "num/usize.rs"] pub mod usize;
#[path = "num/u8.rs"]   pub mod u8;
#[path = "num/u16.rs"]  pub mod u16;
#[path = "num/u32.rs"]  pub mod u32;
#[path = "num/u64.rs"]  pub mod u64;

#[path = "num/f32.rs"]   pub mod f32;
#[path = "num/f64.rs"]   pub mod f64;

pub mod ascii;

pub mod thunk;

/* Common traits */

pub mod num;

/* Runtime and platform support */

#[macro_use]
pub mod thread;

pub mod collections;
pub mod dynamic_lib;
pub mod env;
pub mod ffi;
pub mod fs;
pub mod io;
pub mod net;
pub mod os;
pub mod path;
pub mod process;
pub mod sync;
pub mod time;

#[macro_use]
#[path = "sys/common/mod.rs"] mod sys_common;

#[cfg(unix)]
#[path = "sys/unix/mod.rs"] mod sys;
#[cfg(windows)]
#[path = "sys/windows/mod.rs"] mod sys;

pub mod rt;
mod panicking;
mod rand;

// Some external utilities of the standard library rely on randomness (aka
// rustc_back::TempDir and tests) and need a way to get at the OS rng we've got
// here. This module is not at all intended for stabilization as-is, however,
// but it may be stabilized long-term. As a result we're exposing a hidden,
// unstable module so we can get our build working.
#[doc(hidden)]
#[unstable(feature = "rand")]
pub mod __rand {
    pub use rand::{thread_rng, ThreadRng, Rng};
}

// Modules that exist purely to document + host impl docs for primitive types

mod array;
mod bool;
mod unit;
mod tuple;

// A curious inner-module that's not exported that contains the binding
// 'std' so that macro-expanded references to std::error and such
// can be resolved within libstd.
#[doc(hidden)]
mod std {
    pub use sync; // used for select!()
    pub use error; // used for try!()
    pub use fmt; // used for any formatting strings
    pub use option; // used for thread_local!{}
    pub use rt; // used for panic!()
    pub use vec; // used for vec![]
    pub use cell; // used for tls!
    pub use thread; // used for thread_local!
    pub use marker;  // used for tls!

    // The test runner calls ::std::env::args() but really wants realstd
    #[cfg(test)] pub use realstd::env as env;
    // The test runner requires std::slice::Vector, so re-export std::slice just for it.
    //
    // It is also used in vec![]
    pub use slice;

    pub use boxed; // used for vec![]
}
<|endoftext|>


<commit_before>// A machine-local concurrent LSM tree based on B-tree implementation

use super::btree::level::*;
use super::btree::*;
use crate::client::AsyncClient;
use crate::ram::cell::Cell;
use crate::ram::schema::{Field, Schema};
use crate::ram::types::*;
use crossbeam_epoch::*;
use lightning::map::HashSet;
use std::mem;
use std::sync::atomic::Ordering::{Acquire, Release};
use std::sync::Arc;

pub const LSM_TREE_SCHEMA_NAME: &'static str = "NEB_LSM_TREE";
pub const LSM_TREE_LEVELS_NAME: &'static str = "levels";
pub const LSM_TREE_MIGRATION_NAME: &'static str = "migration";
pub const LAST_LEVEL_MULT_FACTOR: usize = 16;

type LevelTrees = [Box<dyn LevelTree>; NUM_LEVELS];
type LevelCusors = [Box<dyn Cursor>; NUM_LEVELS + 2]; // 2 for mem and trans mem
pub type DeletionSet = HashSet<EntryKey>;

lazy_static! {
    pub static ref LSM_TREE_SCHEMA_ID: u32 = key_hash(LSM_TREE_SCHEMA_NAME) as u32;
    pub static ref LSM_TREE_LEVELS_HASH: u64 = key_hash(LSM_TREE_LEVELS_NAME);
    pub static ref LSM_TREE_MIGRATION_HASH: u64 = key_hash(LSM_TREE_MIGRATION_NAME);
    pub static ref LSM_TREE_SCHEMA: Schema = lsm_treee_schema();
}

pub struct LSMTree {
    pub mem_tree: Atomic<Box<dyn LevelTree>>,
    pub trans_mem_tree: Atomic<Box<dyn LevelTree>>,
    pub disk_trees: LevelTrees,
    pub deletion: Arc<DeletionSet>,
}

impl LSMTree {
    pub async fn create(neb_client: &Arc<AsyncClient>, id: &Id) -> Self {
        let deletion_ref = Arc::new(HashSet::with_capacity(16));
        let tree_m = LevelMTree::new(&deletion_ref);
        let tree_0 = Level0Tree::new(&deletion_ref);
        let tree_1 = Level1Tree::new(&deletion_ref);
        let tree_2 = Level2Tree::new(&deletion_ref);
        tree_0.persist_root(neb_client).await;
        tree_1.persist_root(neb_client).await;
        tree_2.persist_root(neb_client).await;
        let level_ids = vec![tree_m.head_id(), tree_1.head_id(), tree_2.head_id()];
        let lsm_tree_cell = lsm_tree_cell(&level_ids, id, None);
        neb_client.write_cell(lsm_tree_cell).await.unwrap().unwrap();
        Self {
            mem_tree: Atomic::new(box tree_m),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1, box tree_2],
            deletion: deletion_ref,
        }
    }

    pub async fn recover(neb_client: &Arc<AsyncClient>, lsm_tree_id: &Id) -> Self {
        let deletion_ref = Arc::new(HashSet::with_capacity(16));
        let cell = neb_client.read_cell(*lsm_tree_id).await.unwrap().unwrap();
        let trees = &cell.data[*LSM_TREE_LEVELS_HASH];
        let trees_0_val = &trees[0usize];
        let trees_1_val = &trees[1usize];
        let trees_2_val = &trees[2usize];

        let tree_0 =
            Level0Tree::from_head_id(trees_0_val.Id().unwrap(), neb_client, &deletion_ref).await;
        let tree_1 =
            Level1Tree::from_head_id(trees_1_val.Id().unwrap(), neb_client, &deletion_ref).await;
        let tree_2 =
            Level2Tree::from_head_id(trees_2_val.Id().unwrap(), neb_client, &deletion_ref).await;

        Self {
            mem_tree: Atomic::new(box LevelMTree::new(&deletion_ref)),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1, box tree_2],
            deletion: deletion_ref,
        }
    }

    pub fn insert(&self, entry: &EntryKey) -> bool {
        let guard = crossbeam_epoch::pin();
        let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        mem_tree.insert_into(entry)
    }

    pub fn delete(&self, entry: &EntryKey) -> bool {
        if let Some(k) = self.seek(entry, Ordering::Forward).current() {
            if k == entry {
                self.deletion.insert(entry);
                return true;
            }
        }
        return false;
    }

    pub fn seek(&self, entry: &EntryKey, ordering: Ordering) -> LSMTreeCursor {
        LSMTreeCursor::new(entry, self, ordering)
    }

    pub async fn merge_levels(&self) -> bool {
        let mut merged = false;
        {
            let guard = crossbeam_epoch::pin();
            let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
            let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
            if mem_tree.oversized() {
                info!("Memory tree oversized, moving everything to disk tree");
                // Memory tree oversized
                // Will construct a new tree and swap the old one to trans_mem_tree for query and move all
                // keys in old tree to disk tree level 0
                let new_mem_tree: Box<dyn LevelTree> = box LevelMTree::new(&self.deletion);
                let new_mem_tree_ptr = Owned::new(new_mem_tree).into_shared(&guard);
                self.trans_mem_tree.store(mem_tree_ptr, Release);
                self.mem_tree.store(new_mem_tree_ptr, Release);
                info!("Starting moving memory tree...");
                mem_tree.merge_all_to(0, &*self.disk_trees[0], false);
                self.trans_mem_tree.store(Shared::null(), Acquire);
                unsafe {
                    guard.defer_destroy(mem_tree_ptr);
                }
                merged = true;
                info!("Memory tree merging completed");
            } else {
                trace!("Memory tree not oversized");
            }
        }
        storage::wait_until_updated().await;
        for i in 0..self.disk_trees.len() - 1 {
            let level = i + 1;
            if self.disk_trees[i].oversized() {
                info!("Level {} tree oversized, merging", level);
                self.disk_trees[i]
                    .merge_to(level, &*self.disk_trees[i + 1], true);
                storage::wait_until_updated().await;
                info!("Level {} merge completed", level);
                merged = true;
            } else {
                trace!("Level {} tree not oversized", level);
            }
        }
        merged
    }

    pub fn oversized(&self) -> bool {
        self.last_level_tree().count() > self.ideal_capacity()
    }

    pub fn mid_key(&self) -> Option<EntryKey> {
        self.last_level_tree().mid_key()
    }

    pub async fn mark_migration(&self, id: &Id, migration: Option<Id>, client: &Arc<AsyncClient>) {
        let lsm_tree_cell = lsm_tree_cell(
            &self.disk_trees.iter().map(|tree| tree.head_id()).collect(),
            id,
            migration,
        );
        client.update_cell(lsm_tree_cell).await.unwrap().unwrap();
    }

    pub fn merge_keys(&self, keys: Box<Vec<EntryKey>>) {
        self.last_level_tree().merge_with_keys(keys);
    }

    pub fn ideal_capacity(&self) -> usize {
        self.last_level_tree().ideal_capacity() * LAST_LEVEL_MULT_FACTOR
    }

    fn last_level_tree(&self) -> &Box<dyn LevelTree> {
        self.disk_trees.last().unwrap()
    }
}

pub struct LSMTreeCursor {
    cursors: LevelCusors,
    current: Option<(usize, EntryKey)>,
    ordering: Ordering,
    deletion: Arc<DeletionSet>,
}

impl LSMTreeCursor {
    fn new(key: &EntryKey, lsm_tree: &LSMTree, ordering: Ordering) -> Self {
        let guard = crossbeam_epoch::pin();
        let disk_trees = &lsm_tree.disk_trees;
        let mem_tree_ptr = lsm_tree.mem_tree.load(Acquire, &guard);
        let trans_mem_tree_ptr = lsm_tree.trans_mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        let mut cursors = [
            mem_tree.seek_for(key, ordering),
            disk_trees[0].seek_for(key, ordering),
            disk_trees[1].seek_for(key, ordering),
            disk_trees[2].seek_for(key, ordering),
            box DummyCursor // for trans mem
        ];
        if !trans_mem_tree_ptr.is_null() && trans_mem_tree_ptr != mem_tree_ptr {
            let trans_mem_tree = unsafe { trans_mem_tree_ptr.as_ref().unwrap() };
            *cursors.last_mut().unwrap() = trans_mem_tree.seek_for(key, ordering);
        }
        let current = Self::leading_tree_key(&cursors, ordering);
        let deletion = lsm_tree.deletion.clone();
        Self {
            cursors,
            current,
            ordering,
            deletion,
        }
    }

    fn leading_tree_key(cursors: &LevelCusors, ordering: Ordering) -> Option<(usize, EntryKey)> {
        match ordering {
            Ordering::Forward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .min_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
            Ordering::Backward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .max_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn current(&self) -> Option<&EntryKey> {
        self.current.as_ref().map(|(_, k)| k)
    }
    fn next(&mut self) -> Option<EntryKey> {
        loop {
            if let Some((tree_id, _)) = self.current {
                self.cursors[tree_id].next();
                let mut other_entry = Self::leading_tree_key(&self.cursors, self.ordering);
                mem::swap(&mut other_entry, &mut self.current);
                let res = other_entry.map(|(_, key)| key);
                if let Some(k) = &res {
                    if self.deletion.contains(k) {
                        // Skip keys in deletion set
                        continue;
                    }
                }
                return res;
            } else {
                return None;
            }
        }
    }
}

fn lsm_treee_schema() -> Schema {
    Schema {
        id: *LSM_TREE_SCHEMA_ID,
        name: String::from(LSM_TREE_SCHEMA_NAME),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*",
            0,
            false,
            false,
            Some(vec![
                Field::new(
                    LSM_TREE_LEVELS_NAME,
                    type_id_of(Type::Id),
                    false,
                    true,
                    None,
                    vec![],
                ),
                Field::new(
                    LSM_TREE_MIGRATION_NAME,
                    type_id_of(Type::Id),
                    true,
                    false,
                    None,
                    vec![],
                ),
            ]),
            vec![],
        ),
    }
}

fn lsm_tree_cell(level_ids: &Vec<Id>, id: &Id, migration: Option<Id>) -> Cell {
    let mut cell_map = Map::new();
    cell_map.insert_key_id(
        *LSM_TREE_LEVELS_HASH,
        Value::Array(level_ids.iter().map(|id| id.value()).collect()),
    );
    cell_map.insert_key_id(
        *LSM_TREE_MIGRATION_HASH,
        migration.map(|id| Value::Id(id)).unwrap_or(Value::Null),
    );
    Cell::new_with_id(*LSM_TREE_SCHEMA_ID, id, Value::Map(cell_map))
}

impl_btree_level!(LEVEL_M);
type LevelMTreeKeySlice = [EntryKey; LEVEL_M];
type LevelMTreePtrSlice = [NodeCellRef; LEVEL_M + 1];
type LevelMTree = BPlusTree<LevelMTreeKeySlice, LevelMTreePtrSlice>;

impl_btree_level!(LEVEL_0);
type Level0TreeKeySlice = [EntryKey; LEVEL_0];
type Level0TreePtrSlice = [NodeCellRef; LEVEL_0 + 1];
type Level0Tree = BPlusTree<Level0TreeKeySlice, Level0TreePtrSlice>;

impl_btree_level!(LEVEL_1);
type Level1TreeKeySlice = [EntryKey; LEVEL_1];
type Level1TreePtrSlice = [NodeCellRef; LEVEL_1 + 1];
type Level1Tree = BPlusTree<Level1TreeKeySlice, Level1TreePtrSlice>;

impl_btree_level!(LEVEL_2);
type Level2TreeKeySlice = [EntryKey; LEVEL_2];
type Level2TreePtrSlice = [NodeCellRef; LEVEL_2 + 1];
type Level2Tree = BPlusTree<Level2TreeKeySlice, Level2TreePtrSlice>;

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

struct DummyCursor;

impl Cursor for DummyCursor {
    fn next(&mut self) -> Option<EntryKey> { None }
    fn current(&self) -> Option<&EntryKey> { None }
}<commit_msg>fixed memory ordering<commit_after>// A machine-local concurrent LSM tree based on B-tree implementation

use super::btree::level::*;
use super::btree::*;
use crate::client::AsyncClient;
use crate::ram::cell::Cell;
use crate::ram::schema::{Field, Schema};
use crate::ram::types::*;
use crossbeam_epoch::*;
use lightning::map::HashSet;
use std::mem;
use std::sync::atomic::Ordering::{Acquire, Release};
use std::sync::Arc;

pub const LSM_TREE_SCHEMA_NAME: &'static str = "NEB_LSM_TREE";
pub const LSM_TREE_LEVELS_NAME: &'static str = "levels";
pub const LSM_TREE_MIGRATION_NAME: &'static str = "migration";
pub const LAST_LEVEL_MULT_FACTOR: usize = 16;

type LevelTrees = [Box<dyn LevelTree>; NUM_LEVELS];
type LevelCusors = [Box<dyn Cursor>; NUM_LEVELS + 2]; // 2 for mem and trans mem
pub type DeletionSet = HashSet<EntryKey>;

lazy_static! {
    pub static ref LSM_TREE_SCHEMA_ID: u32 = key_hash(LSM_TREE_SCHEMA_NAME) as u32;
    pub static ref LSM_TREE_LEVELS_HASH: u64 = key_hash(LSM_TREE_LEVELS_NAME);
    pub static ref LSM_TREE_MIGRATION_HASH: u64 = key_hash(LSM_TREE_MIGRATION_NAME);
    pub static ref LSM_TREE_SCHEMA: Schema = lsm_treee_schema();
}

pub struct LSMTree {
    pub mem_tree: Atomic<Box<dyn LevelTree>>,
    pub trans_mem_tree: Atomic<Box<dyn LevelTree>>,
    pub disk_trees: LevelTrees,
    pub deletion: Arc<DeletionSet>,
}

impl LSMTree {
    pub async fn create(neb_client: &Arc<AsyncClient>, id: &Id) -> Self {
        let deletion_ref = Arc::new(HashSet::with_capacity(16));
        let tree_m = LevelMTree::new(&deletion_ref);
        let tree_0 = Level0Tree::new(&deletion_ref);
        let tree_1 = Level1Tree::new(&deletion_ref);
        let tree_2 = Level2Tree::new(&deletion_ref);
        tree_0.persist_root(neb_client).await;
        tree_1.persist_root(neb_client).await;
        tree_2.persist_root(neb_client).await;
        let level_ids = vec![tree_m.head_id(), tree_1.head_id(), tree_2.head_id()];
        let lsm_tree_cell = lsm_tree_cell(&level_ids, id, None);
        neb_client.write_cell(lsm_tree_cell).await.unwrap().unwrap();
        Self {
            mem_tree: Atomic::new(box tree_m),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1, box tree_2],
            deletion: deletion_ref,
        }
    }

    pub async fn recover(neb_client: &Arc<AsyncClient>, lsm_tree_id: &Id) -> Self {
        let deletion_ref = Arc::new(HashSet::with_capacity(16));
        let cell = neb_client.read_cell(*lsm_tree_id).await.unwrap().unwrap();
        let trees = &cell.data[*LSM_TREE_LEVELS_HASH];
        let trees_0_val = &trees[0usize];
        let trees_1_val = &trees[1usize];
        let trees_2_val = &trees[2usize];

        let tree_0 =
            Level0Tree::from_head_id(trees_0_val.Id().unwrap(), neb_client, &deletion_ref).await;
        let tree_1 =
            Level1Tree::from_head_id(trees_1_val.Id().unwrap(), neb_client, &deletion_ref).await;
        let tree_2 =
            Level2Tree::from_head_id(trees_2_val.Id().unwrap(), neb_client, &deletion_ref).await;

        Self {
            mem_tree: Atomic::new(box LevelMTree::new(&deletion_ref)),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1, box tree_2],
            deletion: deletion_ref,
        }
    }

    pub fn insert(&self, entry: &EntryKey) -> bool {
        let guard = crossbeam_epoch::pin();
        let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        mem_tree.insert_into(entry)
    }

    pub fn delete(&self, entry: &EntryKey) -> bool {
        if let Some(k) = self.seek(entry, Ordering::Forward).current() {
            if k == entry {
                self.deletion.insert(entry);
                return true;
            }
        }
        return false;
    }

    pub fn seek(&self, entry: &EntryKey, ordering: Ordering) -> LSMTreeCursor {
        LSMTreeCursor::new(entry, self, ordering)
    }

    pub async fn merge_levels(&self) -> bool {
        let mut merged = false;
        {
            let guard = crossbeam_epoch::pin();
            let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
            let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
            if mem_tree.oversized() {
                info!("Memory tree oversized, moving everything to disk tree");
                // Memory tree oversized
                // Will construct a new tree and swap the old one to trans_mem_tree for query and move all
                // keys in old tree to disk tree level 0
                let new_mem_tree: Box<dyn LevelTree> = box LevelMTree::new(&self.deletion);
                let new_mem_tree_ptr = Owned::new(new_mem_tree).into_shared(&guard);
                self.trans_mem_tree.store(mem_tree_ptr, Release);
                self.mem_tree.store(new_mem_tree_ptr, Release);
                info!("Starting moving memory tree...");
                mem_tree.merge_all_to(0, &*self.disk_trees[0], false);
                info!("Memory tree merge completed, reset trans tree and destory old tree");
                self.trans_mem_tree.store(Shared::null(), Release);
                unsafe {
                    guard.defer_destroy(mem_tree_ptr);
                }
                merged = true;
                info!("Memory tree merging completed");
            } else {
                trace!("Memory tree not oversized");
            }
        }
        storage::wait_until_updated().await;
        for i in 0..self.disk_trees.len() - 1 {
            let level = i + 1;
            if self.disk_trees[i].oversized() {
                info!("Level {} tree oversized, merging", level);
                self.disk_trees[i]
                    .merge_to(level, &*self.disk_trees[i + 1], true);
                storage::wait_until_updated().await;
                info!("Level {} merge completed", level);
                merged = true;
            } else {
                trace!("Level {} tree not oversized", level);
            }
        }
        merged
    }

    pub fn oversized(&self) -> bool {
        self.last_level_tree().count() > self.ideal_capacity()
    }

    pub fn mid_key(&self) -> Option<EntryKey> {
        self.last_level_tree().mid_key()
    }

    pub async fn mark_migration(&self, id: &Id, migration: Option<Id>, client: &Arc<AsyncClient>) {
        let lsm_tree_cell = lsm_tree_cell(
            &self.disk_trees.iter().map(|tree| tree.head_id()).collect(),
            id,
            migration,
        );
        client.update_cell(lsm_tree_cell).await.unwrap().unwrap();
    }

    pub fn merge_keys(&self, keys: Box<Vec<EntryKey>>) {
        self.last_level_tree().merge_with_keys(keys);
    }

    pub fn ideal_capacity(&self) -> usize {
        self.last_level_tree().ideal_capacity() * LAST_LEVEL_MULT_FACTOR
    }

    fn last_level_tree(&self) -> &Box<dyn LevelTree> {
        self.disk_trees.last().unwrap()
    }
}

pub struct LSMTreeCursor {
    cursors: LevelCusors,
    current: Option<(usize, EntryKey)>,
    ordering: Ordering,
    deletion: Arc<DeletionSet>,
}

impl LSMTreeCursor {
    fn new(key: &EntryKey, lsm_tree: &LSMTree, ordering: Ordering) -> Self {
        let guard = crossbeam_epoch::pin();
        let disk_trees = &lsm_tree.disk_trees;
        let mem_tree_ptr = lsm_tree.mem_tree.load(Acquire, &guard);
        let trans_mem_tree_ptr = lsm_tree.trans_mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        let mut cursors = [
            mem_tree.seek_for(key, ordering),
            disk_trees[0].seek_for(key, ordering),
            disk_trees[1].seek_for(key, ordering),
            disk_trees[2].seek_for(key, ordering),
            box DummyCursor // for trans mem
        ];
        if !trans_mem_tree_ptr.is_null() && trans_mem_tree_ptr != mem_tree_ptr {
            let trans_mem_tree = unsafe { trans_mem_tree_ptr.as_ref().unwrap() };
            *cursors.last_mut().unwrap() = trans_mem_tree.seek_for(key, ordering);
        }
        let current = Self::leading_tree_key(&cursors, ordering);
        let deletion = lsm_tree.deletion.clone();
        Self {
            cursors,
            current,
            ordering,
            deletion,
        }
    }

    fn leading_tree_key(cursors: &LevelCusors, ordering: Ordering) -> Option<(usize, EntryKey)> {
        match ordering {
            Ordering::Forward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .min_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
            Ordering::Backward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .max_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn current(&self) -> Option<&EntryKey> {
        self.current.as_ref().map(|(_, k)| k)
    }
    fn next(&mut self) -> Option<EntryKey> {
        loop {
            if let Some((tree_id, _)) = self.current {
                self.cursors[tree_id].next();
                let mut other_entry = Self::leading_tree_key(&self.cursors, self.ordering);
                mem::swap(&mut other_entry, &mut self.current);
                let res = other_entry.map(|(_, key)| key);
                if let Some(k) = &res {
                    if self.deletion.contains(k) {
                        // Skip keys in deletion set
                        continue;
                    }
                }
                return res;
            } else {
                return None;
            }
        }
    }
}

fn lsm_treee_schema() -> Schema {
    Schema {
        id: *LSM_TREE_SCHEMA_ID,
        name: String::from(LSM_TREE_SCHEMA_NAME),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*",
            0,
            false,
            false,
            Some(vec![
                Field::new(
                    LSM_TREE_LEVELS_NAME,
                    type_id_of(Type::Id),
                    false,
                    true,
                    None,
                    vec![],
                ),
                Field::new(
                    LSM_TREE_MIGRATION_NAME,
                    type_id_of(Type::Id),
                    true,
                    false,
                    None,
                    vec![],
                ),
            ]),
            vec![],
        ),
    }
}

fn lsm_tree_cell(level_ids: &Vec<Id>, id: &Id, migration: Option<Id>) -> Cell {
    let mut cell_map = Map::new();
    cell_map.insert_key_id(
        *LSM_TREE_LEVELS_HASH,
        Value::Array(level_ids.iter().map(|id| id.value()).collect()),
    );
    cell_map.insert_key_id(
        *LSM_TREE_MIGRATION_HASH,
        migration.map(|id| Value::Id(id)).unwrap_or(Value::Null),
    );
    Cell::new_with_id(*LSM_TREE_SCHEMA_ID, id, Value::Map(cell_map))
}

impl_btree_level!(LEVEL_M);
type LevelMTreeKeySlice = [EntryKey; LEVEL_M];
type LevelMTreePtrSlice = [NodeCellRef; LEVEL_M + 1];
type LevelMTree = BPlusTree<LevelMTreeKeySlice, LevelMTreePtrSlice>;

impl_btree_level!(LEVEL_0);
type Level0TreeKeySlice = [EntryKey; LEVEL_0];
type Level0TreePtrSlice = [NodeCellRef; LEVEL_0 + 1];
type Level0Tree = BPlusTree<Level0TreeKeySlice, Level0TreePtrSlice>;

impl_btree_level!(LEVEL_1);
type Level1TreeKeySlice = [EntryKey; LEVEL_1];
type Level1TreePtrSlice = [NodeCellRef; LEVEL_1 + 1];
type Level1Tree = BPlusTree<Level1TreeKeySlice, Level1TreePtrSlice>;

impl_btree_level!(LEVEL_2);
type Level2TreeKeySlice = [EntryKey; LEVEL_2];
type Level2TreePtrSlice = [NodeCellRef; LEVEL_2 + 1];
type Level2Tree = BPlusTree<Level2TreeKeySlice, Level2TreePtrSlice>;

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

struct DummyCursor;

impl Cursor for DummyCursor {
    fn next(&mut self) -> Option<EntryKey> { None }
    fn current(&self) -> Option<&EntryKey> { None }
}<|endoftext|>