<commit_before>#[macro_use]
extern crate difference;
extern crate cobalt;
extern crate tempdir;
extern crate walkdir;

use std::path::Path;
use std::fs::{self, File};
use std::io::Read;
use tempdir::TempDir;
use walkdir::WalkDir;
use std::error::Error;
use cobalt::Config;

macro_rules! assert_contains {
    ($haystack: expr, $needle: expr) => {
        let text = $haystack.to_owned();
        println!("{}", text);
        assert!(text.contains($needle))
    }
}

fn run_test(name: &str) -> Result<(), cobalt::Error> {
    let target = format!("tests/target/{}/", name);
    let mut config = Config::from_file(format!("tests/fixtures/{}/.cobalt.yml", name))
        .unwrap_or_default();
    let destdir = TempDir::new(name).expect("Tempdir not created");

    config.source = format!("tests/fixtures/{}/", name);
    config.dest = destdir
        .path()
        .to_str()
        .expect("Can't convert destdir to str")
        .to_owned();

    // try to create the target directory, ignore errors
    fs::create_dir_all(&config.dest).is_ok();

    let result = cobalt::build(&config);

    if result.is_ok() {
        let walker = WalkDir::new(&target)
            .into_iter()
            .filter_map(|e| e.ok())
            .filter(|e| e.file_type().is_file());

        // walk through fixture and created tmp directory and compare files
        for entry in walker {
            let relative = entry
                .path()
                .strip_prefix(&target)
                .expect("Comparison error");

            let mut original = String::new();
            File::open(entry.path())
                .expect("Comparison error")
                .read_to_string(&mut original)
                .expect("Could not read to string");

            let dest_file = Path::new(config.dest.as_str()).join(&relative);
            assert!(dest_file.exists(), "{:?} doesn't exist", dest_file);
            let mut created = String::new();
            File::open(dest_file.as_path())
                .expect("Comparison error")
                .read_to_string(&mut created)
                .expect("Could not read to string");

            assert_diff!(&original, &created, " ", 0);
        }

        // ensure no unnecessary files were created
        let walker = WalkDir::new(&config.dest)
            .into_iter()
            .filter_map(|e| e.ok())
            .filter(|e| e.file_type().is_file());

        for entry in walker {
            let extra_file = entry
                .path()
                .strip_prefix(&config.dest)
                .expect("Comparison error");
            let src_file = Path::new(&target).join(&extra_file);

            File::open(&src_file).expect(&format!("File {:?} does not exist in reference ({:?}).",
                                                  entry.path(),
                                                  src_file));
        }
    }

    // clean up
    try!(destdir.close());

    result
}

#[test]
pub fn copy_files() {
    run_test("copy_files").expect("Build error");
}

#[test]
pub fn custom_paths() {
    run_test("custom_paths").expect("Build error");
}

#[test]
pub fn custom_posts_folder() {
    run_test("custom_posts_folder").expect("Build error");
}

#[test]
pub fn custom_post_path() {
    run_test("custom_post_path").expect("Build error");
}

#[test]
pub fn dotfiles() {
    run_test("dotfiles").expect("Build error");
}

#[test]
pub fn drafts() {
    run_test("drafts").expect("Build error");
}

#[test]
pub fn drafts_not_shown_by_default() {
    run_test("drafts_not_shown_by_default").expect("Build error");
}

#[test]
pub fn example() {
    run_test("example").expect("Build error");
}

#[test]
pub fn hidden_posts_folder() {
    run_test("hidden_posts_folder").expect("Build error");
}

#[test]
pub fn custom_template_extensions() {
    run_test("custom_template_extensions").expect("Build error");
}

#[cfg(feature = "syntax-highlight")]
#[test]
pub fn syntax_highlight() {
    // Syntect isn't thread safe, for now run everything in the same test.
    run_test("syntax_highlight").expect("Build error");

    run_test("syntax_highlight_theme").expect("Build error");
}

#[test]
pub fn incomplete_rss() {
    let err = run_test("incomplete_rss");
    assert!(err.is_err());

    let err = err.unwrap_err();
    assert_eq!(format!("{}", err),
               "name, description and link need to be defined in the config file to generate RSS");
    assert_eq!(err.description(), "missing fields in config file");
}

#[test]
pub fn liquid_error() {
    let err = run_test("liquid_error");
    assert!(err.is_err());
    assert_eq!(err.unwrap_err().description(),
               "{{{ is not a valid identifier");
}

#[test]
pub fn liquid_raw() {
    run_test("liquid_escaped").expect("Build error");
}

#[test]
pub fn no_extends_error() {
    let err = run_test("no_extends_error");
    assert!(err.is_err());
    assert_contains!(err.unwrap_err().description(),
                     "Layout default_nonexistent.liquid can not be read (defined in \
                   \"index.html\")");
}

#[test]
pub fn sort_posts() {
    run_test("sort_posts").expect("Build error");
}

#[test]
pub fn post_order() {
    run_test("post_order").expect("Build error");
}

#[test]
pub fn previous_next() {
    run_test("previous_next").expect("Build error");
}

#[test]
pub fn rss() {
    run_test("rss").expect("Build error");
}

#[test]
pub fn jsonfeed() {
    run_test("jsonfeed").expect("Build error");
}

#[test]
pub fn ignore_files() {
    run_test("ignore_files").unwrap();
}

#[test]
pub fn yaml_error() {
    let err = run_test("yaml_error");
    assert!(err.is_err());
    assert_eq!(err.unwrap_err().description(), "scan error");
}

#[test]
pub fn excerpts() {
    run_test("excerpts").unwrap();
}

#[test]
pub fn posts_in_subfolder() {
    run_test("posts_in_subfolder").unwrap();
}

#[test]
pub fn empty_frontmatter() {
    run_test("empty_frontmatter").expect("Build error");
}

#[test]
pub fn querystrings() {
    run_test("querystrings").expect("Build error");
}
<commit_msg>refactor(tests): Make dir diffing reusable.<commit_after>#[macro_use]
extern crate difference;
extern crate cobalt;
extern crate tempdir;
extern crate walkdir;

use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::io::Read;
use tempdir::TempDir;
use walkdir::WalkDir;
use std::error::Error;
use cobalt::Config;

macro_rules! assert_contains {
    ($haystack: expr, $needle: expr) => {
        let text = $haystack.to_owned();
        println!("{}", text);
        assert!(text.contains($needle))
    }
}

fn assert_dirs_eq(expected: &Path, actual: &Path) {
    // Ensure everything was created.
    let walker = WalkDir::new(&actual)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|e| e.file_type().is_file());
    for entry in walker {
        let relative = entry
            .path()
            .strip_prefix(&actual)
            .expect("Comparison error");

        let mut original = String::new();
        File::open(entry.path())
            .expect("Comparison error")
            .read_to_string(&mut original)
            .expect("Could not read to string");

        let dest_file = Path::new(expected).join(&relative);
        assert!(dest_file.exists(), "{:?} doesn't exist", dest_file);
        let mut created = String::new();
        File::open(dest_file.as_path())
            .expect("Comparison error")
            .read_to_string(&mut created)
            .expect("Could not read to string");

        assert_diff!(&original, &created, " ", 0);
    }

    // Ensure no unnecessary files were created
    let walker = WalkDir::new(expected)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|e| e.file_type().is_file());
    for entry in walker {
        let extra_file = entry
            .path()
            .strip_prefix(expected)
            .expect("Comparison error");
        let src_file = Path::new(actual).join(&extra_file);

        File::open(&src_file).expect(&format!("File {:?} does not exist in reference ({:?}).",
                                              entry.path(),
                                              src_file));
    }
}

fn run_test(name: &str) -> Result<(), cobalt::Error> {
    let target = format!("tests/target/{}/", name);
    let target: PathBuf = target.into();
    let mut config = Config::from_file(format!("tests/fixtures/{}/.cobalt.yml", name))
        .unwrap_or_default();
    let destdir = TempDir::new(name).expect("Tempdir not created");

    config.source = format!("tests/fixtures/{}/", name);
    config.dest = destdir
        .path()
        .to_str()
        .expect("Can't convert destdir to str")
        .to_owned();

    // try to create the target directory, ignore errors
    fs::create_dir_all(&config.dest).is_ok();

    let result = cobalt::build(&config);

    if result.is_ok() {
        assert_dirs_eq(Path::new(config.dest.as_str()), &target);
    }

    // clean up
    destdir.close()?;

    result
}

#[test]
pub fn copy_files() {
    run_test("copy_files").expect("Build error");
}

#[test]
pub fn custom_paths() {
    run_test("custom_paths").expect("Build error");
}

#[test]
pub fn custom_posts_folder() {
    run_test("custom_posts_folder").expect("Build error");
}

#[test]
pub fn custom_post_path() {
    run_test("custom_post_path").expect("Build error");
}

#[test]
pub fn dotfiles() {
    run_test("dotfiles").expect("Build error");
}

#[test]
pub fn drafts() {
    run_test("drafts").expect("Build error");
}

#[test]
pub fn drafts_not_shown_by_default() {
    run_test("drafts_not_shown_by_default").expect("Build error");
}

#[test]
pub fn example() {
    run_test("example").expect("Build error");
}

#[test]
pub fn hidden_posts_folder() {
    run_test("hidden_posts_folder").expect("Build error");
}

#[test]
pub fn custom_template_extensions() {
    run_test("custom_template_extensions").expect("Build error");
}

#[cfg(feature = "syntax-highlight")]
#[test]
pub fn syntax_highlight() {
    // Syntect isn't thread safe, for now run everything in the same test.
    run_test("syntax_highlight").expect("Build error");

    run_test("syntax_highlight_theme").expect("Build error");
}

#[test]
pub fn incomplete_rss() {
    let err = run_test("incomplete_rss");
    assert!(err.is_err());

    let err = err.unwrap_err();
    assert_eq!(format!("{}", err),
               "name, description and link need to be defined in the config file to generate RSS");
    assert_eq!(err.description(), "missing fields in config file");
}

#[test]
pub fn liquid_error() {
    let err = run_test("liquid_error");
    assert!(err.is_err());
    assert_eq!(err.unwrap_err().description(),
               "{{{ is not a valid identifier");
}

#[test]
pub fn liquid_raw() {
    run_test("liquid_escaped").expect("Build error");
}

#[test]
pub fn no_extends_error() {
    let err = run_test("no_extends_error");
    assert!(err.is_err());
    assert_contains!(err.unwrap_err().description(),
                     "Layout default_nonexistent.liquid can not be read (defined in \
                   \"index.html\")");
}

#[test]
pub fn sort_posts() {
    run_test("sort_posts").expect("Build error");
}

#[test]
pub fn post_order() {
    run_test("post_order").expect("Build error");
}

#[test]
pub fn previous_next() {
    run_test("previous_next").expect("Build error");
}

#[test]
pub fn rss() {
    run_test("rss").expect("Build error");
}

#[test]
pub fn jsonfeed() {
    run_test("jsonfeed").expect("Build error");
}

#[test]
pub fn ignore_files() {
    run_test("ignore_files").unwrap();
}

#[test]
pub fn yaml_error() {
    let err = run_test("yaml_error");
    assert!(err.is_err());
    assert_eq!(err.unwrap_err().description(), "scan error");
}

#[test]
pub fn excerpts() {
    run_test("excerpts").unwrap();
}

#[test]
pub fn posts_in_subfolder() {
    run_test("posts_in_subfolder").unwrap();
}

#[test]
pub fn empty_frontmatter() {
    run_test("empty_frontmatter").expect("Build error");
}

#[test]
pub fn querystrings() {
    run_test("querystrings").expect("Build error");
}
<|endoftext|>


<commit_before>pub mod grammar;

use fall_tree::File;
use fall_parse::{self, TreeBuilder};
use self::grammar::*;

pub fn parse(text: String) -> File {
    register_node_types();
    fall_parse::parse(
        text,
        grammar::FILE,
        grammar::TOKENIZER,
        &parse_file
    )
}

fn parse_file(b: &mut TreeBuilder) {
    parse_nodes(b);
    b.skip_until(&[KW_TOKENIZER]);
    parse_tokenizer(b);

    b.parse_many(&parse_syn_rule)
}

fn parse_nodes(b: &mut TreeBuilder) -> bool {
    b.start(NODES_DEF);
    if !b.try_eat(KW_NODES) {
        b.rollback(NODES_DEF);
        return false;
    }
    if b.try_eat(LBRACE) {
        b.parse_many(&|b| b.try_eat(IDENT));
        b.try_eat(RBRACE);
    }
    b.finish(NODES_DEF);
    true
}

fn parse_tokenizer(b: &mut TreeBuilder) -> bool {
    b.start(TOKENIZER_DEF);
    if !b.try_eat(KW_TOKENIZER) {
        b.rollback(TOKENIZER_DEF);
        return false;
    }
    if b.try_eat(LBRACE) {
        b.parse_many(&|b| {
            b.skip_until(&[RBRACE, IDENT]);
            parse_lex_rule(b)
        });
        b.try_eat(RBRACE);
    }
    b.finish(TOKENIZER_DEF);
    true
}

fn parse_lex_rule(b: &mut TreeBuilder) -> bool {
    b.start(TOKEN_DEF);
    if !b.try_eat(IDENT) {
        b.rollback(TOKEN_DEF);
        return false
    }
    parse_string(b) && parse_string(b);
    b.finish(TOKEN_DEF);
    true
}

fn parse_syn_rule(b: &mut TreeBuilder) -> bool {
    b.start(RULE_DEF);
    if !b.try_eat(KW_RULE) {
        b.rollback(RULE_DEF);
        return false;
    }
    let r = b.try_eat(IDENT) && b.try_eat(LBRACE);
    if r {
        b.start(ALT);
        b.parse_many(&|b| {
            if b.next_is(RBRACE) || b.current().is_none() {
                false
            } else  {
                b.bump();
                true
            }
        });
        b.finish(ALT);
        b.try_eat(RBRACE);
    }
    b.finish(RULE_DEF);
    true
}

fn parse_string(b: &mut TreeBuilder) -> bool {
    b.start(STRING);
    if b.try_eat(SIMPLE_STRING) || b.try_eat(HASH_STRING) {
        b.finish(STRING);
        true
    } else {
        b.rollback(STRING);
        false
    }
}

fn parse_raw_string(s: &str) -> Option<usize> {
    let quote_start = s.find('"').unwrap();
    let closing = &"\"########################"[..quote_start];
    s[quote_start + 1..].find(closing).map(|i| i + quote_start + 1 + closing.len())
}

#[cfg(test)]
mod tests {
    extern crate difference;
    use super::*;

    fn match_ast(actual: &str, expected: &str) {
        if expected.trim() != actual.trim() {
            difference::print_diff(&actual, &expected, "\n");
            panic!("Mismatch!")
        }
    }

    fn ast(code: &str) -> String {
        parse(code.to_owned()).dump()
    }

    #[test]
    fn empty_file() {
        match_ast(&ast(""), r#"FILE """#)
    }

    #[test]
    fn nodes() {
        match_ast(&ast("nodes { foo bar }"), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    IDENT "foo"
    IDENT "bar"
    RBRACE "}"
"#)
    }

    #[test]
    fn tokenizer() {
        match_ast(&ast(r#"nodes {} tokenizer { foo "foo" id r"\w+" ext "ext" "super::ext"}"#), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    RBRACE "}"
  TOKENIZER_DEF
    KW_TOKENIZER "tokenizer"
    LBRACE "{"
    TOKEN_DEF
      IDENT "foo"
      STRING
        SIMPLE_STRING "\"foo\""
    TOKEN_DEF
      IDENT "id"
      STRING
        SIMPLE_STRING "r\"\\w+\""
    TOKEN_DEF
      IDENT "ext"
      STRING
        SIMPLE_STRING "\"ext\""
      STRING
        SIMPLE_STRING "\"super::ext\""
    RBRACE "}"
"#)
    }

    #[test]
    fn rules() {
        match_ast(&ast(r#"nodes {} tokenizer {}
rule f { foo <commit> ( bar )* }

rule b { foo | bar }"#), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    RBRACE "}"
  TOKENIZER_DEF
    KW_TOKENIZER "tokenizer"
    LBRACE "{"
    RBRACE "}"
  RULE_DEF
    KW_RULE "rule"
    IDENT "f"
    LBRACE "{"
    ALT
      IDENT "foo"
      ERROR "<"
      IDENT "commit"
      ERROR ">"
      ERROR "("
      IDENT "bar"
      ERROR ")"
      START "*"
    RBRACE "}"
  RULE_DEF
    KW_RULE "rule"
    IDENT "b"
    LBRACE "{"
    ALT
      IDENT "foo"
      PIPE "|"
      IDENT "bar"
    RBRACE "}"
"#)
    }
}
<commit_msg>Better parsing of alternatives<commit_after>pub mod grammar;

use fall_tree::File;
use fall_parse::{self, TreeBuilder};
use self::grammar::*;

pub fn parse(text: String) -> File {
    register_node_types();
    fall_parse::parse(
        text,
        grammar::FILE,
        grammar::TOKENIZER,
        &parse_file
    )
}

fn parse_file(b: &mut TreeBuilder) {
    parse_nodes(b);
    b.skip_until(&[KW_TOKENIZER]);
    parse_tokenizer(b);

    b.parse_many(&parse_syn_rule)
}

fn parse_nodes(b: &mut TreeBuilder) -> bool {
    b.start(NODES_DEF);
    if !b.try_eat(KW_NODES) {
        b.rollback(NODES_DEF);
        return false;
    }
    if b.try_eat(LBRACE) {
        b.parse_many(&|b| b.try_eat(IDENT));
        b.try_eat(RBRACE);
    }
    b.finish(NODES_DEF);
    true
}

fn parse_tokenizer(b: &mut TreeBuilder) -> bool {
    b.start(TOKENIZER_DEF);
    if !b.try_eat(KW_TOKENIZER) {
        b.rollback(TOKENIZER_DEF);
        return false;
    }
    if b.try_eat(LBRACE) {
        b.parse_many(&|b| {
            b.skip_until(&[RBRACE, IDENT]);
            parse_lex_rule(b)
        });
        b.try_eat(RBRACE);
    }
    b.finish(TOKENIZER_DEF);
    true
}

fn parse_lex_rule(b: &mut TreeBuilder) -> bool {
    b.start(TOKEN_DEF);
    if !b.try_eat(IDENT) {
        b.rollback(TOKEN_DEF);
        return false
    }
    parse_string(b) && parse_string(b);
    b.finish(TOKEN_DEF);
    true
}

fn parse_syn_rule(b: &mut TreeBuilder) -> bool {
    b.start(RULE_DEF);
    if !b.try_eat(KW_RULE) {
        b.rollback(RULE_DEF);
        return false;
    }
    if b.try_eat(IDENT) && b.try_eat(LBRACE) {
        b.parse_many(&|b| {
            if b.next_is(RBRACE) || b.current().is_none() {
                false
            } else {
                parse_alt(b);
                if !b.next_is(RBRACE) {
                    b.try_eat(PIPE);
                }
                true
            }
        });
    }
    b.finish(RULE_DEF);
    true
}

fn parse_alt(b: &mut TreeBuilder) {
    b.start(ALT);
    b.parse_many(&|b| {
        if b.current().is_none() || b.next_is(RBRACE) || b.next_is(PIPE) {
            false
        } else {
            b.bump();
            true
        }
    });
    b.finish(ALT)
}

fn parse_string(b: &mut TreeBuilder) -> bool {
    b.start(STRING);
    if b.try_eat(SIMPLE_STRING) || b.try_eat(HASH_STRING) {
        b.finish(STRING);
        true
    } else {
        b.rollback(STRING);
        false
    }
}

fn parse_raw_string(s: &str) -> Option<usize> {
    let quote_start = s.find('"').unwrap();
    let closing = &"\"########################"[..quote_start];
    s[quote_start + 1..].find(closing).map(|i| i + quote_start + 1 + closing.len())
}

#[cfg(test)]
mod tests {
    extern crate difference;

    use super::*;

    fn match_ast(actual: &str, expected: &str) {
        if expected.trim() != actual.trim() {
            difference::print_diff(&actual, &expected, "\n");
            panic!("Mismatch!")
        }
    }

    fn ast(code: &str) -> String {
        parse(code.to_owned()).dump()
    }

    #[test]
    fn empty_file() {
        match_ast(&ast(""), r#"FILE """#)
    }

    #[test]
    fn nodes() {
        match_ast(&ast("nodes { foo bar }"), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    IDENT "foo"
    IDENT "bar"
    RBRACE "}"
"#)
    }

    #[test]
    fn tokenizer() {
        match_ast(&ast(r#"nodes {} tokenizer { foo "foo" id r"\w+" ext "ext" "super::ext"}"#), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    RBRACE "}"
  TOKENIZER_DEF
    KW_TOKENIZER "tokenizer"
    LBRACE "{"
    TOKEN_DEF
      IDENT "foo"
      STRING
        SIMPLE_STRING "\"foo\""
    TOKEN_DEF
      IDENT "id"
      STRING
        SIMPLE_STRING "r\"\\w+\""
    TOKEN_DEF
      IDENT "ext"
      STRING
        SIMPLE_STRING "\"ext\""
      STRING
        SIMPLE_STRING "\"super::ext\""
    RBRACE "}"
"#)
    }

    #[test]
    #[ignore]
    fn rules() {
        match_ast(&ast(r#"nodes {} tokenizer {}
rule f { foo <commit> ( bar )* }

rule b { foo | bar }"#), r#"
FILE
  NODES_DEF
    KW_NODES "nodes"
    LBRACE "{"
    RBRACE "}"
  TOKENIZER_DEF
    KW_TOKENIZER "tokenizer"
    LBRACE "{"
    RBRACE "}"
  RULE_DEF
    KW_RULE "rule"
    IDENT "f"
    LBRACE "{"
    ALT
      IDENT "foo"
      ERROR "<"
      IDENT "commit"
      ERROR ">"
      ERROR "("
      IDENT "bar"
      ERROR ")"
      START "*"
    RBRACE "}"
  RULE_DEF
    KW_RULE "rule"
    IDENT "b"
    LBRACE "{"
    ALT
      IDENT "foo"
    PIPE "|"
    ALT
      IDENT "bar"
    RBRACE "}"
"#)
    }
}
<|endoftext|>


<commit_before>mod common;

use crate::common::assert_linked_with;
use linkify::LinkFinder;

#[test]
fn no_links() {
    assert_not_linked("");
    assert_not_linked("foo");
    assert_not_linked(":");
    assert_not_linked("://");
    assert_not_linked(":::");
}

#[test]
fn schemes() {
    assert_not_linked("://foo");
    assert_not_linked("1://foo");
    assert_not_linked("123://foo");
    assert_not_linked("+://foo");
    assert_not_linked("-://foo");
    assert_not_linked(".://foo");
    assert_not_linked("1abc://foo");
    assert_linked("a://foo", "|a://foo|");
    assert_linked("a123://foo", "|a123://foo|");
    assert_linked("a123b://foo", "|a123b://foo|");
    assert_linked("a+b://foo", "|a+b://foo|");
    assert_linked("a-b://foo", "|a-b://foo|");
    assert_linked("a.b://foo", "|a.b://foo|");
    assert_linked("ABC://foo", "|ABC://foo|");
    assert_linked(".http://example.org/", ".|http://example.org/|");
    assert_linked("1.http://example.org/", "1.|http://example.org/|");
}

#[test]
fn authority() {
    assert_not_linked("ab://");
    assert_not_linked("http://");
    assert_not_linked("http:// ");
    assert_not_linked("\"http://\"");
    assert_not_linked("\"http://...\", ");

    assert_linked("http://a.", "|http://a|.");
}

#[test]
fn single_links() {
    assert_linked("ab://c", "|ab://c|");
    assert_linked("http://example.org/", "|http://example.org/|");
    assert_linked("http://example.org/123", "|http://example.org/123|");
    assert_linked(
        "http://example.org/?foo=test&bar=123",
        "|http://example.org/?foo=test&bar=123|",
    );
    assert_linked(
        "http://example.org/?foo=%20",
        "|http://example.org/?foo=%20|",
    );
    assert_linked("http://example.org/%3C", "|http://example.org/%3C|");
}

#[test]
fn space_characters_stop_url() {
    assert_linked("foo http://example.org/", "foo |http://example.org/|");
    assert_linked("http://example.org/ bar", "|http://example.org/| bar");
    assert_linked("http://example.org/\tbar", "|http://example.org/|\tbar");
    assert_linked("http://example.org/\nbar", "|http://example.org/|\nbar");
    assert_linked(
        "http://example.org/\u{0B}bar",
        "|http://example.org/|\u{0B}bar",
    );
    assert_linked(
        "http://example.org/\u{0C}bar",
        "|http://example.org/|\u{0C}bar",
    );
    assert_linked("http://example.org/\rbar", "|http://example.org/|\rbar");
}

#[test]
fn illegal_characters_stop_url() {
    assert_linked("http://example.org/<", "|http://example.org/|<");
    assert_linked("http://example.org/>", "|http://example.org/|>");
    assert_linked("http://example.org/<>", "|http://example.org/|<>");
    assert_linked("http://example.org/\u{00}", "|http://example.org/|\u{00}");
    assert_linked("http://example.org/\u{0E}", "|http://example.org/|\u{0E}");
    assert_linked("http://example.org/\u{7F}", "|http://example.org/|\u{7F}");
    assert_linked("http://example.org/\u{9F}", "|http://example.org/|\u{9F}");
}

#[test]
fn delimiter_at_end() {
    assert_linked("http://example.org/.", "|http://example.org/|.");
    assert_linked("http://example.org/..", "|http://example.org/|..");
    assert_linked("http://example.org/,", "|http://example.org/|,");
    assert_linked("http://example.org/:", "|http://example.org/|:");
    assert_linked("http://example.org/?", "|http://example.org/|?");
    assert_linked("http://example.org/!", "|http://example.org/|!");
    assert_linked("http://example.org/;", "|http://example.org/|;");
}

#[test]
fn matching_punctuation() {
    assert_linked("http://example.org/a(b)", "|http://example.org/a(b)|");
    assert_linked("http://example.org/a[b]", "|http://example.org/a[b]|");
    assert_linked("http://example.org/a{b}", "|http://example.org/a{b}|");
    assert_linked("http://example.org/a'b'", "|http://example.org/a'b'|");
    assert_linked("(http://example.org/)", "(|http://example.org/|)");
    assert_linked("[http://example.org/]", "[|http://example.org/|]");
    assert_linked("{http://example.org/}", "{|http://example.org/|}");
    assert_linked("\"http://example.org/\"", "\"|http://example.org/|\"");
    assert_linked("'http://example.org/'", "'|http://example.org/|'");
}

#[test]
fn matching_punctuation_tricky() {
    assert_linked("((http://example.org/))", "((|http://example.org/|))");
    assert_linked(
        "((http://example.org/a(b)))",
        "((|http://example.org/a(b)|))",
    );
    assert_linked("[(http://example.org/)]", "[(|http://example.org/|)]");
    assert_linked("(http://example.org/).", "(|http://example.org/|).");
    assert_linked("(http://example.org/.)", "(|http://example.org/|.)");
    assert_linked("http://example.org/>", "|http://example.org/|>");
    // not sure about these
    assert_linked("http://example.org/(", "|http://example.org/|(");
    assert_linked("http://example.org/(.", "|http://example.org/|(.");
    assert_linked("http://example.org/]()", "|http://example.org/|]()");
}

#[test]
fn single_quote() {
    assert_linked(
        "http://example.org/\'_(foo)",
        "|http://example.org/\'_(foo)|",
    );
    assert_linked(
        "http://example.org/\'_(foo)\'",
        "|http://example.org/\'_(foo)\'|",
    );
    assert_linked("http://example.org/\'\'", "|http://example.org/\'\'|");
    assert_linked("http://example.org/\'\'\'", "|http://example.org/\'\'|\'");
    assert_linked("http://example.org/\'.", "|http://example.org/|\'.");
    assert_linked("http://example.org/\'a", "|http://example.org/\'a|");
    assert_linked("http://example.org/it's", "|http://example.org/it's|");
}

#[test]
fn double_quote() {
    // " not allowed in URLs
    assert_linked("http://example.org/\"a", "|http://example.org/|\"a");
    assert_linked("http://example.org/\"a\"", "|http://example.org/|\"a\"");
}

#[test]
fn grave_quote() {
    // ` not allowed in URLs
    assert_linked("http://example.org/`a", "|http://example.org/|`a");
    assert_linked("http://example.org/`a`", "|http://example.org/|`a`");
}

#[test]
fn html() {
    assert_linked("http://example.org\">", "|http://example.org|\">");
    assert_linked("http://example.org'>", "|http://example.org|'>");
    assert_linked("http://example.org\"/>", "|http://example.org|\"/>");
    assert_linked("http://example.org'/>", "|http://example.org|'/>");
    assert_linked("http://example.org
", "|http://example.org|
");
    assert_linked("http://example.org
", "|http://example.org|
");
}

#[test]
fn css() {
    assert_linked("http://example.org\");", "|http://example.org|\");");
    assert_linked("http://example.org');", "|http://example.org|');");
}

#[test]
fn slash() {
    assert_linked("http://example.org/", "|http://example.org/|");
    assert_linked("http://example.org/a/", "|http://example.org/a/|");
    assert_linked("http://example.org//", "|http://example.org//|");
}

#[test]
fn multiple() {
    assert_linked(
        "http://one.org/ http://two.org/",
        "|http://one.org/| |http://two.org/|",
    );
    assert_linked(
        "http://one.org/ : http://two.org/",
        "|http://one.org/| : |http://two.org/|",
    );
    assert_linked(
        "(http://one.org/)(http://two.org/)",
        "(|http://one.org/|)(|http://two.org/|)",
    );
}

#[test]
fn international() {
    assert_linked(
        "http://üñîçøðé.com/ä",
        "|http://üñîçøðé.com/ä|",
    );
    assert_linked("http://example.org/\u{A1}", "|http://example.org/\u{A1}|");
    assert_linked("http://example.org/\u{A2}", "|http://example.org/\u{A2}|");
    assert_linked(
        "http://example.org/\u{1F600}",
        "|http://example.org/\u{1F600}|",
    );
    assert_linked("http://example.org/\u{A2}/", "|http://example.org/\u{A2}/|");
    assert_linked(
        "http://xn--c1h.example.com/",
        "|http://xn--c1h.example.com/|",
    );
}

#[test]
fn fuzz() {
    assert_not_linked("ab:/ϸ");
}

fn assert_not_linked(s: &str) {
    assert_linked(s, s);
}

fn assert_linked(input: &str, expected: &str) {
    let finder = LinkFinder::new();
    assert_linked_with(&finder, input, expected);
}
<commit_msg>Add test for parsing image URLs<commit_after>mod common;

use crate::common::assert_linked_with;
use linkify::LinkFinder;

#[test]
fn no_links() {
    assert_not_linked("");
    assert_not_linked("foo");
    assert_not_linked(":");
    assert_not_linked("://");
    assert_not_linked(":::");
}

#[test]
fn schemes() {
    assert_not_linked("://foo");
    assert_not_linked("1://foo");
    assert_not_linked("123://foo");
    assert_not_linked("+://foo");
    assert_not_linked("-://foo");
    assert_not_linked(".://foo");
    assert_not_linked("1abc://foo");
    assert_linked("a://foo", "|a://foo|");
    assert_linked("a123://foo", "|a123://foo|");
    assert_linked("a123b://foo", "|a123b://foo|");
    assert_linked("a+b://foo", "|a+b://foo|");
    assert_linked("a-b://foo", "|a-b://foo|");
    assert_linked("a.b://foo", "|a.b://foo|");
    assert_linked("ABC://foo", "|ABC://foo|");
    assert_linked(".http://example.org/", ".|http://example.org/|");
    assert_linked("1.http://example.org/", "1.|http://example.org/|");
}

#[test]
fn authority() {
    assert_not_linked("ab://");
    assert_not_linked("http://");
    assert_not_linked("http:// ");
    assert_not_linked("\"http://\"");
    assert_not_linked("\"http://...\", ");

    assert_linked("http://a.", "|http://a|.");
}

#[test]
fn single_links() {
    assert_linked("ab://c", "|ab://c|");
    assert_linked("http://example.org/", "|http://example.org/|");
    assert_linked("http://example.org/123", "|http://example.org/123|");
    assert_linked(
        "http://example.org/?foo=test&bar=123",
        "|http://example.org/?foo=test&bar=123|",
    );
    assert_linked(
        "http://example.org/?foo=%20",
        "|http://example.org/?foo=%20|",
    );
    assert_linked("http://example.org/%3C", "|http://example.org/%3C|");
}

#[test]
fn space_characters_stop_url() {
    assert_linked("foo http://example.org/", "foo |http://example.org/|");
    assert_linked("http://example.org/ bar", "|http://example.org/| bar");
    assert_linked("http://example.org/\tbar", "|http://example.org/|\tbar");
    assert_linked("http://example.org/\nbar", "|http://example.org/|\nbar");
    assert_linked(
        "http://example.org/\u{0B}bar",
        "|http://example.org/|\u{0B}bar",
    );
    assert_linked(
        "http://example.org/\u{0C}bar",
        "|http://example.org/|\u{0C}bar",
    );
    assert_linked("http://example.org/\rbar", "|http://example.org/|\rbar");
}

#[test]
fn illegal_characters_stop_url() {
    assert_linked("http://example.org/<", "|http://example.org/|<");
    assert_linked("http://example.org/>", "|http://example.org/|>");
    assert_linked("http://example.org/<>", "|http://example.org/|<>");
    assert_linked("http://example.org/\u{00}", "|http://example.org/|\u{00}");
    assert_linked("http://example.org/\u{0E}", "|http://example.org/|\u{0E}");
    assert_linked("http://example.org/\u{7F}", "|http://example.org/|\u{7F}");
    assert_linked("http://example.org/\u{9F}", "|http://example.org/|\u{9F}");
}

#[test]
fn delimiter_at_end() {
    assert_linked("http://example.org/.", "|http://example.org/|.");
    assert_linked("http://example.org/..", "|http://example.org/|..");
    assert_linked("http://example.org/,", "|http://example.org/|,");
    assert_linked("http://example.org/:", "|http://example.org/|:");
    assert_linked("http://example.org/?", "|http://example.org/|?");
    assert_linked("http://example.org/!", "|http://example.org/|!");
    assert_linked("http://example.org/;", "|http://example.org/|;");
}

#[test]
fn matching_punctuation() {
    assert_linked("http://example.org/a(b)", "|http://example.org/a(b)|");
    assert_linked("http://example.org/a[b]", "|http://example.org/a[b]|");
    assert_linked("http://example.org/a{b}", "|http://example.org/a{b}|");
    assert_linked("http://example.org/a'b'", "|http://example.org/a'b'|");
    assert_linked("(http://example.org/)", "(|http://example.org/|)");
    assert_linked("[http://example.org/]", "[|http://example.org/|]");
    assert_linked("{http://example.org/}", "{|http://example.org/|}");
    assert_linked("\"http://example.org/\"", "\"|http://example.org/|\"");
    assert_linked("'http://example.org/'", "'|http://example.org/|'");
}

#[test]
fn matching_punctuation_tricky() {
    assert_linked("((http://example.org/))", "((|http://example.org/|))");
    assert_linked(
        "((http://example.org/a(b)))",
        "((|http://example.org/a(b)|))",
    );
    assert_linked("[(http://example.org/)]", "[(|http://example.org/|)]");
    assert_linked("(http://example.org/).", "(|http://example.org/|).");
    assert_linked("(http://example.org/.)", "(|http://example.org/|.)");
    assert_linked("http://example.org/>", "|http://example.org/|>");
    // not sure about these
    assert_linked("http://example.org/(", "|http://example.org/|(");
    assert_linked("http://example.org/(.", "|http://example.org/|(.");
    assert_linked("http://example.org/]()", "|http://example.org/|]()");
}

#[test]
fn single_quote() {
    assert_linked(
        "http://example.org/\'_(foo)",
        "|http://example.org/\'_(foo)|",
    );
    assert_linked(
        "http://example.org/\'_(foo)\'",
        "|http://example.org/\'_(foo)\'|",
    );
    assert_linked("http://example.org/\'\'", "|http://example.org/\'\'|");
    assert_linked("http://example.org/\'\'\'", "|http://example.org/\'\'|\'");
    assert_linked("http://example.org/\'.", "|http://example.org/|\'.");
    assert_linked("http://example.org/\'a", "|http://example.org/\'a|");
    assert_linked("http://example.org/it's", "|http://example.org/it's|");
}

#[test]
fn double_quote() {
    // " not allowed in URLs
    assert_linked("http://example.org/\"a", "|http://example.org/|\"a");
    assert_linked("http://example.org/\"a\"", "|http://example.org/|\"a\"");
}

#[test]
fn grave_quote() {
    // ` not allowed in URLs
    assert_linked("http://example.org/`a", "|http://example.org/|`a");
    assert_linked("http://example.org/`a`", "|http://example.org/|`a`");
}

#[test]
fn html() {
    assert_linked("http://example.org\">", "|http://example.org|\">");
    assert_linked("http://example.org'>", "|http://example.org|'>");
    assert_linked("http://example.org\"/>", "|http://example.org|\"/>");
    assert_linked("http://example.org'/>", "|http://example.org|'/>");
    assert_linked("http://example.org
", "|http://example.org|
");
    assert_linked("http://example.org
", "|http://example.org|
");
}

#[test]
fn css() {
    assert_linked("http://example.org\");", "|http://example.org|\");");
    assert_linked("http://example.org');", "|http://example.org|');");
}

#[test]
fn images() {
    assert_linked(
        r#""#,
        r#""#,
    );
}

#[test]
fn slash() {
    assert_linked("http://example.org/", "|http://example.org/|");
    assert_linked("http://example.org/a/", "|http://example.org/a/|");
    assert_linked("http://example.org//", "|http://example.org//|");
}

#[test]
fn multiple() {
    assert_linked(
        "http://one.org/ http://two.org/",
        "|http://one.org/| |http://two.org/|",
    );
    assert_linked(
        "http://one.org/ : http://two.org/",
        "|http://one.org/| : |http://two.org/|",
    );
    assert_linked(
        "(http://one.org/)(http://two.org/)",
        "(|http://one.org/|)(|http://two.org/|)",
    );
}

#[test]
fn international() {
    assert_linked("http://üñîçøðé.com/ä", "|http://üñîçøðé.com/ä|");
    assert_linked("http://example.org/\u{A1}", "|http://example.org/\u{A1}|");
    assert_linked("http://example.org/\u{A2}", "|http://example.org/\u{A2}|");
    assert_linked(
        "http://example.org/\u{1F600}",
        "|http://example.org/\u{1F600}|",
    );
    assert_linked("http://example.org/\u{A2}/", "|http://example.org/\u{A2}/|");
    assert_linked(
        "http://xn--c1h.example.com/",
        "|http://xn--c1h.example.com/|",
    );
}

#[test]
fn fuzz() {
    assert_not_linked("ab:/ϸ");
}

fn assert_not_linked(s: &str) {
    assert_linked(s, s);
}

fn assert_linked(input: &str, expected: &str) {
    let finder = LinkFinder::new();
    assert_linked_with(&finder, input, expected);
}
<|endoftext|>


<commit_before>// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.

//! A module contains test cases of stale peers gc.

use std::sync::Arc;
use std::thread;
use std::time::*;

use kvproto::raft_serverpb::{PeerState, RegionLocalState};
use raft::eraftpb::MessageType;

use engine_rocks::Compat;
use engine_traits::Peekable;
use engine_traits::CF_RAFT;
use test_raftstore::*;

/// A helper function for testing the behaviour of the gc of stale peer
/// which is out of region.
/// If a peer detects the leader is missing for a specified long time,
/// it should consider itself as a stale peer which is removed from the region.
/// This test case covers the following scenario:
/// At first, there are three peer A, B, C in the cluster, and A is leader.
/// Peer B gets down. And then A adds D, E, F into the cluster.
/// Peer D becomes leader of the new cluster, and then removes peer A, B, C.
/// After all these peer in and out, now the cluster has peer D, E, F.
/// If peer B goes up at this moment, it still thinks it is one of the cluster
/// and has peers A, C. However, it could not reach A, C since they are removed from
/// the cluster or probably destroyed.
/// Meantime, D, E, F would not reach B, Since it's not in the cluster anymore.
/// In this case, Peer B would notice that the leader is missing for a long time,
/// and it would check with pd to confirm whether it's still a member of the cluster.
/// If not, it should destroy itself as a stale peer which is removed out already.
fn test_stale_peer_out_of_region<T: Simulator>(cluster: &mut Cluster<T>) {
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    pd_client.must_add_peer(r1, new_peer(2, 2));
    pd_client.must_add_peer(r1, new_peer(3, 3));
    let (key, value) = (b"k1", b"v1");
    cluster.must_put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);
    must_get_equal(&engine_2, key, value);

    // Isolate peer 2 from rest of the cluster.
    cluster.add_send_filter(IsolationFilterFactory::new(2));

    // In case 2 is leader, it will fail to pass the healthy nodes check,
    // so remove isolated node first. Because 2 is isolated, so it can't remove itself.
    pd_client.must_remove_peer(r1, new_peer(2, 2));

    // Add peer [(4, 4), (5, 5), (6, 6)].
    pd_client.must_add_peer(r1, new_peer(4, 4));
    pd_client.must_add_peer(r1, new_peer(5, 5));
    pd_client.must_add_peer(r1, new_peer(6, 6));

    // Remove peer [(1, 1), (3, 3)].
    pd_client.must_remove_peer(r1, new_peer(1, 1));
    pd_client.must_remove_peer(r1, new_peer(3, 3));

    // Keep peer 2 isolated. Otherwise whether peer 3 is destroyed or not,
    // it will handle the stale raft message from peer 2 and cause peer 2 to
    // destroy itself earlier than this test case expects.

    // Wait for max_leader_missing_duration to time out.
    cluster.must_remove_region(2, r1);

    // Check whether this region is still functional properly.
    let (key2, value2) = (b"k2", b"v2");
    cluster.must_put(key2, value2);
    assert_eq!(cluster.get(key2), Some(value2.to_vec()));

    // Check whether peer(2, 2) and its data are destroyed.
    must_get_none(&engine_2, key);
    must_get_none(&engine_2, key2);
    let state_key = keys::region_state_key(1);
    let state: RegionLocalState = engine_2
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);
}

#[test]
fn test_node_stale_peer_out_of_region() {
    let count = 6;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_out_of_region(&mut cluster);
}

#[test]
fn test_server_stale_peer_out_of_region() {
    let count = 6;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_out_of_region(&mut cluster);
}

/// A help function for testing the behaviour of the gc of stale peer
/// which is out or region.
/// If a peer detects the leader is missing for a specified long time,
/// it should consider itself as a stale peer which is removed from the region.
/// This test case covers the following scenario:
/// A peer, B is initialized as a replicated peer without data after
/// receiving a single raft AE message. But then it goes through some process like
/// the case of `test_stale_peer_out_of_region`, it's removed out of the region
/// and wouldn't be contacted anymore.
/// In both cases, peer B would notice that the leader is missing for a long time,
/// and it's an initialized peer without any data. It would destroy itself as
/// as stale peer directly and should not impact other region data on the same store.
fn test_stale_peer_without_data<T: Simulator>(cluster: &mut Cluster<T>, right_derive: bool) {
    cluster.cfg.raft_store.right_derive_when_split = right_derive;

    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    cluster.must_put(b"k1", b"v1");
    cluster.must_put(b"k3", b"v3");
    let region = cluster.get_region(b"");
    pd_client.must_add_peer(r1, new_peer(2, 2));
    cluster.must_split(&region, b"k2");
    pd_client.must_add_peer(r1, new_peer(3, 3));

    let engine3 = cluster.get_engine(3);
    if right_derive {
        must_get_none(&engine3, b"k1");
        must_get_equal(&engine3, b"k3", b"v3");
    } else {
        must_get_equal(&engine3, b"k1", b"v1");
        must_get_none(&engine3, b"k3");
    }

    let new_region = if right_derive {
        cluster.get_region(b"k1")
    } else {
        cluster.get_region(b"k3")
    };
    let new_region_id = new_region.get_id();
    // Block peer (3, 4) at receiving snapshot, but not the heartbeat
    cluster.add_send_filter(CloneFilterFactory(
        RegionPacketFilter::new(new_region_id, 3).msg_type(MessageType::MsgSnapshot),
    ));

    pd_client.must_add_peer(new_region_id, new_peer(3, 4));

    // Wait for the heartbeat broadcasted from peer (1, 1000) to peer (3, 4).
    cluster.must_region_exist(new_region_id, 3);

    // And then isolate peer (3, 4) from peer (1, 1000).
    cluster.add_send_filter(IsolationFilterFactory::new(3));

    pd_client.must_remove_peer(new_region_id, new_peer(3, 4));

    cluster.must_remove_region(3, new_region_id);

    // There must be no data on store 3 belongs to new region
    if right_derive {
        must_get_none(&engine3, b"k1");
    } else {
        must_get_none(&engine3, b"k3");
    }

    // Check whether peer(3, 4) is destroyed.
    // Before peer 4 is destroyed, a tombstone mark will be written into the engine.
    // So we could check the tombstone mark to make sure peer 4 is destroyed.
    let state_key = keys::region_state_key(new_region_id);
    let state: RegionLocalState = engine3
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);

    // other region should not be affected.
    if right_derive {
        must_get_equal(&engine3, b"k3", b"v3");
    } else {
        must_get_equal(&engine3, b"k1", b"v1");
    }
}

#[test]
fn test_node_stale_peer_without_data_left_derive_when_split() {
    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, false);
}

#[test]
fn test_node_stale_peer_without_data_right_derive_when_split() {
    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, true);
}

#[test]
fn test_server_stale_peer_without_data_left_derive_when_split() {
    let count = 3;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, false);
}

#[test]
fn test_server_stale_peer_without_data_right_derive_when_split() {
    let count = 3;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, true);
}

/// A help function for testing the behaviour of the gc of stale learner
/// which is out or region.
#[test]
fn test_stale_learner() {
    let mut cluster = new_server_cluster(0, 4);
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    pd_client.must_add_peer(r1, new_peer(2, 2));
    pd_client.must_add_peer(r1, new_learner_peer(3, 3));
    cluster.must_put(b"k1", b"v1");
    let engine3 = cluster.get_engine(3);
    must_get_equal(&engine3, b"k1", b"v1");

    // And then isolate peer on store 3 from leader.
    cluster.add_send_filter(IsolationFilterFactory::new(3));

    // Add a new peer to increase the conf version.
    pd_client.must_add_peer(r1, new_peer(4, 4));

    // It should not be deleted.
    thread::sleep(Duration::from_secs(3));
    must_get_equal(&engine3, b"k1", b"v1");

    // Promote the learner
    pd_client.must_add_peer(r1, new_peer(3, 3));

    // It should not be deleted.
    thread::sleep(Duration::from_secs(3));
    must_get_equal(&engine3, b"k1", b"v1");

    // Delete the learner
    pd_client.must_remove_peer(r1, new_peer(3, 3));

    // Check not leader should fail, all data should be removed.
    must_get_none(&engine3, b"k1");
    let state_key = keys::region_state_key(r1);
    let state: RegionLocalState = engine3
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);
}
<commit_msg>Fix test `test_server_stale_peer_out_of_region` (#9030)<commit_after>// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.

//! A module contains test cases of stale peers gc.

use std::sync::Arc;
use std::thread;
use std::time::*;

use kvproto::raft_serverpb::{PeerState, RegionLocalState};
use raft::eraftpb::MessageType;

use engine_rocks::Compat;
use engine_traits::Peekable;
use engine_traits::CF_RAFT;
use test_raftstore::*;

/// A helper function for testing the behaviour of the gc of stale peer
/// which is out of region.
/// If a peer detects the leader is missing for a specified long time,
/// it should consider itself as a stale peer which is removed from the region.
/// This test case covers the following scenario:
/// At first, there are three peer A, B, C in the cluster, and A is leader.
/// Peer B gets down. And then A adds D, E, F into the cluster.
/// Peer D becomes leader of the new cluster, and then removes peer A, B, C.
/// After all these peer in and out, now the cluster has peer D, E, F.
/// If peer B goes up at this moment, it still thinks it is one of the cluster
/// and has peers A, C. However, it could not reach A, C since they are removed from
/// the cluster or probably destroyed.
/// Meantime, D, E, F would not reach B, Since it's not in the cluster anymore.
/// In this case, Peer B would notice that the leader is missing for a long time,
/// and it would check with pd to confirm whether it's still a member of the cluster.
/// If not, it should destroy itself as a stale peer which is removed out already.
fn test_stale_peer_out_of_region<T: Simulator>(cluster: &mut Cluster<T>) {
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    pd_client.must_add_peer(r1, new_learner_peer(2, 2));
    pd_client.must_add_peer(r1, new_peer(2, 2));
    pd_client.must_add_peer(r1, new_learner_peer(3, 3));
    pd_client.must_add_peer(r1, new_peer(3, 3));
    let (key, value) = (b"k1", b"v1");
    cluster.must_put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);
    must_get_equal(&engine_2, key, value);

    // Isolate peer 2 from rest of the cluster.
    cluster.add_send_filter(IsolationFilterFactory::new(2));

    // In case 2 is leader, it will fail to pass the healthy nodes check,
    // so remove isolated node first. Because 2 is isolated, so it can't remove itself.
    pd_client.must_remove_peer(r1, new_peer(2, 2));

    // Add peer [(4, 4), (5, 5), (6, 6)].
    pd_client.must_add_peer(r1, new_learner_peer(4, 4));
    pd_client.must_add_peer(r1, new_peer(4, 4));
    pd_client.must_add_peer(r1, new_learner_peer(5, 5));
    pd_client.must_add_peer(r1, new_peer(5, 5));
    pd_client.must_add_peer(r1, new_learner_peer(6, 6));
    pd_client.must_add_peer(r1, new_peer(6, 6));

    // Remove peer [(1, 1), (3, 3)].
    pd_client.must_remove_peer(r1, new_peer(1, 1));
    pd_client.must_remove_peer(r1, new_peer(3, 3));

    // Keep peer 2 isolated. Otherwise whether peer 3 is destroyed or not,
    // it will handle the stale raft message from peer 2 and cause peer 2 to
    // destroy itself earlier than this test case expects.

    // Wait for max_leader_missing_duration to time out.
    cluster.must_remove_region(2, r1);

    // Check whether this region is still functional properly.
    let (key2, value2) = (b"k2", b"v2");
    cluster.must_put(key2, value2);
    assert_eq!(cluster.get(key2), Some(value2.to_vec()));

    // Check whether peer(2, 2) and its data are destroyed.
    must_get_none(&engine_2, key);
    must_get_none(&engine_2, key2);
    let state_key = keys::region_state_key(1);
    let state: RegionLocalState = engine_2
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);
}

#[test]
fn test_node_stale_peer_out_of_region() {
    let count = 6;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_out_of_region(&mut cluster);
}

#[test]
fn test_server_stale_peer_out_of_region() {
    let count = 6;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_out_of_region(&mut cluster);
}

/// A help function for testing the behaviour of the gc of stale peer
/// which is out or region.
/// If a peer detects the leader is missing for a specified long time,
/// it should consider itself as a stale peer which is removed from the region.
/// This test case covers the following scenario:
/// A peer, B is initialized as a replicated peer without data after
/// receiving a single raft AE message. But then it goes through some process like
/// the case of `test_stale_peer_out_of_region`, it's removed out of the region
/// and wouldn't be contacted anymore.
/// In both cases, peer B would notice that the leader is missing for a long time,
/// and it's an initialized peer without any data. It would destroy itself as
/// as stale peer directly and should not impact other region data on the same store.
fn test_stale_peer_without_data<T: Simulator>(cluster: &mut Cluster<T>, right_derive: bool) {
    cluster.cfg.raft_store.right_derive_when_split = right_derive;

    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    cluster.must_put(b"k1", b"v1");
    cluster.must_put(b"k3", b"v3");
    let region = cluster.get_region(b"");
    pd_client.must_add_peer(r1, new_peer(2, 2));
    cluster.must_split(&region, b"k2");
    pd_client.must_add_peer(r1, new_peer(3, 3));

    let engine3 = cluster.get_engine(3);
    if right_derive {
        must_get_none(&engine3, b"k1");
        must_get_equal(&engine3, b"k3", b"v3");
    } else {
        must_get_equal(&engine3, b"k1", b"v1");
        must_get_none(&engine3, b"k3");
    }

    let new_region = if right_derive {
        cluster.get_region(b"k1")
    } else {
        cluster.get_region(b"k3")
    };
    let new_region_id = new_region.get_id();
    // Block peer (3, 4) at receiving snapshot, but not the heartbeat
    cluster.add_send_filter(CloneFilterFactory(
        RegionPacketFilter::new(new_region_id, 3).msg_type(MessageType::MsgSnapshot),
    ));

    pd_client.must_add_peer(new_region_id, new_peer(3, 4));

    // Wait for the heartbeat broadcasted from peer (1, 1000) to peer (3, 4).
    cluster.must_region_exist(new_region_id, 3);

    // And then isolate peer (3, 4) from peer (1, 1000).
    cluster.add_send_filter(IsolationFilterFactory::new(3));

    pd_client.must_remove_peer(new_region_id, new_peer(3, 4));

    cluster.must_remove_region(3, new_region_id);

    // There must be no data on store 3 belongs to new region
    if right_derive {
        must_get_none(&engine3, b"k1");
    } else {
        must_get_none(&engine3, b"k3");
    }

    // Check whether peer(3, 4) is destroyed.
    // Before peer 4 is destroyed, a tombstone mark will be written into the engine.
    // So we could check the tombstone mark to make sure peer 4 is destroyed.
    let state_key = keys::region_state_key(new_region_id);
    let state: RegionLocalState = engine3
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);

    // other region should not be affected.
    if right_derive {
        must_get_equal(&engine3, b"k3", b"v3");
    } else {
        must_get_equal(&engine3, b"k1", b"v1");
    }
}

#[test]
fn test_node_stale_peer_without_data_left_derive_when_split() {
    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, false);
}

#[test]
fn test_node_stale_peer_without_data_right_derive_when_split() {
    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, true);
}

#[test]
fn test_server_stale_peer_without_data_left_derive_when_split() {
    let count = 3;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, false);
}

#[test]
fn test_server_stale_peer_without_data_right_derive_when_split() {
    let count = 3;
    let mut cluster = new_server_cluster(0, count);
    test_stale_peer_without_data(&mut cluster, true);
}

/// A help function for testing the behaviour of the gc of stale learner
/// which is out or region.
#[test]
fn test_stale_learner() {
    let mut cluster = new_server_cluster(0, 4);
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer number check.
    pd_client.disable_default_operator();

    let r1 = cluster.run_conf_change();
    pd_client.must_add_peer(r1, new_peer(2, 2));
    pd_client.must_add_peer(r1, new_learner_peer(3, 3));
    cluster.must_put(b"k1", b"v1");
    let engine3 = cluster.get_engine(3);
    must_get_equal(&engine3, b"k1", b"v1");

    // And then isolate peer on store 3 from leader.
    cluster.add_send_filter(IsolationFilterFactory::new(3));

    // Add a new peer to increase the conf version.
    pd_client.must_add_peer(r1, new_peer(4, 4));

    // It should not be deleted.
    thread::sleep(Duration::from_secs(3));
    must_get_equal(&engine3, b"k1", b"v1");

    // Promote the learner
    pd_client.must_add_peer(r1, new_peer(3, 3));

    // It should not be deleted.
    thread::sleep(Duration::from_secs(3));
    must_get_equal(&engine3, b"k1", b"v1");

    // Delete the learner
    pd_client.must_remove_peer(r1, new_peer(3, 3));

    // Check not leader should fail, all data should be removed.
    must_get_none(&engine3, b"k1");
    let state_key = keys::region_state_key(r1);
    let state: RegionLocalState = engine3
        .c()
        .get_msg_cf(CF_RAFT, &state_key)
        .unwrap()
        .unwrap();
    assert_eq!(state.get_state(), PeerState::Tombstone);
}
<|endoftext|>


<commit_before>use crate::{manager::service::Service,
            util};
use habitat_core::{env,
                   package::PackageIdent,
                   service::ServiceGroup,
                   ChannelIdent};
use std::{self,
          time::Duration};
use tokio::{self,
            time};

// TODO (CM): Yes, the variable value should be "period" and not
// "frequency"... we need to fix that.
const PERIOD_BYPASS_CHECK_ENVVAR: &str = "HAB_UPDATE_STRATEGY_FREQUENCY_BYPASS_CHECK";

habitat_core::env_config_duration!(
    /// Represents how far apart checks for updates to individual services
    /// are, in milliseconds.
    PackageUpdateWorkerPeriod,
    // TODO (CM): Yes, the variable value should be "period" and not
    // "frequency"... we need to fix that.
    HAB_UPDATE_STRATEGY_FREQUENCY_MS => from_millis,
    PackageUpdateWorkerPeriod::MIN_ALLOWED);

impl PackageUpdateWorkerPeriod {
    const MIN_ALLOWED: Duration = Duration::from_secs(60);

    fn get() -> Duration {
        let val = PackageUpdateWorkerPeriod::configured_value().into();
        if val >= PackageUpdateWorkerPeriod::MIN_ALLOWED
           || env::var(PERIOD_BYPASS_CHECK_ENVVAR).is_ok()
        {
            val
        } else {
            PackageUpdateWorkerPeriod::MIN_ALLOWED
        }
    }
}

/// When `run`, a `PackageUpdateWorker` returns a future that continuously checks for a change in
/// version of the package being run by a service. If a change is detected, the package is installed
/// and its identifier returned.
pub struct PackageUpdateWorker {
    service_group: ServiceGroup,
    ident:         PackageIdent,
    // TODO (DM): This field should always be fully qualified. We need a
    // type to encapsulate that.
    full_ident:    PackageIdent,
    channel:       ChannelIdent,
    builder_url:   String,
}

impl From<&Service> for PackageUpdateWorker {
    fn from(service: &Service) -> Self {
        Self { service_group: service.service_group.clone(),
               ident:         service.spec_ident.clone(),
               full_ident:    service.pkg.ident.clone(),
               channel:       service.channel.clone(),
               builder_url:   service.bldr_url.clone(), }
    }
}

impl PackageUpdateWorker {
    /// The package update worker future.
    ///
    /// If no `install_ident` is specified the service ident is used when searching for the package.
    /// If an `install_ident` is specified, it is used when searching for a package. If a fully
    /// qualified package ident is used, the future will only resolve when that exact package is
    /// found.
    // TODO (DM): The returned package ident should always be fully qualified. We need a type to
    // encapsulate that.
    pub async fn run(&self, install_ident: Option<PackageIdent>) -> PackageIdent {
        let install_ident = install_ident.unwrap_or_else(|| self.ident.clone());
        let install_source = install_ident.clone().into();
        loop {
            // TODO (DM): The entire behavior of the updater depends on this `install` function. It
            // is what the old updater used, but its exact behavior should be tested/documented.
            match util::pkg::install_no_ui(&self.builder_url, &install_source, &self.channel).await
            {
                Ok(package) => {
                    // Only allow updates never rollbacks.
                    if package.ident > install_ident {
                        debug!("Service updater found change to '{}' from channel '{}' for \
                                service group '{}' replacing '{}'",
                               self.ident, self.channel, self.service_group, self.full_ident);
                        break package.ident;
                    } else {
                        trace!("Service updater did not find changes to '{}' from channel '{}' \
                                for service group '{}'",
                               self.ident,
                               self.channel,
                               self.service_group)
                    }
                }
                Err(err) => {
                    warn!("Service updater failed to install '{}' from channel '{}' for service \
                           group '{}', err: {}",
                          self.ident, self.channel, self.service_group, err)
                }
            }
            time::delay_for(PackageUpdateWorkerPeriod::get()).await;
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use habitat_core::locked_env_var;

    #[test]
    fn worker_period_default_is_equal_to_minimum_allowed_value() {
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::MIN_ALLOWED);
    }

    locked_env_var!(HAB_UPDATE_STRATEGY_FREQUENCY_MS, lock_period_var);
    locked_env_var!(HAB_UPDATE_STRATEGY_FREQUENCY_BYPASS_CHECK, lock_bypass_var);

    #[test]
    fn worker_period_must_be_positive() {
        use std::str::FromStr as _;
        assert!(PackageUpdateWorkerPeriod::from_str("-123").is_err());
        assert!(PackageUpdateWorkerPeriod::from_str("0").is_ok());
        assert!(PackageUpdateWorkerPeriod::from_str("5").is_ok());
    }

    #[test]
    fn worker_period_must_be_bypassed_by_non_empty_value() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("123");
        bypass.set(""); // empty string isn't allowed

        assert_ne!(PackageUpdateWorkerPeriod::get(), Duration::from_millis(123));
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_defaults_properly() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.unset();
        bypass.unset();

        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_can_be_overridden_by_env_var() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("120000");
        bypass.unset();
        let expected_period: PackageUpdateWorkerPeriod =
            PackageUpdateWorkerPeriod(Duration::from_millis(120_000));
        assert!(expected_period.0 >= PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(expected_period.0, PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_cannot_be_overridden_to_a_very_small_value_by_default() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("1"); // This is TOO low
        bypass.unset();
        assert!(Duration::from_millis(1) < PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_cannot_be_overridden_by_a_non_number() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("this is not a number");
        bypass.unset();
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_can_be_overridden_by_a_small_value_with_bypass_var() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("5000");
        bypass.set("1");
        let expected_period: PackageUpdateWorkerPeriod =
            PackageUpdateWorkerPeriod(Duration::from_millis(5000));
        assert!(expected_period.0 < PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(expected_period.0, PackageUpdateWorkerPeriod::get());
    }
}
<commit_msg>Fix idents in package update worker<commit_after>use crate::{manager::service::Service,
            util};
use habitat_core::{env,
                   package::PackageIdent,
                   service::ServiceGroup,
                   ChannelIdent};
use std::{self,
          time::Duration};
use tokio::{self,
            time};

// TODO (CM): Yes, the variable value should be "period" and not
// "frequency"... we need to fix that.
const PERIOD_BYPASS_CHECK_ENVVAR: &str = "HAB_UPDATE_STRATEGY_FREQUENCY_BYPASS_CHECK";

habitat_core::env_config_duration!(
    /// Represents how far apart checks for updates to individual services
    /// are, in milliseconds.
    PackageUpdateWorkerPeriod,
    // TODO (CM): Yes, the variable value should be "period" and not
    // "frequency"... we need to fix that.
    HAB_UPDATE_STRATEGY_FREQUENCY_MS => from_millis,
    PackageUpdateWorkerPeriod::MIN_ALLOWED);

impl PackageUpdateWorkerPeriod {
    const MIN_ALLOWED: Duration = Duration::from_secs(60);

    fn get() -> Duration {
        let val = PackageUpdateWorkerPeriod::configured_value().into();
        if val >= PackageUpdateWorkerPeriod::MIN_ALLOWED
           || env::var(PERIOD_BYPASS_CHECK_ENVVAR).is_ok()
        {
            val
        } else {
            PackageUpdateWorkerPeriod::MIN_ALLOWED
        }
    }
}

/// When `run`, a `PackageUpdateWorker` returns a future that continuously checks for a change in
/// version of the package being run by a service. If a change is detected, the package is installed
/// and its identifier returned.
pub struct PackageUpdateWorker {
    service_group: ServiceGroup,
    ident:         PackageIdent,
    // TODO (DM): This field should always be fully qualified. We need a
    // type to encapsulate that.
    full_ident:    PackageIdent,
    channel:       ChannelIdent,
    builder_url:   String,
}

impl From<&Service> for PackageUpdateWorker {
    fn from(service: &Service) -> Self {
        Self { service_group: service.service_group.clone(),
               ident:         service.spec_ident.clone(),
               full_ident:    service.pkg.ident.clone(),
               channel:       service.channel.clone(),
               builder_url:   service.bldr_url.clone(), }
    }
}

impl PackageUpdateWorker {
    /// The package update worker future.
    ///
    /// If no `install_ident` is specified the service ident is used when searching for the package.
    /// If an `install_ident` is specified, it is used when searching for a package. If a fully
    /// qualified package ident is used, the future will only resolve when that exact package is
    /// found.
    // TODO (DM): The returned package ident should always be fully qualified. We need a type to
    // encapsulate that.
    pub async fn run(&self, install_ident: Option<PackageIdent>) -> PackageIdent {
        let install_ident = install_ident.unwrap_or_else(|| self.ident.clone());
        let install_source = install_ident.clone().into();
        loop {
            // TODO (DM): The entire behavior of the updater depends on this `install` function. It
            // is what the old updater used, but its exact behavior should be tested/documented.
            match util::pkg::install_no_ui(&self.builder_url, &install_source, &self.channel).await
            {
                Ok(package) => {
                    // Only allow updates never rollbacks.
                    if package.ident > self.full_ident {
                        debug!("'{}' service updater found change from '{}' to '{}' for '{}' in \
                                channel '{}'",
                               self.service_group,
                               self.full_ident,
                               package.ident,
                               install_ident,
                               self.channel,);
                        break package.ident;
                    } else {
                        trace!("'{}' service updater did not find change from '{}' for '{}' in \
                                channel '{}'",
                               self.service_group,
                               self.full_ident,
                               install_ident,
                               self.channel,)
                    }
                }
                Err(err) => {
                    warn!("Service updater failed to install '{}' from channel '{}' for service \
                           group '{}', err: {}",
                          self.ident, self.channel, self.service_group, err)
                }
            }
            time::delay_for(PackageUpdateWorkerPeriod::get()).await;
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use habitat_core::locked_env_var;

    #[test]
    fn worker_period_default_is_equal_to_minimum_allowed_value() {
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::MIN_ALLOWED);
    }

    locked_env_var!(HAB_UPDATE_STRATEGY_FREQUENCY_MS, lock_period_var);
    locked_env_var!(HAB_UPDATE_STRATEGY_FREQUENCY_BYPASS_CHECK, lock_bypass_var);

    #[test]
    fn worker_period_must_be_positive() {
        use std::str::FromStr as _;
        assert!(PackageUpdateWorkerPeriod::from_str("-123").is_err());
        assert!(PackageUpdateWorkerPeriod::from_str("0").is_ok());
        assert!(PackageUpdateWorkerPeriod::from_str("5").is_ok());
    }

    #[test]
    fn worker_period_must_be_bypassed_by_non_empty_value() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("123");
        bypass.set(""); // empty string isn't allowed

        assert_ne!(PackageUpdateWorkerPeriod::get(), Duration::from_millis(123));
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_defaults_properly() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.unset();
        bypass.unset();

        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_can_be_overridden_by_env_var() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("120000");
        bypass.unset();
        let expected_period: PackageUpdateWorkerPeriod =
            PackageUpdateWorkerPeriod(Duration::from_millis(120_000));
        assert!(expected_period.0 >= PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(expected_period.0, PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_cannot_be_overridden_to_a_very_small_value_by_default() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("1"); // This is TOO low
        bypass.unset();
        assert!(Duration::from_millis(1) < PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_cannot_be_overridden_by_a_non_number() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("this is not a number");
        bypass.unset();
        assert_eq!(PackageUpdateWorkerPeriod::default().0,
                   PackageUpdateWorkerPeriod::get());
    }

    #[test]
    fn worker_period_can_be_overridden_by_a_small_value_with_bypass_var() {
        let period = lock_period_var();
        let bypass = lock_bypass_var();

        period.set("5000");
        bypass.set("1");
        let expected_period: PackageUpdateWorkerPeriod =
            PackageUpdateWorkerPeriod(Duration::from_millis(5000));
        assert!(expected_period.0 < PackageUpdateWorkerPeriod::MIN_ALLOWED);
        assert_eq!(expected_period.0, PackageUpdateWorkerPeriod::get());
    }
}
<|endoftext|>


<commit_before>use super::*;
use crate::{ssz_encode, Encodable};

impl CachedTreeHash<u64> for u64 {
    fn item_type() -> ItemType {
        ItemType::Basic
    }

    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
        Ok(TreeHashCache::from_bytes(merkleize(ssz_encode(self)))?)
    }

    fn num_bytes(&self) -> usize {
        8
    }

    fn offsets(&self) -> Result<Vec<usize>, Error> {
        Err(Error::ShouldNotProduceOffsetHandler)
    }

    fn num_child_nodes(&self) -> usize {
        0
    }

    fn packed_encoding(&self) -> Vec<u8> {
        ssz_encode(self)
    }

    fn packing_factor() -> usize {
        32 / 8
    }

    fn cached_hash_tree_root(
        &self,
        other: &Self,
        cache: &mut TreeHashCache,
        chunk: usize,
    ) -> Result<usize, Error> {
        if self != other {
            let leaf = merkleize(ssz_encode(self));
            cache.modify_chunk(chunk, &leaf)?;
        }

        Ok(chunk + 1)
    }
}

impl<T> CachedTreeHash<Vec<T>> for Vec<T>
where
    T: CachedTreeHash<T>,
{
    fn item_type() -> ItemType {
        ItemType::List
    }

    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
        match T::item_type() {
            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self))),
            ItemType::Composite | ItemType::List => {
                let subtrees = self
                    .iter()
                    .map(|item| TreeHashCache::new(item))
                    .collect::<Result<Vec<TreeHashCache>, _>>()?;

                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
            }
        }
    }

    fn offsets(&self) -> Result<Vec<usize>, Error> {
        let offsets = match T::item_type() {
            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
            ItemType::Composite | ItemType::List => {
                let mut offsets = vec![];

                for item in self {
                    offsets.push(OffsetHandler::new(item, 0)?.total_nodes())
                }

                offsets
            }
        };

        Ok(offsets)
    }

    fn num_child_nodes(&self) -> usize {
        // TODO
        42
    }

    fn num_bytes(&self) -> usize {
        self.iter().fold(0, |acc, item| acc + item.num_bytes())
    }

    fn packed_encoding(&self) -> Vec<u8> {
        panic!("List should never be packed")
    }

    fn packing_factor() -> usize {
        1
    }

    fn cached_hash_tree_root(
        &self,
        other: &Vec<T>,
        cache: &mut TreeHashCache,
        chunk: usize,
    ) -> Result<usize, Error> {
        let offset_handler = OffsetHandler::new(self, chunk)?;

        if self.len().next_power_of_two() != other.len().next_power_of_two() {
            panic!("not implemented: vary between power-of-two boundary");
        }

        match T::item_type() {
            ItemType::Basic => {
                let leaves = get_packed_leaves(self);

                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
                        cache.maybe_update_chunk(*chunk, latest)?;
                    }
                }
                let first_leaf_chunk = offset_handler.first_leaf_node()?;
                cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves);
            }
            ItemType::Composite | ItemType::List => {
                let mut i = offset_handler.num_leaf_nodes;
                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                    i -= 1;
                    match (other.get(i), self.get(i)) {
                        // The item existed in the previous list and exsits in the current list.
                        (Some(old), Some(new)) => {
                            new.cached_hash_tree_root(old, cache, start_chunk)?;
                        }
                        // The item existed in the previous list but does not exist in this list.
                        //
                        // I.e., the list has been shortened.
                        (Some(old), None) => {
                            // Splice out the entire tree of the removed node, replacing it with a
                            // single padding node.
                            let end_chunk = OffsetHandler::new(old, start_chunk)?.next_node();
                            cache.chunk_splice(start_chunk..end_chunk, vec![0; HASHSIZE]);
                        }
                        // The item existed in the previous list but does exist in this list.
                        //
                        // I.e., the list has been lengthened.
                        (None, Some(new)) => {
                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
                            cache.chunk_splice(start_chunk..start_chunk + 1, bytes);
                        }
                        // The item didn't exist in the old list and doesn't exist in the new list,
                        // nothing to do.
                        (None, None) => {}
                    };
                }
            }
        }

        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
            if cache.either_modified(children)? {
                dbg!(parent);
                dbg!(children);
                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
            }
        }

        Ok(offset_handler.next_node())
    }
}

fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
where
    T: CachedTreeHash<T>,
{
    let num_packed_bytes = vec.num_bytes();
    let num_leaves = num_sanitized_leaves(num_packed_bytes);

    let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);

    for item in vec {
        packed.append(&mut item.packed_encoding());
    }

    sanitise_bytes(packed)
}
<commit_msg>Start implementing grow merkle fn<commit_after>use super::*;
use crate::{ssz_encode, Encodable};

impl CachedTreeHash<u64> for u64 {
    fn item_type() -> ItemType {
        ItemType::Basic
    }

    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
        Ok(TreeHashCache::from_bytes(merkleize(ssz_encode(self)))?)
    }

    fn num_bytes(&self) -> usize {
        8
    }

    fn offsets(&self) -> Result<Vec<usize>, Error> {
        Err(Error::ShouldNotProduceOffsetHandler)
    }

    fn num_child_nodes(&self) -> usize {
        0
    }

    fn packed_encoding(&self) -> Vec<u8> {
        ssz_encode(self)
    }

    fn packing_factor() -> usize {
        32 / 8
    }

    fn cached_hash_tree_root(
        &self,
        other: &Self,
        cache: &mut TreeHashCache,
        chunk: usize,
    ) -> Result<usize, Error> {
        if self != other {
            let leaf = merkleize(ssz_encode(self));
            cache.modify_chunk(chunk, &leaf)?;
        }

        Ok(chunk + 1)
    }
}

impl<T> CachedTreeHash<Vec<T>> for Vec<T>
where
    T: CachedTreeHash<T>,
{
    fn item_type() -> ItemType {
        ItemType::List
    }

    fn build_tree_hash_cache(&self) -> Result<TreeHashCache, Error> {
        match T::item_type() {
            ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self))),
            ItemType::Composite | ItemType::List => {
                let subtrees = self
                    .iter()
                    .map(|item| TreeHashCache::new(item))
                    .collect::<Result<Vec<TreeHashCache>, _>>()?;

                TreeHashCache::from_leaves_and_subtrees(self, subtrees)
            }
        }
    }

    fn offsets(&self) -> Result<Vec<usize>, Error> {
        let offsets = match T::item_type() {
            ItemType::Basic => vec![1; self.len() / T::packing_factor()],
            ItemType::Composite | ItemType::List => {
                let mut offsets = vec![];

                for item in self {
                    offsets.push(OffsetHandler::new(item, 0)?.total_nodes())
                }

                offsets
            }
        };

        Ok(offsets)
    }

    fn num_child_nodes(&self) -> usize {
        // TODO
        42
    }

    fn num_bytes(&self) -> usize {
        self.iter().fold(0, |acc, item| acc + item.num_bytes())
    }

    fn packed_encoding(&self) -> Vec<u8> {
        panic!("List should never be packed")
    }

    fn packing_factor() -> usize {
        1
    }

    fn cached_hash_tree_root(
        &self,
        other: &Vec<T>,
        cache: &mut TreeHashCache,
        chunk: usize,
    ) -> Result<usize, Error> {
        let offset_handler = OffsetHandler::new(self, chunk)?;

        if self.len().next_power_of_two() != other.len().next_power_of_two() {
            panic!("not implemented: vary between power-of-two boundary");
        }

        match T::item_type() {
            ItemType::Basic => {
                let leaves = get_packed_leaves(self);

                for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() {
                    if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) {
                        cache.maybe_update_chunk(*chunk, latest)?;
                    }
                }
                let first_leaf_chunk = offset_handler.first_leaf_node()?;
                cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves);
            }
            ItemType::Composite | ItemType::List => {
                let mut i = offset_handler.num_leaf_nodes;
                for &start_chunk in offset_handler.iter_leaf_nodes().rev() {
                    i -= 1;
                    match (other.get(i), self.get(i)) {
                        // The item existed in the previous list and exsits in the current list.
                        (Some(old), Some(new)) => {
                            new.cached_hash_tree_root(old, cache, start_chunk)?;
                        }
                        // The item existed in the previous list but does not exist in this list.
                        //
                        // I.e., the list has been shortened.
                        (Some(old), None) => {
                            // Splice out the entire tree of the removed node, replacing it with a
                            // single padding node.
                            let end_chunk = OffsetHandler::new(old, start_chunk)?.next_node();
                            cache.chunk_splice(start_chunk..end_chunk, vec![0; HASHSIZE]);
                        }
                        // The item existed in the previous list but does exist in this list.
                        //
                        // I.e., the list has been lengthened.
                        (None, Some(new)) => {
                            let bytes: Vec<u8> = TreeHashCache::new(new)?.into();
                            cache.chunk_splice(start_chunk..start_chunk + 1, bytes);
                        }
                        // The item didn't exist in the old list and doesn't exist in the new list,
                        // nothing to do.
                        (None, None) => {}
                    };
                }
            }
        }

        for (&parent, children) in offset_handler.iter_internal_nodes().rev() {
            if cache.either_modified(children)? {
                dbg!(parent);
                dbg!(children);
                cache.modify_chunk(parent, &cache.hash_children(children)?)?;
            }
        }

        Ok(offset_handler.next_node())
    }
}

/// New vec is bigger than old vec.
fn grow_merkle_cache(cache: Vec<u8>, to: usize) -> Vec<u8> {
    let new = Vec::with_capacity(to * HASHSIZE);

    let i = cache.len() / HASHSIZE;
    let j = to;

    assert_eq!(i.next_power_of_two(), i);
    assert_eq!(j.next_power_of_two(), j);

    while i > 0 {

    }

    new
}

fn get_packed_leaves<T>(vec: &Vec<T>) -> Vec<u8>
where
    T: CachedTreeHash<T>,
{
    let num_packed_bytes = vec.num_bytes();
    let num_leaves = num_sanitized_leaves(num_packed_bytes);

    let mut packed = Vec::with_capacity(num_leaves * HASHSIZE);

    for item in vec {
        packed.append(&mut item.packed_encoding());
    }

    sanitise_bytes(packed)
}
<|endoftext|>


<commit_before>extern crate system;

use system::syscall::sys_iopl;

use pci::{Pci, PciBar, PciClass};

mod pci;

fn enumerate_pci() {
    println!("PCI BS/DV/FN VEND:DEVI CL.SC.IN.RV");

    let pci = Pci::new();
    for bus in pci.buses() {
        for dev in bus.devs() {
            for func in dev.funcs() {
                if let Some(header) = func.header() {
                    println!("PCI {:>02X}/{:>02X}/{:>02X} {:>04X}:{:>04X} {:>02X}.{:>02X}.{:>02X}.{:>02X} {:?}",
                            bus.num, dev.num, func.num,
                            header.vendor_id, header.device_id,
                            header.class, header.subclass, header.interface, header.revision,
                            PciClass::from(header.class));
                    for i in 0..header.bars.len() {
                        match PciBar::from(header.bars[i]) {
                            PciBar::None => (),
                            PciBar::Memory(address) => println!("    BAR {} {:>08X}", i, address),
                            PciBar::Port(address) => println!("    BAR {} {:>04X}", i, address)
                        }
                    }
                }
            }
        }
    }
}

fn main() {
    unsafe { sys_iopl(3).unwrap() };

    enumerate_pci();
}
<commit_msg>Indicate which driver should be loaded<commit_after>extern crate system;

use system::syscall::sys_iopl;

use pci::{Pci, PciBar, PciClass};

mod pci;

fn enumerate_pci() {
    println!("PCI BS/DV/FN VEND:DEVI CL.SC.IN.RV");

    let pci = Pci::new();
    for bus in pci.buses() {
        for dev in bus.devs() {
            for func in dev.funcs() {
                if let Some(header) = func.header() {
                    println!("PCI {:>02X}/{:>02X}/{:>02X} {:>04X}:{:>04X} {:>02X}.{:>02X}.{:>02X}.{:>02X} {:?}",
                            bus.num, dev.num, func.num,
                            header.vendor_id, header.device_id,
                            header.class, header.subclass, header.interface, header.revision,
                            PciClass::from(header.class));

                    for i in 0..header.bars.len() {
                        match PciBar::from(header.bars[i]) {
                            PciBar::None => (),
                            PciBar::Memory(address) => println!("    BAR {} {:>08X}", i, address),
                            PciBar::Port(address) => println!("    BAR {} {:>04X}", i, address)
                        }
                    }

                    match PciClass::from(header.class) {
                        PciClass::Storage => match header.subclass {
                            0x01 => {
                                println!("    + IDE");
                            },
                            0x06 => {
                                println!("    + SATA");
                            },
                            _ => ()
                        },
                        PciClass::SerialBus => match header.subclass {
                            0x03 => match header.interface {
                                0x00 => {
                                    println!("    + UHCI");
                                },
                                0x10 => {
                                    println!("    + OHCI");
                                },
                                0x20 => {
                                    println!("    + EHCI");
                                },
                                0x30 => {
                                    println!("    + XHCI");
                                },
                                _ => ()
                            },
                            _ => ()
                        },
                        _ => ()
                    }
                }
            }
        }
    }
}

fn main() {
    unsafe { sys_iopl(3).unwrap() };

    enumerate_pci();
}
<|endoftext|>


<commit_before>use std::collections::VecDeque;

#[macro_use]
extern crate strider;
use strider::SliceRing;
use strider::SliceRingImpl;

#[test]
fn test_slice_ring_unoptimized() {
    test_slice_ring!(VecDeque::<i32>::new());
}
#[test]
fn test_slice_ring_optimized() {
    test_slice_ring!(SliceRingImpl::<i32>::new());
}
<commit_msg>add test that exposes bug in SliceRingImpl<commit_after>use std::collections::VecDeque;

use std::io::{Cursor, Read, Write};

#[macro_use]
extern crate strider;
use strider::SliceRing;
use strider::SliceRingImpl;

macro_rules! test_string_windowing {
    ($new:expr) => {{
        const WINDOW_SIZE: usize = 8;
        const STEP_SIZE: usize = 2;
        let mut input = Cursor::new("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
        let mut output = Cursor::new(Vec::<u8>::new());
        let mut ring = $new;
        let mut input_buffer: &mut [u8] = &mut [0; 4];
        let mut window_buffer: &mut [u8] = &mut [0; WINDOW_SIZE];

        loop {
            let input_count = input.read(input_buffer).unwrap();
            if input_count == 0 { break; }

            ring.push_many_back(&input_buffer[..input_count]);
            // read as long as enough samples are present (remain) in ring
            while WINDOW_SIZE <= ring.len() {
                ring.read_many_front(window_buffer);
                output.write(window_buffer).unwrap();
                // step
                ring.drop_many_front(STEP_SIZE);
            }
        }
        let actual = String::from_utf8(output.into_inner()).unwrap();
        let mut expected = String::new();
        expected.push_str("ABCDEFGH");
        expected.push_str("CDEFGHIJ");
        expected.push_str("EFGHIJKL");
        expected.push_str("GHIJKLMN");
        expected.push_str("IJKLMNOP");
        expected.push_str("KLMNOPQR");
        expected.push_str("MNOPQRST");
        expected.push_str("OPQRSTUV");
        expected.push_str("QRSTUVWX");
        expected.push_str("STUVWXYZ");
        assert_eq!(actual, expected);
    }}
}
#[test]
fn test_test_string_windowing_deque() {
    test_string_windowing!(VecDeque::<u8>::new());
}
#[test]
fn test_string_windowing_optimized() {
    test_string_windowing!(SliceRingImpl::<u8>::new());
}

#[test]
fn test_slice_ring_deque() {
    test_slice_ring!(VecDeque::<i32>::new());
}
#[test]
fn test_slice_ring_optimized() {
    test_slice_ring!(SliceRingImpl::<i32>::new());
}
<|endoftext|>