<commit_before>use std::borrow::Cow::{Owned, Borrowed};

use pulldown_cmark as cmark;
use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
use slug::slugify;
use syntect::easy::HighlightLines;
use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};

use errors::Result;
use utils::site::resolve_internal_link;
use highlighting::{get_highlighter, THEME_SET};
use link_checker::check_url;

use table_of_contents::{TempHeader, Header, make_table_of_contents};
use context::RenderContext;

const CONTINUE_READING: &str = "

\n";

pub struct Rendered {
    pub body: String,
    pub summary_len: Option<usize>,
    pub toc: Vec
}

// We might have cases where the slug is already present in our list of anchor
// for example an article could have several titles named Example
// We add a counter after the slug if the slug is already present, which
// means we will have example, example-1, example-2 etc
fn find_anchor(anchors: &[String], name: String, level: u8) -> String {
    if level == 0 && !anchors.contains(&name) {
        return name.to_string();
    }

    let new_anchor = format!("{}-{}", name, level + 1);
    if !anchors.contains(&new_anchor) {
        return new_anchor;
    }

    find_anchor(anchors, name, level + 1)
}

fn is_colocated_asset_link(link: &str) -> bool {
    !link.contains("/")  // http://, ftp://, ../ etc
        && !link.starts_with("mailto:")
}

pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Rendered> {
    // the rendered html
    let mut html = String::with_capacity(content.len());
    // Set while parsing
    let mut error = None;

    let mut highlighter: Option<HighlightLines> = None;
    // If we get text in header, we need to insert the id and a anchor
    let mut in_header = false;
    // pulldown_cmark can send several text events for a title if there are markdown
    // specific characters like `!` in them. We only want to insert the anchor the first time
    let mut header_created = false;
    let mut anchors: Vec<String> = vec![];

    let mut headers = vec![];
    // Defaults to a 0 level so not a real header
    // It should be an Option ideally but not worth the hassle to update
    let mut temp_header = TempHeader::default();

    let mut opts = Options::empty();
    let mut has_summary = false;
    opts.insert(OPTION_ENABLE_TABLES);
    opts.insert(OPTION_ENABLE_FOOTNOTES);

    {
        let parser = Parser::new_ext(content, opts).map(|event| {
            match event {
                Event::Text(text) => {
                    // Header first
                    if in_header {
                        if header_created {
                            temp_header.push(&text);
                            return Event::Html(Owned(String::new()));
                        }
                        let id = find_anchor(&anchors, slugify(&text), 0);
                        anchors.push(id.clone());
                        // update the header and add it to the list
                        temp_header.permalink = format!("{}#{}", context.current_page_permalink, id);
                        temp_header.id = id;
                        // += as we might have some  or other things already there
                        temp_header.title += &text;
                        header_created = true;
                        return Event::Html(Owned(String::new()));
                    }

                    // if we are in the middle of a code block
                    if let Some(ref mut highlighter) = highlighter {
                        let highlighted = &highlighter.highlight(&text);
                        let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes);
                        return Event::Html(Owned(html));
                    }

                    // Business as usual
                    Event::Text(text)
                }
                Event::Start(Tag::CodeBlock(ref info)) => {
                    if !context.config.highlight_code {
                        return Event::Html(Owned("
".to_string()));
                    }

                    let theme = &THEME_SET.themes[&context.config.highlight_theme];
                    match get_highlighter(&theme, info, context.base_path, &context.config.extra_syntaxes) {
                        Ok(h) => highlighter = Some(h),
                        Err(err) => {
                            error = Some(format!("Could not load syntax: {}", err).into());
                            return Event::Html(Owned(String::new()));
                        }
                    }
                    let snippet = start_coloured_html_snippet(theme);
                    Event::Html(Owned(snippet))
                }
                Event::End(Tag::CodeBlock(_)) => {
                    if !context.config.highlight_code {
                        return Event::Html(Owned("
\n".to_string()));
                    }
                    // reset highlight and close the code block
                    highlighter = None;
                    Event::Html(Owned("
".to_string()))
                }
                Event::Start(Tag::Image(src, title)) => {
                    if is_colocated_asset_link(&src) {
                        return Event::Start(
                            Tag::Image(
                                Owned(format!("{}{}", context.current_page_permalink, src)),
                                title,
                            )
                        );
                    }

                    Event::Start(Tag::Image(src, title))
                }
                Event::Start(Tag::Link(link, title)) => {
                    // A few situations here:
                    // - it could be a relative link (starting with `./`)
                    // - it could be a link to a co-located asset
                    // - it could be a normal link
                    // - any of those can be in a header or not: if it's in a header
                    //   we need to append to a string
                    let fixed_link = if link.starts_with("./") {
                        match resolve_internal_link(&link, context.permalinks) {
                            Ok(url) => url,
                            Err(_) => {
                                error = Some(format!("Relative link {} not found.", link).into());
                                return Event::Html(Owned(String::new()));
                            }
                        }
                    } else if is_colocated_asset_link(&link) {
                        format!("{}{}", context.current_page_permalink, link)
                    } else {
                        if context.config.check_external_links && !link.starts_with('#') {
                            let res = check_url(&link);
                            if res.is_valid() {
                                link.to_string()
                            } else {
                                error = Some(
                                    format!("Link {} is not valid: {}", link, res.message()).into()
                                );
                                String::new()
                            }
                        } else {
                            link.to_string()
                        }
                    };

                    if in_header {
                        let html = if title.is_empty() {
                            format!("", fixed_link)
                        } else {
                            format!("", fixed_link, title)
                        };
                        temp_header.push(&html);
                        return Event::Html(Owned(String::new()));
                    }

                    Event::Start(Tag::Link(Owned(fixed_link), title))
                }
                Event::End(Tag::Link(_, _)) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Owned(String::new()));
                    }
                    event
                }
                Event::Start(Tag::Code) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Owned(String::new()));
                    }
                    event
                }
                Event::End(Tag::Code) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Owned(String::new()));
                    }
                    event
                }
                Event::Start(Tag::Header(num)) => {
                    in_header = true;
                    temp_header = TempHeader::new(num);
                    Event::Html(Owned(String::new()))
                }
                Event::End(Tag::Header(_)) => {
                    // End of a header, reset all the things and return the stringified
                    // version of the header
                    in_header = false;
                    header_created = false;
                    let val = temp_header.to_string(context.tera, context.insert_anchor);
                    headers.push(temp_header.clone());
                    temp_header = TempHeader::default();
                    Event::Html(Owned(val))
                }
                Event::Html(ref markup) if markup.contains("") => {
                    has_summary = true;
                    Event::Html(Borrowed(CONTINUE_READING))
                }
                _ => event,
            }
        });

        cmark::html::push_html(&mut html, parser);
    }

    if let Some(e) = error {
        return Err(e)
    } else {
        html = html.replace("

", "").replace("

", "
");
        Ok(Rendered {
            summary_len: if has_summary { html.find(CONTINUE_READING) } else { None },
            body: html,
            toc: make_table_of_contents(&headers)
        })
    }
}
<commit_msg>Markdown parsing: prefer Borrowed over Owned where possible<commit_after>use std::borrow::Cow::{Owned, Borrowed};

use pulldown_cmark as cmark;
use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
use slug::slugify;
use syntect::easy::HighlightLines;
use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};

use errors::Result;
use utils::site::resolve_internal_link;
use highlighting::{get_highlighter, THEME_SET};
use link_checker::check_url;

use table_of_contents::{TempHeader, Header, make_table_of_contents};
use context::RenderContext;

const CONTINUE_READING: &str = "

\n";

pub struct Rendered {
    pub body: String,
    pub summary_len: Option<usize>,
    pub toc: Vec
}

// We might have cases where the slug is already present in our list of anchor
// for example an article could have several titles named Example
// We add a counter after the slug if the slug is already present, which
// means we will have example, example-1, example-2 etc
fn find_anchor(anchors: &[String], name: String, level: u8) -> String {
    if level == 0 && !anchors.contains(&name) {
        return name.to_string();
    }

    let new_anchor = format!("{}-{}", name, level + 1);
    if !anchors.contains(&new_anchor) {
        return new_anchor;
    }

    find_anchor(anchors, name, level + 1)
}

fn is_colocated_asset_link(link: &str) -> bool {
    !link.contains("/")  // http://, ftp://, ../ etc
        && !link.starts_with("mailto:")
}

pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Rendered> {
    // the rendered html
    let mut html = String::with_capacity(content.len());
    // Set while parsing
    let mut error = None;

    let mut highlighter: Option<HighlightLines> = None;
    // If we get text in header, we need to insert the id and a anchor
    let mut in_header = false;
    // pulldown_cmark can send several text events for a title if there are markdown
    // specific characters like `!` in them. We only want to insert the anchor the first time
    let mut header_created = false;
    let mut anchors: Vec<String> = vec![];

    let mut headers = vec![];
    // Defaults to a 0 level so not a real header
    // It should be an Option ideally but not worth the hassle to update
    let mut temp_header = TempHeader::default();

    let mut opts = Options::empty();
    let mut has_summary = false;
    opts.insert(OPTION_ENABLE_TABLES);
    opts.insert(OPTION_ENABLE_FOOTNOTES);

    {
        let parser = Parser::new_ext(content, opts).map(|event| {
            match event {
                Event::Text(text) => {
                    // Header first
                    if in_header {
                        if header_created {
                            temp_header.push(&text);
                            return Event::Html(Borrowed(""));
                        }
                        let id = find_anchor(&anchors, slugify(&text), 0);
                        anchors.push(id.clone());
                        // update the header and add it to the list
                        temp_header.permalink = format!("{}#{}", context.current_page_permalink, id);
                        temp_header.id = id;
                        // += as we might have some  or other things already there
                        temp_header.title += &text;
                        header_created = true;
                        return Event::Html(Borrowed(""));
                    }

                    // if we are in the middle of a code block
                    if let Some(ref mut highlighter) = highlighter {
                        let highlighted = &highlighter.highlight(&text);
                        let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes);
                        return Event::Html(Owned(html));
                    }

                    // Business as usual
                    Event::Text(text)
                }
                Event::Start(Tag::CodeBlock(ref info)) => {
                    if !context.config.highlight_code {
                        return Event::Html(Borrowed("
"));
                    }

                    let theme = &THEME_SET.themes[&context.config.highlight_theme];
                    match get_highlighter(&theme, info, context.base_path, &context.config.extra_syntaxes) {
                        Ok(h) => highlighter = Some(h),
                        Err(err) => {
                            error = Some(format!("Could not load syntax: {}", err).into());
                            return Event::Html(Borrowed(""));
                        }
                    }
                    let snippet = start_coloured_html_snippet(theme);
                    Event::Html(Owned(snippet))
                }
                Event::End(Tag::CodeBlock(_)) => {
                    if !context.config.highlight_code {
                        return Event::Html(Borrowed("
\n"));
                    }
                    // reset highlight and close the code block
                    highlighter = None;
                    Event::Html(Borrowed("
"))
                }
                Event::Start(Tag::Image(src, title)) => {
                    if is_colocated_asset_link(&src) {
                        return Event::Start(
                            Tag::Image(
                                Owned(format!("{}{}", context.current_page_permalink, src)),
                                title,
                            )
                        );
                    }

                    Event::Start(Tag::Image(src, title))
                }
                Event::Start(Tag::Link(link, title)) => {
                    // A few situations here:
                    // - it could be a relative link (starting with `./`)
                    // - it could be a link to a co-located asset
                    // - it could be a normal link
                    // - any of those can be in a header or not: if it's in a header
                    //   we need to append to a string
                    let fixed_link = if link.starts_with("./") {
                        match resolve_internal_link(&link, context.permalinks) {
                            Ok(url) => url,
                            Err(_) => {
                                error = Some(format!("Relative link {} not found.", link).into());
                                return Event::Html(Borrowed(""));
                            }
                        }
                    } else if is_colocated_asset_link(&link) {
                        format!("{}{}", context.current_page_permalink, link)
                    } else {
                        if context.config.check_external_links && !link.starts_with('#') {
                            let res = check_url(&link);
                            if res.is_valid() {
                                link.to_string()
                            } else {
                                error = Some(
                                    format!("Link {} is not valid: {}", link, res.message()).into()
                                );
                                String::new()
                            }
                        } else {
                            link.to_string()
                        }
                    };

                    if in_header {
                        let html = if title.is_empty() {
                            format!("", fixed_link)
                        } else {
                            format!("", fixed_link, title)
                        };
                        temp_header.push(&html);
                        return Event::Html(Borrowed(""));
                    }

                    Event::Start(Tag::Link(Owned(fixed_link), title))
                }
                Event::End(Tag::Link(_, _)) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Borrowed(""));
                    }
                    event
                }
                Event::Start(Tag::Code) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Borrowed(""));
                    }
                    event
                }
                Event::End(Tag::Code) => {
                    if in_header {
                        temp_header.push("");
                        return Event::Html(Borrowed(""));
                    }
                    event
                }
                Event::Start(Tag::Header(num)) => {
                    in_header = true;
                    temp_header = TempHeader::new(num);
                    Event::Html(Borrowed(""))
                }
                Event::End(Tag::Header(_)) => {
                    // End of a header, reset all the things and return the stringified
                    // version of the header
                    in_header = false;
                    header_created = false;
                    let val = temp_header.to_string(context.tera, context.insert_anchor);
                    headers.push(temp_header.clone());
                    temp_header = TempHeader::default();
                    Event::Html(Owned(val))
                }
                Event::Html(ref markup) if markup.contains("") => {
                    has_summary = true;
                    Event::Html(Borrowed(CONTINUE_READING))
                }
                _ => event,
            }
        });

        cmark::html::push_html(&mut html, parser);
    }

    if let Some(e) = error {
        return Err(e)
    } else {
        html = html.replace("

", "").replace("

", "
");
        Ok(Rendered {
            summary_len: if has_summary { html.find(CONTINUE_READING) } else { None },
            body: html,
            toc: make_table_of_contents(&headers)
        })
    }
}
<|endoftext|>


<commit_before>#![cfg(target_os = "linux")]

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use sled::{Db, Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            slice_to_u32(&*v),
            lowest,
            "expected key {} to have value {}, instead it had value {} in db: {:?}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v),
            tree
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    tests::setup_logger();
    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 1000);
        value.append(&mut vec![0u8; additional_len]);

        tree.insert(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .page_consolidation_threshold(10)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        .io_buf_size(1_000)
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .page_consolidation_threshold(10)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        .io_buf_size(1_000)
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(5000)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<commit_msg>Fix logger import on crash test<commit_after>#![cfg(target_os = "linux")]

mod common;

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use sled::{Db, Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            slice_to_u32(&*v),
            lowest,
            "expected key {} to have value {}, instead it had value {} in db: {:?}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v),
            tree
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    common::setup_logger();

    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 1000);
        value.append(&mut vec![0u8; additional_len]);

        tree.insert(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .page_consolidation_threshold(10)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        .io_buf_size(1_000)
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .page_consolidation_threshold(10)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        .io_buf_size(1_000)
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(5000)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<|endoftext|>


<commit_before>//! Using MIDI, verify that the user is able to play a given exercise correctly.  The question
//! should be enough of a description to uniquely describe what should be played.  The answer
//! should be a parseable description of what needs to be played.
//!
//! The answer should be a JSON description of what to play, of one of the following types:
//!
//! { "type": "scale", "base": "C♯", "intervals": "WWHWWWH",
//!   "hands": 1, "octaves": 2, "style": "updown" }
//!
//! This describes a scale.  The base is the first note of the scale.  Sharps and flags can either
//! be "#" and "b" or Unicode characters "♯", and "♭".  The intervals describe the intervals
//! between each note.  Valid characters are 'H' for a half step, 'W' for a whole step, 'm' for a
//! minor third, 'M' for a major third, and '4' for a fourth.  It will be checked that the
//! intervals cover a full octabe.  Hands describes how many hands should be playing the exercise,
//! and octaves are how many octaves to play.  The currently only supported style is "updown",
//! which requires the scale be played from base to base, for the given number of octaves and then
//! back down.  Scales can be played starting in any octave.
//!
//! { "type": "voicing", "chords": [
//!    { "base": "D", "notes": [ "3", "7", "9", "5" },
//!    { "base": "G", "notes": [ "7", "3", "#9", "#5" },
//!    { "base": "C", "notes": [ "3", "7", "9", "5", "11" } ] }
//!
//! This describes a voicing exercise, which is a series of chords to be played.  Each chord has a
//! base, which may or may not actually be played.  The notes in the chord are described in musical
//! notation (again with text or Unicode sharp/flat indicators).  The chord will be built using the
//! next note of the given function.  The octave will be chosen so that the lowest non-root note
//! will be between F#-F surrounding middle C.
//!
//! For the notes, values can be chord positions.  It is also possible to have a note that is a
//! placeholder, as an empty string, which indicates that the following note should be moved up an
//! octave.

#![deny(missing_docs)]

extern crate editdistancewf;
extern crate portmidi;
extern crate timelearn;
#[macro_use] extern crate lazy_static;
extern crate serde;
#[macro_use] extern crate serde_derive;
#[macro_use] extern crate serde_json;

use portmidi::{PortMidi, InputPort};
use serde_json::{Value};
use std::error;
use std::io::{self, Write};
use std::result;
use std::thread;
use std::time::Duration;
pub use timelearn::{Problem, Status, User};

mod note;
use note::Note;

mod scales;
use scales::{Scale, ScaleSeq, Voicing};

/// Just box the errors for now.  TODO: Use a proper error type.
pub type Result<T> = result::Result<T, Box<error::Error + Send + Sync>>;

/// A learner based on playing the exercises on a MIDI device.
pub struct MidiLearn {
    input: InputPort,
}

impl MidiLearn {
    /// Construct a new MIDI learning, using the default MIDI port.  Invokes `f` with the
    /// MidiLearn.
    pub fn with_new<F, R>(f: F) -> Result<R>
        where F: FnOnce(&mut MidiLearn) -> R
    {
        let pm = PortMidi::new()?;
        let input = pm.default_input_port(4096)?;
        let mut learn = MidiLearn {
            input: input,
        };
        let result = f(&mut learn);
        Ok(result)
    }

}

impl User for MidiLearn {
    /// Practice a single exercise.  Waits for the user to play the exercise, and then returns a
    /// status indicating how well it was played.
    fn single(&mut self, word: &Problem) -> Result<Status> {
        let st1 = self.single_once(word)?;

        let mut stn = st1;
        loop {
            match stn {
                // If good, return the initial status.
                Status::Continue(4) => return Ok(st1),
                Status::Stopped => return Ok(Status::Stopped),
                _ => (),
            }
            println!("** Mistakes made, please play again **");
            stn = self.single_once(word)?;
        }
    }
}

impl MidiLearn {
    /// Ask the user once to play.
    fn single_once(&mut self, word: &Problem) -> Result<Status> {
        println!("Play: {}", word.question);

        let json: Value = serde_json::from_str(&word.answer)?;
        // println!("Json: {:?}, {:?}", json, json["typr"]);
        if is_type(&json, "scale") {
            let scale: Scale = serde_json::from_value(json)?;

            let mut seq = ScaleSeq::from_scale(&scale)?;
            // println!("scale: {:?}", seq);

            self.drain()?;
            let user = self.record_scale()?;
            // println!("Played: {:?}", user);
            if seq.adjust_octave(&user) {
                // println!("First note good: {:?}", seq);
                let diff_count = seq.differences(&user);
                println!("There are {} differences", diff_count);
                if diff_count <= 3 {
                    Ok(Status::Continue(4 - diff_count as u8))
                } else {
                    Ok(Status::Continue(1))
                }
            } else {
                println!("First note mismatch, stopping");
                Ok(Status::Stopped)
            }
        } else if is_type(&json, "voicing") {
            let chords: Voicing = serde_json::from_value(json)?;

            let seq = ScaleSeq::from_voicing(&chords)?;
            println!("chords: {:?}", seq);
            self.drain()?;
            let user = self.record_scale()?;
            println!("user: {:?}", user);
            if user[0].len() == 1 {
                println!("Single note, stopping");
                Ok(Status::Stopped)
            } else {
                let diff_count = seq.differences(&user);
                println!("There are {} differences", diff_count);
                if diff_count == 0 {
                    Ok(Status::Continue(4))
                } else {
                    Ok(Status::Continue(1))
                }
            }
        } else {
            return Err(format!("Invalid type: {:?}", json["type"].as_str()).into());
        }
    }
}

impl MidiLearn {
    /// Drain any queued midi events.
    fn drain(&mut self) -> Result<()> {
        loop {
            match self.input.read()? {
                Some(_) => (),
                None => break,
            }
        }
        Ok(())
    }

    /// Record the user playing a scale.  Consider the scale done after a small pause of no
    /// playing.
    fn record_scale(&mut self) -> Result<Vec<Vec<Note>>> {
        let mut notes: Vec<Vec<Note>> = vec![];
        let mut idle_count = 0;
        let mut last_time = 0;
        loop {
            match self.input.read()? {
                Some(ev) => {
                    // We only care about note down events here.
                    if ev.message.status & 0xf0 == 0x90 {
                        if !notes.is_empty() && ev.timestamp - last_time < 80 {
                            notes.last_mut().unwrap().push(Note(ev.message.data1));
                            // println!("chord : {:?}", notes);
                        } else {
                            // Individual note.
                            notes.push(vec![Note(ev.message.data1)]);
                            // println!("single: {:?}", notes);
                        }
                        idle_count = 0;
                        last_time = ev.timestamp;
                    }
                },
                None => {
                    idle_count += 1;
                    if notes.is_empty() {
                        idle_count = 0;
                    }
                    if idle_count >= 6 {
                        println!("Timing out");
                        break;
                    }
                    thread::sleep(Duration::from_millis(250));
                }
            }
        }

        // Sort the chords, since the incoming order could differ.
        for ch in notes.iter_mut() {
            ch.sort();
        }

        Ok(notes)
    }
}

impl Write for MidiLearn {
    fn flush(&mut self) -> io::Result<()> {
        io::stdout().flush()
    }

    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        io::stdout().write(buf)
    }
}

// Determine if the json has a the given type.
fn is_type(json: &Value, t: &str) -> bool {
    json["type"] == json!(t)
}
<commit_msg>midilearn: Comment out debug messages<commit_after>//! Using MIDI, verify that the user is able to play a given exercise correctly.  The question
//! should be enough of a description to uniquely describe what should be played.  The answer
//! should be a parseable description of what needs to be played.
//!
//! The answer should be a JSON description of what to play, of one of the following types:
//!
//! { "type": "scale", "base": "C♯", "intervals": "WWHWWWH",
//!   "hands": 1, "octaves": 2, "style": "updown" }
//!
//! This describes a scale.  The base is the first note of the scale.  Sharps and flags can either
//! be "#" and "b" or Unicode characters "♯", and "♭".  The intervals describe the intervals
//! between each note.  Valid characters are 'H' for a half step, 'W' for a whole step, 'm' for a
//! minor third, 'M' for a major third, and '4' for a fourth.  It will be checked that the
//! intervals cover a full octabe.  Hands describes how many hands should be playing the exercise,
//! and octaves are how many octaves to play.  The currently only supported style is "updown",
//! which requires the scale be played from base to base, for the given number of octaves and then
//! back down.  Scales can be played starting in any octave.
//!
//! { "type": "voicing", "chords": [
//!    { "base": "D", "notes": [ "3", "7", "9", "5" },
//!    { "base": "G", "notes": [ "7", "3", "#9", "#5" },
//!    { "base": "C", "notes": [ "3", "7", "9", "5", "11" } ] }
//!
//! This describes a voicing exercise, which is a series of chords to be played.  Each chord has a
//! base, which may or may not actually be played.  The notes in the chord are described in musical
//! notation (again with text or Unicode sharp/flat indicators).  The chord will be built using the
//! next note of the given function.  The octave will be chosen so that the lowest non-root note
//! will be between F#-F surrounding middle C.
//!
//! For the notes, values can be chord positions.  It is also possible to have a note that is a
//! placeholder, as an empty string, which indicates that the following note should be moved up an
//! octave.

#![deny(missing_docs)]

extern crate editdistancewf;
extern crate portmidi;
extern crate timelearn;
#[macro_use] extern crate lazy_static;
extern crate serde;
#[macro_use] extern crate serde_derive;
#[macro_use] extern crate serde_json;

use portmidi::{PortMidi, InputPort};
use serde_json::{Value};
use std::error;
use std::io::{self, Write};
use std::result;
use std::thread;
use std::time::Duration;
pub use timelearn::{Problem, Status, User};

mod note;
use note::Note;

mod scales;
use scales::{Scale, ScaleSeq, Voicing};

/// Just box the errors for now.  TODO: Use a proper error type.
pub type Result<T> = result::Result<T, Box<error::Error + Send + Sync>>;

/// A learner based on playing the exercises on a MIDI device.
pub struct MidiLearn {
    input: InputPort,
}

impl MidiLearn {
    /// Construct a new MIDI learning, using the default MIDI port.  Invokes `f` with the
    /// MidiLearn.
    pub fn with_new<F, R>(f: F) -> Result<R>
        where F: FnOnce(&mut MidiLearn) -> R
    {
        let pm = PortMidi::new()?;
        let input = pm.default_input_port(4096)?;
        let mut learn = MidiLearn {
            input: input,
        };
        let result = f(&mut learn);
        Ok(result)
    }

}

impl User for MidiLearn {
    /// Practice a single exercise.  Waits for the user to play the exercise, and then returns a
    /// status indicating how well it was played.
    fn single(&mut self, word: &Problem) -> Result<Status> {
        let st1 = self.single_once(word)?;

        let mut stn = st1;
        loop {
            match stn {
                // If good, return the initial status.
                Status::Continue(4) => return Ok(st1),
                Status::Stopped => return Ok(Status::Stopped),
                _ => (),
            }
            println!("** Mistakes made, please play again **");
            stn = self.single_once(word)?;
        }
    }
}

impl MidiLearn {
    /// Ask the user once to play.
    fn single_once(&mut self, word: &Problem) -> Result<Status> {
        println!("Play: {}", word.question);

        let json: Value = serde_json::from_str(&word.answer)?;
        // println!("Json: {:?}, {:?}", json, json["typr"]);
        if is_type(&json, "scale") {
            let scale: Scale = serde_json::from_value(json)?;

            let mut seq = ScaleSeq::from_scale(&scale)?;
            // println!("scale: {:?}", seq);

            self.drain()?;
            let user = self.record_scale()?;
            // println!("Played: {:?}", user);
            if seq.adjust_octave(&user) {
                // println!("First note good: {:?}", seq);
                let diff_count = seq.differences(&user);
                println!("There are {} differences", diff_count);
                if diff_count <= 3 {
                    Ok(Status::Continue(4 - diff_count as u8))
                } else {
                    Ok(Status::Continue(1))
                }
            } else {
                println!("First note mismatch, stopping");
                Ok(Status::Stopped)
            }
        } else if is_type(&json, "voicing") {
            let chords: Voicing = serde_json::from_value(json)?;

            let seq = ScaleSeq::from_voicing(&chords)?;
            // println!("chords: {:?}", seq);
            self.drain()?;
            let user = self.record_scale()?;
            // println!("user: {:?}", user);
            if user[0].len() == 1 {
                println!("Single note, stopping");
                Ok(Status::Stopped)
            } else {
                let diff_count = seq.differences(&user);
                println!("There are {} differences", diff_count);
                if diff_count == 0 {
                    Ok(Status::Continue(4))
                } else {
                    Ok(Status::Continue(1))
                }
            }
        } else {
            return Err(format!("Invalid type: {:?}", json["type"].as_str()).into());
        }
    }
}

impl MidiLearn {
    /// Drain any queued midi events.
    fn drain(&mut self) -> Result<()> {
        loop {
            match self.input.read()? {
                Some(_) => (),
                None => break,
            }
        }
        Ok(())
    }

    /// Record the user playing a scale.  Consider the scale done after a small pause of no
    /// playing.
    fn record_scale(&mut self) -> Result<Vec<Vec<Note>>> {
        let mut notes: Vec<Vec<Note>> = vec![];
        let mut idle_count = 0;
        let mut last_time = 0;
        loop {
            match self.input.read()? {
                Some(ev) => {
                    // We only care about note down events here.
                    if ev.message.status & 0xf0 == 0x90 {
                        if !notes.is_empty() && ev.timestamp - last_time < 80 {
                            notes.last_mut().unwrap().push(Note(ev.message.data1));
                            // println!("chord : {:?}", notes);
                        } else {
                            // Individual note.
                            notes.push(vec![Note(ev.message.data1)]);
                            // println!("single: {:?}", notes);
                        }
                        idle_count = 0;
                        last_time = ev.timestamp;
                    }
                },
                None => {
                    idle_count += 1;
                    if notes.is_empty() {
                        idle_count = 0;
                    }
                    if idle_count >= 6 {
                        println!("Timing out");
                        break;
                    }
                    thread::sleep(Duration::from_millis(250));
                }
            }
        }

        // Sort the chords, since the incoming order could differ.
        for ch in notes.iter_mut() {
            ch.sort();
        }

        Ok(notes)
    }
}

impl Write for MidiLearn {
    fn flush(&mut self) -> io::Result<()> {
        io::stdout().flush()
    }

    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        io::stdout().write(buf)
    }
}

// Determine if the json has a the given type.
fn is_type(json: &Value, t: &str) -> bool {
    json["type"] == json!(t)
}
<|endoftext|>


<commit_before>/// Declares an item with a doc attribute computed by some macro expression.
/// This allows documentation to be dynamically generated based on input.
/// Necessary to work around <https://github.com/rust-lang/rust/issues/52607>.
macro_rules! doc_concat {
    ( $( #[doc = $doc:expr] $( $thing:tt )* )* ) => ( $( #[doc = $doc] $( $thing )* )* );
}

macro_rules! partial_eq_string {
    ($id:ty $([$( $g:ident ),*])?) => {
        partial_eq_string!(@imp $(<$($g),*>)?, $id, str);
        partial_eq_string!(@imp $(<$($g),*>)?, $id, &str);
        partial_eq_string!(@imp $(<$($g),*>)?, $id, String);
        partial_eq_string!(@imp $(<$($g),*>)?, str, $id);
        partial_eq_string!(@imp $(<$($g),*>)?, &str, $id);
        partial_eq_string!(@imp $(<$($g),*>)?, String, $id);
    };
    (@imp $(<$( $g:ident ),*>)?, $l:ty, $r:ty) => {
        impl $(<$($g),*>)? PartialEq<$r> for $l {
            fn eq(&self, other: &$r) -> bool {
                AsRef::<str>::as_ref(self)
                    == AsRef::<str>::as_ref(other)
            }
        }
    }
}

macro_rules! as_str_based_impls {
    ($id:ty) => {
        impl AsRef<str> for $id {
            fn as_ref(&self) -> &str {
                self.as_str()
            }
        }

        impl std::fmt::Display for $id {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(f, "{}", self.as_str())
            }
        }

        impl PartialEq for $id {
            fn eq(&self, other: &Self) -> bool {
                self.as_str() == other.as_str()
            }
        }

        impl Eq for $id {}

        impl PartialOrd for $id {
            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
                PartialOrd::partial_cmp(self.as_str(), other.as_str())
            }
        }

        impl Ord for $id {
            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
                Ord::cmp(self.as_str(), other.as_str())
            }
        }

        impl std::hash::Hash for $id {
            fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
                self.as_str().hash(state);
            }
        }

        #[cfg(feature = "serde")]
        impl serde::Serialize for $id {
            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
            where
                S: serde::Serializer,
            {
                serializer.serialize_str(self.as_str())
            }
        }
    };
}

macro_rules! common_impls {
    ($id:ty, $try_from:ident, $desc:literal) => {
        impl $id {
            doc_concat! {
                #[doc = concat!("Creates a string slice from this `", stringify!($id), "`")]
                pub fn as_str(&self) -> &str {
                    &self.full_id
                }
            }

            doc_concat! {
                #[doc = concat!("Creates a byte slice from this `", stringify!($id), "`")]
                pub fn as_bytes(&self) -> &[u8] {
                    self.full_id.as_bytes()
                }
            }

            doc_concat! {
                #[doc = concat!("Converts this `", stringify!($id), "` into a `String`")]
                pub fn into_string(self) -> String {
                    self.full_id.into()
                }
            }

            doc_concat! {
                #[doc = concat!("Converts this `", stringify!($id), "` into a `Vec<u8>`")]
                pub fn into_bytes(self) -> Vec<u8> {
                    Box::<[u8]>::from(self.full_id).into()
                }
            }
        }

        impl From<$id> for String {
            fn from(id: $id) -> Self {
                id.into_string()
            }
        }

        impl From<$id> for Vec<u8> {
            fn from(id: $id) -> Self {
                id.into_bytes()
            }
        }

        impl std::str::FromStr for $id {
            type Err = crate::Error;

            fn from_str(s: &str) -> Result<Self, Self::Err> {
                $try_from(s)
            }
        }

        impl std::convert::TryFrom<&str> for $id {
            type Error = crate::Error;

            fn try_from(s: &str) -> Result<Self, Self::Error> {
                $try_from(s)
            }
        }

        impl std::convert::TryFrom<String> for $id {
            type Error = crate::Error;

            fn try_from(s: String) -> Result<Self, Self::Error> {
                $try_from(s)
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for $id {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                crate::deserialize_id(deserializer, $desc)
            }
        }

        as_str_based_impls!($id);
        partial_eq_string!($id);
    };
}

macro_rules! opaque_identifier_common_impls {
    ($id:ty) => {
        impl $id {
            fn from_borrowed(s: &str) -> &Self {
                unsafe { std::mem::transmute(s) }
            }

            pub(super) fn from_owned(s: Box<str>) -> Box<Self> {
                unsafe { Box::from_raw(Box::into_raw(s) as _) }
            }

            fn into_owned(self: Box<Self>) -> Box<str> {
                unsafe { Box::from_raw(Box::into_raw(self) as _) }
            }

            doc_concat! {
                #[doc = concat!("Creates a string slice from this `", stringify!($id), "`.")]
                pub fn as_str(&self) -> &str {
                    &self.0
                }
            }

            doc_concat! {
                #[doc = concat!("Creates a byte slice from this `", stringify!($id), "`.")]
                pub fn as_bytes(&self) -> &[u8] {
                    self.0.as_bytes()
                }
            }
        }

        impl std::fmt::Debug for $id {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                <str as std::fmt::Debug>::fmt(self.as_str(), f)
            }
        }

        impl Clone for Box<$id> {
            fn clone(&self) -> Self {
                (**self).to_owned()
            }
        }

        impl ToOwned for $id {
            type Owned = Box<$id>;

            fn to_owned(&self) -> Self::Owned {
                Self::from_owned(self.0.into())
            }
        }

        impl From<&$id> for Box<$id> {
            fn from(id: &$id) -> Self {
                id.to_owned()
            }
        }

        impl AsRef<str> for Box<$id> {
            fn as_ref(&self) -> &str {
                self.as_str()
            }
        }

        impl From<&$id> for std::rc::Rc<$id> {
            fn from(s: &$id) -> std::rc::Rc<$id> {
                let rc = std::rc::Rc::<str>::from(s.as_str());
                unsafe { std::rc::Rc::from_raw(std::rc::Rc::into_raw(rc) as *const $id) }
            }
        }

        impl From<&$id> for std::sync::Arc<$id> {
            fn from(s: &$id) -> std::sync::Arc<$id> {
                let arc = std::sync::Arc::<str>::from(s.as_str());
                unsafe { std::sync::Arc::from_raw(std::sync::Arc::into_raw(arc) as *const $id) }
            }
        }

        as_str_based_impls!($id);
        partial_eq_string!($id);
        partial_eq_string!(Box<$id>);
    };
}

macro_rules! opaque_identifier {
    (
        $( #[doc = $docs:literal] )*
        $vis:vis type $id:ident;
    ) => {
        $( #[doc = $docs] )*
        #[repr(transparent)]
        pub struct $id(str);

        opaque_identifier_common_impls!($id);

        impl<'a> From<&'a str> for &'a $id {
            fn from(s: &'a str) -> Self {
                $id::from_borrowed(s)
            }
        }

        impl From<&str> for Box<$id> {
            fn from(s: &str) -> Self {
                $id::from_owned(s.into())
            }
        }

        impl From<String> for Box<$id> {
            fn from(s: String) -> Self {
                $id::from_owned(s.into())
            }
        }

        impl From<Box<$id>> for String {
            fn from(id: Box<$id>) -> Self {
                id.into_owned().into()
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for Box<$id> {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                Box::<str>::deserialize(deserializer).map($id::from_owned)
            }
        }
    };
}

macro_rules! opaque_identifier_validated {
    (
        $( #[doc = $docs:literal] )*
        $vis:vis type $id:ident [ $validate_id:ident ];
    ) => {
        $( #[doc = $docs] )*
        #[repr(transparent)]
        pub struct $id(str);

        opaque_identifier_common_impls!($id);

        impl From<Box<$id>> for String {
            fn from(id: Box<$id>) -> Self {
                id.into_owned().into()
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for Box<$id> {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                use serde::de::Error;

                let s = String::deserialize(deserializer)?;

                match try_from(s) {
                    Ok(o) => Ok(o),
                    Err(e) => Err(D::Error::custom(e)),
                }
            }
        }

        fn try_from<S>(s: S) -> Result<Box<$id>, crate::Error>
        where
            S: AsRef<str> + Into<Box<str>>,
        {
            $validate_id(s.as_ref())?;
            Ok($id::from_owned(s.into()))
        }

        impl<'a> std::convert::TryFrom<&'a str> for &'a $id {
            type Error = crate::Error;

            fn try_from(s: &'a str) -> Result<Self, Self::Error> {
                $validate_id(s)?;
                Ok($id::from_borrowed(s))
            }
        }

        impl std::str::FromStr for Box<$id> {
            type Err = crate::Error;

            fn from_str(s: &str) -> Result<Self, Self::Err> {
                try_from(s)
            }
        }

        impl std::convert::TryFrom<&str> for Box<$id> {
            type Error = crate::Error;

            fn try_from(s: &str) -> Result<Self, Self::Error> {
                try_from(s)
            }
        }

        impl std::convert::TryFrom<String> for Box<$id> {
            type Error = crate::Error;

            fn try_from(s: String) -> Result<Self, Self::Error> {
                try_from(s)
            }
        }
    }
}
<commit_msg>identifiers: Add more PartialEq implementations<commit_after>/// Declares an item with a doc attribute computed by some macro expression.
/// This allows documentation to be dynamically generated based on input.
/// Necessary to work around <https://github.com/rust-lang/rust/issues/52607>.
macro_rules! doc_concat {
    ( $( #[doc = $doc:expr] $( $thing:tt )* )* ) => ( $( #[doc = $doc] $( $thing )* )* );
}

macro_rules! partial_eq_string {
    ($id:ty $([$( $g:ident ),*])?) => {
        partial_eq_string!(@imp $(<$($g),*>)?, $id, str);
        partial_eq_string!(@imp $(<$($g),*>)?, $id, &str);
        partial_eq_string!(@imp $(<$($g),*>)?, $id, String);
        partial_eq_string!(@imp $(<$($g),*>)?, str, $id);
        partial_eq_string!(@imp $(<$($g),*>)?, &str, $id);
        partial_eq_string!(@imp $(<$($g),*>)?, String, $id);
    };
    (@imp $(<$( $g:ident ),*>)?, $l:ty, $r:ty) => {
        impl $(<$($g),*>)? PartialEq<$r> for $l {
            fn eq(&self, other: &$r) -> bool {
                AsRef::<str>::as_ref(self)
                    == AsRef::<str>::as_ref(other)
            }
        }
    }
}

macro_rules! as_str_based_impls {
    ($id:ty) => {
        impl AsRef<str> for $id {
            fn as_ref(&self) -> &str {
                self.as_str()
            }
        }

        impl std::fmt::Display for $id {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(f, "{}", self.as_str())
            }
        }

        impl PartialEq for $id {
            fn eq(&self, other: &Self) -> bool {
                self.as_str() == other.as_str()
            }
        }

        impl Eq for $id {}

        impl PartialOrd for $id {
            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
                PartialOrd::partial_cmp(self.as_str(), other.as_str())
            }
        }

        impl Ord for $id {
            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
                Ord::cmp(self.as_str(), other.as_str())
            }
        }

        impl std::hash::Hash for $id {
            fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
                self.as_str().hash(state);
            }
        }

        #[cfg(feature = "serde")]
        impl serde::Serialize for $id {
            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
            where
                S: serde::Serializer,
            {
                serializer.serialize_str(self.as_str())
            }
        }
    };
}

macro_rules! common_impls {
    ($id:ty, $try_from:ident, $desc:literal) => {
        impl $id {
            doc_concat! {
                #[doc = concat!("Creates a string slice from this `", stringify!($id), "`")]
                pub fn as_str(&self) -> &str {
                    &self.full_id
                }
            }

            doc_concat! {
                #[doc = concat!("Creates a byte slice from this `", stringify!($id), "`")]
                pub fn as_bytes(&self) -> &[u8] {
                    self.full_id.as_bytes()
                }
            }

            doc_concat! {
                #[doc = concat!("Converts this `", stringify!($id), "` into a `String`")]
                pub fn into_string(self) -> String {
                    self.full_id.into()
                }
            }

            doc_concat! {
                #[doc = concat!("Converts this `", stringify!($id), "` into a `Vec<u8>`")]
                pub fn into_bytes(self) -> Vec<u8> {
                    Box::<[u8]>::from(self.full_id).into()
                }
            }
        }

        impl From<$id> for String {
            fn from(id: $id) -> Self {
                id.into_string()
            }
        }

        impl From<$id> for Vec<u8> {
            fn from(id: $id) -> Self {
                id.into_bytes()
            }
        }

        impl std::str::FromStr for $id {
            type Err = crate::Error;

            fn from_str(s: &str) -> Result<Self, Self::Err> {
                $try_from(s)
            }
        }

        impl std::convert::TryFrom<&str> for $id {
            type Error = crate::Error;

            fn try_from(s: &str) -> Result<Self, Self::Error> {
                $try_from(s)
            }
        }

        impl std::convert::TryFrom<String> for $id {
            type Error = crate::Error;

            fn try_from(s: String) -> Result<Self, Self::Error> {
                $try_from(s)
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for $id {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                crate::deserialize_id(deserializer, $desc)
            }
        }

        as_str_based_impls!($id);
        partial_eq_string!($id);
    };
}

macro_rules! opaque_identifier_common_impls {
    ($id:ty) => {
        impl $id {
            fn from_borrowed(s: &str) -> &Self {
                unsafe { std::mem::transmute(s) }
            }

            pub(super) fn from_owned(s: Box<str>) -> Box<Self> {
                unsafe { Box::from_raw(Box::into_raw(s) as _) }
            }

            fn into_owned(self: Box<Self>) -> Box<str> {
                unsafe { Box::from_raw(Box::into_raw(self) as _) }
            }

            doc_concat! {
                #[doc = concat!("Creates a string slice from this `", stringify!($id), "`.")]
                pub fn as_str(&self) -> &str {
                    &self.0
                }
            }

            doc_concat! {
                #[doc = concat!("Creates a byte slice from this `", stringify!($id), "`.")]
                pub fn as_bytes(&self) -> &[u8] {
                    self.0.as_bytes()
                }
            }
        }

        impl std::fmt::Debug for $id {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                <str as std::fmt::Debug>::fmt(self.as_str(), f)
            }
        }

        impl Clone for Box<$id> {
            fn clone(&self) -> Self {
                (**self).to_owned()
            }
        }

        impl ToOwned for $id {
            type Owned = Box<$id>;

            fn to_owned(&self) -> Self::Owned {
                Self::from_owned(self.0.into())
            }
        }

        impl From<&$id> for Box<$id> {
            fn from(id: &$id) -> Self {
                id.to_owned()
            }
        }

        impl AsRef<str> for Box<$id> {
            fn as_ref(&self) -> &str {
                self.as_str()
            }
        }

        impl From<&$id> for std::rc::Rc<$id> {
            fn from(s: &$id) -> std::rc::Rc<$id> {
                let rc = std::rc::Rc::<str>::from(s.as_str());
                unsafe { std::rc::Rc::from_raw(std::rc::Rc::into_raw(rc) as *const $id) }
            }
        }

        impl From<&$id> for std::sync::Arc<$id> {
            fn from(s: &$id) -> std::sync::Arc<$id> {
                let arc = std::sync::Arc::<str>::from(s.as_str());
                unsafe { std::sync::Arc::from_raw(std::sync::Arc::into_raw(arc) as *const $id) }
            }
        }

        impl PartialEq<$id> for Box<$id> {
            fn eq(&self, other: &$id) -> bool {
                self.as_str() == other.as_str()
            }
        }

        impl PartialEq<&'_ $id> for Box<$id> {
            fn eq(&self, other: &&$id) -> bool {
                self.as_str() == other.as_str()
            }
        }

        impl PartialEq<Box<$id>> for $id {
            fn eq(&self, other: &Box<$id>) -> bool {
                self.as_str() == other.as_str()
            }
        }

        impl PartialEq<Box<$id>> for &'_ $id {
            fn eq(&self, other: &Box<$id>) -> bool {
                self.as_str() == other.as_str()
            }
        }

        as_str_based_impls!($id);
        partial_eq_string!($id);
        partial_eq_string!(Box<$id>);
    };
}

macro_rules! opaque_identifier {
    (
        $( #[doc = $docs:literal] )*
        $vis:vis type $id:ident;
    ) => {
        $( #[doc = $docs] )*
        #[repr(transparent)]
        pub struct $id(str);

        opaque_identifier_common_impls!($id);

        impl<'a> From<&'a str> for &'a $id {
            fn from(s: &'a str) -> Self {
                $id::from_borrowed(s)
            }
        }

        impl From<&str> for Box<$id> {
            fn from(s: &str) -> Self {
                $id::from_owned(s.into())
            }
        }

        impl From<String> for Box<$id> {
            fn from(s: String) -> Self {
                $id::from_owned(s.into())
            }
        }

        impl From<Box<$id>> for String {
            fn from(id: Box<$id>) -> Self {
                id.into_owned().into()
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for Box<$id> {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                Box::<str>::deserialize(deserializer).map($id::from_owned)
            }
        }
    };
}

macro_rules! opaque_identifier_validated {
    (
        $( #[doc = $docs:literal] )*
        $vis:vis type $id:ident [ $validate_id:ident ];
    ) => {
        $( #[doc = $docs] )*
        #[repr(transparent)]
        pub struct $id(str);

        opaque_identifier_common_impls!($id);

        impl From<Box<$id>> for String {
            fn from(id: Box<$id>) -> Self {
                id.into_owned().into()
            }
        }

        #[cfg(feature = "serde")]
        impl<'de> serde::Deserialize<'de> for Box<$id> {
            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
            where
                D: serde::Deserializer<'de>,
            {
                use serde::de::Error;

                let s = String::deserialize(deserializer)?;

                match try_from(s) {
                    Ok(o) => Ok(o),
                    Err(e) => Err(D::Error::custom(e)),
                }
            }
        }

        fn try_from<S>(s: S) -> Result<Box<$id>, crate::Error>
        where
            S: AsRef<str> + Into<Box<str>>,
        {
            $validate_id(s.as_ref())?;
            Ok($id::from_owned(s.into()))
        }

        impl<'a> std::convert::TryFrom<&'a str> for &'a $id {
            type Error = crate::Error;

            fn try_from(s: &'a str) -> Result<Self, Self::Error> {
                $validate_id(s)?;
                Ok($id::from_borrowed(s))
            }
        }

        impl std::str::FromStr for Box<$id> {
            type Err = crate::Error;

            fn from_str(s: &str) -> Result<Self, Self::Err> {
                try_from(s)
            }
        }

        impl std::convert::TryFrom<&str> for Box<$id> {
            type Error = crate::Error;

            fn try_from(s: &str) -> Result<Self, Self::Error> {
                try_from(s)
            }
        }

        impl std::convert::TryFrom<String> for Box<$id> {
            type Error = crate::Error;

            fn try_from(s: String) -> Result<Self, Self::Error> {
                try_from(s)
            }
        }
    }
}
<|endoftext|>


<commit_before>use indoc::indoc;
use serde_derive::Deserialize;
use serde_yaml::Deserializer;
use std::fmt::Debug;

fn test_error<T>(yaml: &str, expected: &str)
where
    T: serde::de::DeserializeOwned + Debug,
{
    let result = serde_yaml::from_str::<T>(yaml);
    assert_eq!(expected, format!("{}", result.unwrap_err()));
}

#[test]
fn test_incorrect_type() {
    let yaml = indoc! {"
        ---
        str
    "};
    let expected = "invalid type: string \"str\", expected i16 at line 2 column 1";
    test_error::<i16>(yaml, expected);
}

#[test]
fn test_incorrect_nested_type() {
    #[derive(Deserialize, Debug)]
    struct A {
        #[allow(dead_code)]
        b: Vec<B>,
    }
    #[derive(Deserialize, Debug)]
    enum B {
        C(C),
    }
    #[derive(Deserialize, Debug)]
    struct C {
        #[allow(dead_code)]
        d: bool,
    }
    let yaml = indoc! {"
        ---
        b:
          - C:
              d: fase
    "};
    let expected =
        "b[0].C.d: invalid type: string \"fase\", expected a boolean at line 4 column 10";
    test_error::<A>(yaml, expected);
}

#[test]
fn test_empty() {
    let expected = "EOF while parsing a value";
    test_error::<String>("", expected);
}

#[test]
fn test_missing_field() {
    #[derive(Deserialize, Debug)]
    struct Basic {
        #[allow(dead_code)]
        v: bool,
        #[allow(dead_code)]
        w: bool,
    }
    let yaml = indoc! {"
        ---
        v: true
    "};
    let expected = "missing field `w` at line 2 column 1";
    test_error::<Basic>(yaml, expected);
}

#[test]
fn test_unknown_anchor() {
    let yaml = indoc! {"
        ---
        *some
    "};
    let expected = "unknown anchor at line 2 column 1";
    test_error::<String>(yaml, expected);
}

#[test]
fn test_ignored_unknown_anchor() {
    #[derive(Deserialize, Debug)]
    struct Wrapper {
        #[allow(dead_code)]
        c: (),
    }
    let yaml = indoc! {"
        ---
        b: [*a]
        c: ~
    "};
    let expected = "unknown anchor at line 2 column 5";
    test_error::<Wrapper>(yaml, expected);
}

#[test]
fn test_two_documents() {
    let yaml = indoc! {"
        ---
        0
        ---
        1
    "};
    let expected = "deserializing from YAML containing more than one document is not supported";
    test_error::<usize>(yaml, expected);
}

#[test]
fn test_second_document_syntax_error() {
    let yaml = indoc! {"
        ---
        0
        ---
        ]
    "};

    let mut de = Deserializer::from_str(yaml);
    let first_doc = de.next().unwrap();
    let result = <usize as serde::Deserialize>::deserialize(first_doc);
    assert_eq!(0, result.unwrap());

    let second_doc = de.next().unwrap();
    let result = <usize as serde::Deserialize>::deserialize(second_doc);
    let expected = "did not find expected node content at line 4 column 1, while parsing a block node at line 4 column 1";
    assert_eq!(expected, result.unwrap_err().to_string());
}

#[test]
fn test_variant_map_wrong_size() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(usize),
    }
    let yaml = indoc! {r#"
        ---
        "V": 16
        "other": 32
    "#};
    let expected = "invalid length 2, expected map containing 1 entry";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_variant_not_a_map() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(usize),
    }
    let yaml = indoc! {r#"
        ---
        - "V"
    "#};
    let expected = "invalid type: sequence, expected string or singleton map at line 2 column 1";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_variant_not_string() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(bool),
    }
    let yaml = indoc! {r#"
        ---
        {}: true
    "#};
    let expected = "invalid type: map, expected variant of enum `E` at line 2 column 1";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_bad_bool() {
    let yaml = indoc! {"
        ---
        !!bool str
    "};
    let expected = "invalid value: string \"str\", expected a boolean at line 2 column 1";
    test_error::<bool>(yaml, expected);
}

#[test]
fn test_bad_int() {
    let yaml = indoc! {"
        ---
        !!int str
    "};
    let expected = "invalid value: string \"str\", expected an integer at line 2 column 1";
    test_error::<i64>(yaml, expected);
}

#[test]
fn test_bad_float() {
    let yaml = indoc! {"
        ---
        !!float str
    "};
    let expected = "invalid value: string \"str\", expected a float at line 2 column 1";
    test_error::<f64>(yaml, expected);
}

#[test]
fn test_bad_null() {
    let yaml = indoc! {"
        ---
        !!null str
    "};
    let expected = "invalid value: string \"str\", expected null at line 2 column 1";
    test_error::<()>(yaml, expected);
}

#[test]
fn test_short_tuple() {
    let yaml = indoc! {"
        ---
        [0, 0]
    "};
    let expected = "invalid length 2, expected a tuple of size 3 at line 2 column 1";
    test_error::<(u8, u8, u8)>(yaml, expected);
}

#[test]
fn test_long_tuple() {
    let yaml = indoc! {"
        ---
        [0, 0, 0]
    "};
    let expected = "invalid length 3, expected sequence of 2 elements at line 2 column 1";
    test_error::<(u8, u8)>(yaml, expected);
}

#[test]
fn test_invalid_scalar_type() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: [(); 1],
    }

    let yaml = "x:\n";
    let expected = "x: invalid type: unit value, expected an array of length 1 at line 1 column 3";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_infinite_recursion_objects() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "&a {'x': *a}";
    let expected = "recursion limit exceeded at position 0";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_infinite_recursion_arrays() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "&a [*a]";
    let expected = "recursion limit exceeded at position 0";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_finite_recursion_objects() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "{'x':".repeat(1_000) + &"}".repeat(1_000);
    let expected = "recursion limit exceeded at line 1 column 641";
    test_error::<S>(&yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_finite_recursion_arrays() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "[".repeat(1_000) + &"]".repeat(1_000);
    let expected = "recursion limit exceeded at line 1 column 129";
    test_error::<S>(&yaml, expected);
}
<commit_msg>Replace "{}" format with to_string()<commit_after>use indoc::indoc;
use serde_derive::Deserialize;
use serde_yaml::Deserializer;
use std::fmt::Debug;

fn test_error<T>(yaml: &str, expected: &str)
where
    T: serde::de::DeserializeOwned + Debug,
{
    let result = serde_yaml::from_str::<T>(yaml);
    assert_eq!(expected, result.unwrap_err().to_string());
}

#[test]
fn test_incorrect_type() {
    let yaml = indoc! {"
        ---
        str
    "};
    let expected = "invalid type: string \"str\", expected i16 at line 2 column 1";
    test_error::<i16>(yaml, expected);
}

#[test]
fn test_incorrect_nested_type() {
    #[derive(Deserialize, Debug)]
    struct A {
        #[allow(dead_code)]
        b: Vec<B>,
    }
    #[derive(Deserialize, Debug)]
    enum B {
        C(C),
    }
    #[derive(Deserialize, Debug)]
    struct C {
        #[allow(dead_code)]
        d: bool,
    }
    let yaml = indoc! {"
        ---
        b:
          - C:
              d: fase
    "};
    let expected =
        "b[0].C.d: invalid type: string \"fase\", expected a boolean at line 4 column 10";
    test_error::<A>(yaml, expected);
}

#[test]
fn test_empty() {
    let expected = "EOF while parsing a value";
    test_error::<String>("", expected);
}

#[test]
fn test_missing_field() {
    #[derive(Deserialize, Debug)]
    struct Basic {
        #[allow(dead_code)]
        v: bool,
        #[allow(dead_code)]
        w: bool,
    }
    let yaml = indoc! {"
        ---
        v: true
    "};
    let expected = "missing field `w` at line 2 column 1";
    test_error::<Basic>(yaml, expected);
}

#[test]
fn test_unknown_anchor() {
    let yaml = indoc! {"
        ---
        *some
    "};
    let expected = "unknown anchor at line 2 column 1";
    test_error::<String>(yaml, expected);
}

#[test]
fn test_ignored_unknown_anchor() {
    #[derive(Deserialize, Debug)]
    struct Wrapper {
        #[allow(dead_code)]
        c: (),
    }
    let yaml = indoc! {"
        ---
        b: [*a]
        c: ~
    "};
    let expected = "unknown anchor at line 2 column 5";
    test_error::<Wrapper>(yaml, expected);
}

#[test]
fn test_two_documents() {
    let yaml = indoc! {"
        ---
        0
        ---
        1
    "};
    let expected = "deserializing from YAML containing more than one document is not supported";
    test_error::<usize>(yaml, expected);
}

#[test]
fn test_second_document_syntax_error() {
    let yaml = indoc! {"
        ---
        0
        ---
        ]
    "};

    let mut de = Deserializer::from_str(yaml);
    let first_doc = de.next().unwrap();
    let result = <usize as serde::Deserialize>::deserialize(first_doc);
    assert_eq!(0, result.unwrap());

    let second_doc = de.next().unwrap();
    let result = <usize as serde::Deserialize>::deserialize(second_doc);
    let expected = "did not find expected node content at line 4 column 1, while parsing a block node at line 4 column 1";
    assert_eq!(expected, result.unwrap_err().to_string());
}

#[test]
fn test_variant_map_wrong_size() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(usize),
    }
    let yaml = indoc! {r#"
        ---
        "V": 16
        "other": 32
    "#};
    let expected = "invalid length 2, expected map containing 1 entry";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_variant_not_a_map() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(usize),
    }
    let yaml = indoc! {r#"
        ---
        - "V"
    "#};
    let expected = "invalid type: sequence, expected string or singleton map at line 2 column 1";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_variant_not_string() {
    #[derive(Deserialize, Debug)]
    enum E {
        V(bool),
    }
    let yaml = indoc! {r#"
        ---
        {}: true
    "#};
    let expected = "invalid type: map, expected variant of enum `E` at line 2 column 1";
    test_error::<E>(yaml, expected);
}

#[test]
fn test_bad_bool() {
    let yaml = indoc! {"
        ---
        !!bool str
    "};
    let expected = "invalid value: string \"str\", expected a boolean at line 2 column 1";
    test_error::<bool>(yaml, expected);
}

#[test]
fn test_bad_int() {
    let yaml = indoc! {"
        ---
        !!int str
    "};
    let expected = "invalid value: string \"str\", expected an integer at line 2 column 1";
    test_error::<i64>(yaml, expected);
}

#[test]
fn test_bad_float() {
    let yaml = indoc! {"
        ---
        !!float str
    "};
    let expected = "invalid value: string \"str\", expected a float at line 2 column 1";
    test_error::<f64>(yaml, expected);
}

#[test]
fn test_bad_null() {
    let yaml = indoc! {"
        ---
        !!null str
    "};
    let expected = "invalid value: string \"str\", expected null at line 2 column 1";
    test_error::<()>(yaml, expected);
}

#[test]
fn test_short_tuple() {
    let yaml = indoc! {"
        ---
        [0, 0]
    "};
    let expected = "invalid length 2, expected a tuple of size 3 at line 2 column 1";
    test_error::<(u8, u8, u8)>(yaml, expected);
}

#[test]
fn test_long_tuple() {
    let yaml = indoc! {"
        ---
        [0, 0, 0]
    "};
    let expected = "invalid length 3, expected sequence of 2 elements at line 2 column 1";
    test_error::<(u8, u8)>(yaml, expected);
}

#[test]
fn test_invalid_scalar_type() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: [(); 1],
    }

    let yaml = "x:\n";
    let expected = "x: invalid type: unit value, expected an array of length 1 at line 1 column 3";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_infinite_recursion_objects() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "&a {'x': *a}";
    let expected = "recursion limit exceeded at position 0";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_infinite_recursion_arrays() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "&a [*a]";
    let expected = "recursion limit exceeded at position 0";
    test_error::<S>(yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_finite_recursion_objects() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "{'x':".repeat(1_000) + &"}".repeat(1_000);
    let expected = "recursion limit exceeded at line 1 column 641";
    test_error::<S>(&yaml, expected);
}

#[cfg(not(miri))]
#[test]
fn test_finite_recursion_arrays() {
    #[derive(Deserialize, Debug)]
    struct S {
        #[allow(dead_code)]
        x: Option<Box<S>>,
    }

    let yaml = "[".repeat(1_000) + &"]".repeat(1_000);
    let expected = "recursion limit exceeded at line 1 column 129";
    test_error::<S>(&yaml, expected);
}
<|endoftext|>


<commit_before>//! Handle process life-time and message passing for proc-macro client

use std::{
    convert::{TryFrom, TryInto},
    ffi::{OsStr, OsString},
    io::{self, BufRead, BufReader, Write},
    path::{Path, PathBuf},
    process::{Child, Command, Stdio},
    sync::{Arc, Weak},
};

use crossbeam_channel::{bounded, Receiver, Sender};
use tt::Subtree;

use crate::{
    msg::{ErrorCode, Message, Request, Response, ResponseError},
    rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind},
};

#[derive(Debug, Default)]
pub(crate) struct ProcMacroProcessSrv {
    inner: Weak<Sender<Task>>,
}

#[derive(Debug)]
pub(crate) struct ProcMacroProcessThread {
    // XXX: drop order is significant
    sender: Arc<Sender<Task>>,
    handle: jod_thread::JoinHandle<()>,
}

impl ProcMacroProcessSrv {
    pub(crate) fn run(
        process_path: PathBuf,
        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
    ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> {
        let process = Process::run(process_path, args)?;

        let (task_tx, task_rx) = bounded(0);
        let handle = jod_thread::spawn(move || {
            client_loop(task_rx, process);
        });

        let task_tx = Arc::new(task_tx);
        let srv = ProcMacroProcessSrv { inner: Arc::downgrade(&task_tx) };
        let thread = ProcMacroProcessThread { handle, sender: task_tx };

        Ok((thread, srv))
    }

    pub(crate) fn find_proc_macros(
        &self,
        dylib_path: &Path,
    ) -> Result<Vec<(String, ProcMacroKind)>, tt::ExpansionError> {
        let task = ListMacrosTask { lib: dylib_path.to_path_buf() };

        let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?;
        Ok(result.macros)
    }

    pub(crate) fn custom_derive(
        &self,
        dylib_path: &Path,
        subtree: &Subtree,
        derive_name: &str,
    ) -> Result<Subtree, tt::ExpansionError> {
        let task = ExpansionTask {
            macro_body: subtree.clone(),
            macro_name: derive_name.to_string(),
            attributes: None,
            lib: dylib_path.to_path_buf(),
        };

        let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?;
        Ok(result.expansion)
    }

    pub(crate) fn send_task<R>(&self, req: Request) -> Result<R, tt::ExpansionError>
    where
        R: TryFrom<Response, Error = &'static str>,
    {
        let (result_tx, result_rx) = bounded(0);
        let sender = match self.inner.upgrade() {
            None => {
                return Err(tt::ExpansionError::Unknown("Proc macro process is closed.".into()))
            }
            Some(it) => it,
        };
        sender.send(Task { req, result_tx }).unwrap();
        let res = result_rx
            .recv()
            .map_err(|_| tt::ExpansionError::Unknown("Proc macro thread is closed.".into()))?;

        match res {
            Some(Response::Error(err)) => {
                return Err(tt::ExpansionError::ExpansionError(err.message));
            }
            Some(res) => Ok(res.try_into().map_err(|err| {
                tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err))
            })?),
            None => Err(tt::ExpansionError::Unknown("Empty result".into())),
        }
    }
}

fn client_loop(task_rx: Receiver<Task>, mut process: Process) {
    let (mut stdin, mut stdout) = process.stdio().expect("couldn't access child stdio");

    for Task { req, result_tx } in task_rx {
        match send_request(&mut stdin, &mut stdout, req) {
            Ok(res) => result_tx.send(res).unwrap(),
            Err(_err) => {
                let res = Response::Error(ResponseError {
                    code: ErrorCode::ServerErrorEnd,
                    message: "Server closed".into(),
                });
                result_tx.send(res.into()).unwrap();
                // Restart the process
                if process.restart().is_err() {
                    break;
                }
                let stdio = match process.stdio() {
                    None => break,
                    Some(it) => it,
                };
                stdin = stdio.0;
                stdout = stdio.1;
            }
        }
    }
}

struct Task {
    req: Request,
    result_tx: Sender<Option<Response>>,
}

struct Process {
    path: PathBuf,
    args: Vec<OsString>,
    child: Child,
}

impl Drop for Process {
    fn drop(&mut self) {
        let _ = self.child.kill();
    }
}

impl Process {
    fn run(
        path: PathBuf,
        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
    ) -> io::Result<Process> {
        let args = args.into_iter().map(|s| s.as_ref().into()).collect();
        let child = mk_child(&path, &args)?;
        Ok(Process { path, args, child })
    }

    fn restart(&mut self) -> io::Result<()> {
        let _ = self.child.kill();
        self.child = mk_child(&self.path, &self.args)?;
        Ok(())
    }

    fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> {
        let stdin = self.child.stdin.take()?;
        let stdout = self.child.stdout.take()?;
        let read = BufReader::new(stdout);

        Some((stdin, read))
    }
}

fn mk_child(path: &Path, args: impl IntoIterator<Item = impl AsRef<OsStr>>) -> io::Result<Child> {
    Command::new(&path)
        .args(args)
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::inherit())
        .spawn()
}

fn send_request(
    mut writer: &mut impl Write,
    mut reader: &mut impl BufRead,
    req: Request,
) -> io::Result<Option<Response>> {
    req.write(&mut writer)?;
    Ok(Response::read(&mut reader)?)
}
<commit_msg>Don't respawn proc macro server on crash<commit_after>//! Handle process life-time and message passing for proc-macro client

use std::{
    convert::{TryFrom, TryInto},
    ffi::{OsStr, OsString},
    io::{self, BufRead, BufReader, Write},
    path::{Path, PathBuf},
    process::{Child, Command, Stdio},
    sync::{Arc, Weak},
};

use crossbeam_channel::{bounded, Receiver, Sender};
use tt::Subtree;

use crate::{
    msg::{ErrorCode, Message, Request, Response, ResponseError},
    rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask, ProcMacroKind},
};

#[derive(Debug, Default)]
pub(crate) struct ProcMacroProcessSrv {
    inner: Weak<Sender<Task>>,
}

#[derive(Debug)]
pub(crate) struct ProcMacroProcessThread {
    // XXX: drop order is significant
    sender: Arc<Sender<Task>>,
    handle: jod_thread::JoinHandle<()>,
}

impl ProcMacroProcessSrv {
    pub(crate) fn run(
        process_path: PathBuf,
        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
    ) -> io::Result<(ProcMacroProcessThread, ProcMacroProcessSrv)> {
        let process = Process::run(process_path, args)?;

        let (task_tx, task_rx) = bounded(0);
        let handle = jod_thread::spawn(move || {
            client_loop(task_rx, process);
        });

        let task_tx = Arc::new(task_tx);
        let srv = ProcMacroProcessSrv { inner: Arc::downgrade(&task_tx) };
        let thread = ProcMacroProcessThread { handle, sender: task_tx };

        Ok((thread, srv))
    }

    pub(crate) fn find_proc_macros(
        &self,
        dylib_path: &Path,
    ) -> Result<Vec<(String, ProcMacroKind)>, tt::ExpansionError> {
        let task = ListMacrosTask { lib: dylib_path.to_path_buf() };

        let result: ListMacrosResult = self.send_task(Request::ListMacro(task))?;
        Ok(result.macros)
    }

    pub(crate) fn custom_derive(
        &self,
        dylib_path: &Path,
        subtree: &Subtree,
        derive_name: &str,
    ) -> Result<Subtree, tt::ExpansionError> {
        let task = ExpansionTask {
            macro_body: subtree.clone(),
            macro_name: derive_name.to_string(),
            attributes: None,
            lib: dylib_path.to_path_buf(),
        };

        let result: ExpansionResult = self.send_task(Request::ExpansionMacro(task))?;
        Ok(result.expansion)
    }

    pub(crate) fn send_task<R>(&self, req: Request) -> Result<R, tt::ExpansionError>
    where
        R: TryFrom<Response, Error = &'static str>,
    {
        let (result_tx, result_rx) = bounded(0);
        let sender = match self.inner.upgrade() {
            None => return Err(tt::ExpansionError::Unknown("proc macro process is closed".into())),
            Some(it) => it,
        };
        sender
            .send(Task { req, result_tx })
            .map_err(|_| tt::ExpansionError::Unknown("proc macro server crashed".into()))?;

        let res = result_rx
            .recv()
            .map_err(|_| tt::ExpansionError::Unknown("proc macro server crashed".into()))?;

        match res {
            Some(Response::Error(err)) => {
                return Err(tt::ExpansionError::ExpansionError(err.message));
            }
            Some(res) => Ok(res.try_into().map_err(|err| {
                tt::ExpansionError::Unknown(format!("Fail to get response, reason : {:#?} ", err))
            })?),
            None => Err(tt::ExpansionError::Unknown("Empty result".into())),
        }
    }
}

fn client_loop(task_rx: Receiver<Task>, mut process: Process) {
    let (mut stdin, mut stdout) = process.stdio().expect("couldn't access child stdio");

    for Task { req, result_tx } in task_rx {
        match send_request(&mut stdin, &mut stdout, req) {
            Ok(res) => result_tx.send(res).unwrap(),
            Err(_err) => {
                log::error!(
                    "proc macro server crashed, server process state: {:?}",
                    process.child.try_wait()
                );
                let res = Response::Error(ResponseError {
                    code: ErrorCode::ServerErrorEnd,
                    message: "proc macro server crashed".into(),
                });
                result_tx.send(res.into()).unwrap();
                // Exit the thread.
                break;
            }
        }
    }
}

struct Task {
    req: Request,
    result_tx: Sender<Option<Response>>,
}

struct Process {
    child: Child,
}

impl Drop for Process {
    fn drop(&mut self) {
        let _ = self.child.kill();
    }
}

impl Process {
    fn run(
        path: PathBuf,
        args: impl IntoIterator<Item = impl AsRef<OsStr>>,
    ) -> io::Result<Process> {
        let args: Vec<OsString> = args.into_iter().map(|s| s.as_ref().into()).collect();
        let child = mk_child(&path, &args)?;
        Ok(Process { child })
    }

    fn stdio(&mut self) -> Option<(impl Write, impl BufRead)> {
        let stdin = self.child.stdin.take()?;
        let stdout = self.child.stdout.take()?;
        let read = BufReader::new(stdout);

        Some((stdin, read))
    }
}

fn mk_child(path: &Path, args: impl IntoIterator<Item = impl AsRef<OsStr>>) -> io::Result<Child> {
    Command::new(&path)
        .args(args)
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::inherit())
        .spawn()
}

fn send_request(
    mut writer: &mut impl Write,
    mut reader: &mut impl BufRead,
    req: Request,
) -> io::Result<Option<Response>> {
    req.write(&mut writer)?;
    Ok(Response::read(&mut reader)?)
}
<|endoftext|>