<commit_before>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use gst::glib;
use gst::prelude::*;

use pretty_assertions::assert_eq;

fn init() {
    use std::sync::Once;
    static INIT: Once = Once::new();

    INIT.call_once(|| {
        gst::init().unwrap();
        gstrsclosedcaption::plugin_register_static().expect("mccenc test");
    });
}

/// Encode a single CDP packet and compare the output
#[test]
fn test_encode() {
    init();

    let input = [
        0x96, 0x69, 0x52, 0x4f, 0x67, 0x00, 0x00, 0x72, 0xf4, 0xfc, 0x80, 0x80, 0xfd, 0x80, 0x80,
        0xff, 0x02, 0x22, 0xfe, 0x8c, 0xff, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x73, 0x91, 0x81, 0x65, 0x6e, 0x67,
        0x81, 0x7f, 0xff, 0x74, 0x00, 0x00, 0x1c,
    ];

    let expected_output = b"File Format=MacCaption_MCC V1.0\r\n\
\r\n\
///////////////////////////////////////////////////////////////////////////////////\r\n\
// Computer Prompting and Captioning Company\r\n\
// Ancillary Data Packet Transfer File\r\n\
//\r\n\
// Permission to generate this format is granted provided that\r\n\
//   1. This ANC Transfer file format is used on an as-is basis and no warranty is given, and\r\n\
//   2. This entire descriptive information text is included in a generated .mcc file.\r\n\
//\r\n\
// General file format:\r\n\
//   HH:MM:SS:FF(tab)[Hexadecimal ANC data in groups of 2 characters]\r\n\
//     Hexadecimal data starts with the Ancillary Data Packet DID (Data ID defined in S291M)\r\n\
//       and concludes with the Check Sum following the User Data Words.\r\n\
//     Each time code line must contain at most one complete ancillary data packet.\r\n\
//     To transfer additional ANC Data successive lines may contain identical time code.\r\n\
//     Time Code Rate=[24, 25, 30, 30DF, 50, 60]\r\n\
//\r\n\
//   ANC data bytes may be represented by one ASCII character according to the following schema:\r\n\
//     G  FAh 00h 00h\r\n\
//     H  2 x (FAh 00h 00h)\r\n\
//     I  3 x (FAh 00h 00h)\r\n\
//     J  4 x (FAh 00h 00h)\r\n\
//     K  5 x (FAh 00h 00h)\r\n\
//     L  6 x (FAh 00h 00h)\r\n\
//     M  7 x (FAh 00h 00h)\r\n\
//     N  8 x (FAh 00h 00h)\r\n\
//     O  9 x (FAh 00h 00h)\r\n\
//     P  FBh 80h 80h\r\n\
//     Q  FCh 80h 80h\r\n\
//     R  FDh 80h 80h\r\n\
//     S  96h 69h\r\n\
//     T  61h 01h\r\n\
//     U  E1h 00h 00h 00h\r\n\
//     Z  00h\r\n\
//\r\n\
///////////////////////////////////////////////////////////////////////////////////\r\n\
\r\n\
UUID=14720C04-857D-40E2-86FC-F080DE44CE74\r\n\
Creation Program=GStreamer MCC Encoder 0.8.0\r\n\
Creation Date=Thursday, December 27, 2018\r\n\
Creation Time=17:34:47\r\n\
Time Code Rate=30DF\r\n\
\r\n\
11:12:13;14	T52S524F67ZZ72F4QRFF0222FE8CFFOM739181656E67817FFF74ZZ1CZ\r\n";

    let mut h = gst_check::Harness::new("mccenc");
    {
        let enc = h.element().expect("could not create encoder");
        enc.set_property("uuid", &"14720C04-857D-40E2-86FC-F080DE44CE74")
            .unwrap();
        enc.set_property(
            "creation-date",
            &glib::DateTime::new_utc(2018, 12, 27, 17, 34, 47.0).unwrap(),
        )
        .unwrap();
    }

    h.set_src_caps_str(
        "closedcaption/x-cea-708, format=(string)cdp, framerate=(fraction)30000/1001",
    );

    let tc = gst_video::ValidVideoTimeCode::new(
        gst::Fraction::new(30000, 1001),
        None,
        gst_video::VideoTimeCodeFlags::DROP_FRAME,
        11,
        12,
        13,
        14,
        0,
    )
    .unwrap();

    let buf = {
        let mut buf = gst::Buffer::from_mut_slice(Vec::from(input));
        let buf_ref = buf.get_mut().unwrap();
        gst_video::VideoTimeCodeMeta::add(buf_ref, &tc);
        buf_ref.set_pts(gst::ClockTime::from_seconds(0));
        buf
    };

    assert_eq!(h.push(buf), Ok(gst::FlowSuccess::Ok));
    h.push_event(gst::event::Eos::new());

    let buf = h.pull().expect("Couldn't pull buffer");
    let timecode = buf
        .meta::<gst_video::VideoTimeCodeMeta>()
        .expect("No timecode for buffer")
        .tc();
    assert_eq!(timecode, tc);

    let pts = buf.pts().unwrap();
    assert_eq!(pts, gst::ClockTime::ZERO);

    let map = buf.map_readable().expect("Couldn't map buffer readable");
    assert_eq!(
        std::str::from_utf8(map.as_ref()),
        std::str::from_utf8(expected_output.as_ref())
    );
}
<commit_msg>closedcaption/mccenc: Fix unit test to not require changing the version number in the output every time<commit_after>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use gst::glib;
use gst::prelude::*;

use pretty_assertions::assert_eq;

fn init() {
    use std::sync::Once;
    static INIT: Once = Once::new();

    INIT.call_once(|| {
        gst::init().unwrap();
        gstrsclosedcaption::plugin_register_static().expect("mccenc test");
    });
}

/// Encode a single CDP packet and compare the output
#[test]
fn test_encode() {
    init();

    let input = [
        0x96, 0x69, 0x52, 0x4f, 0x67, 0x00, 0x00, 0x72, 0xf4, 0xfc, 0x80, 0x80, 0xfd, 0x80, 0x80,
        0xff, 0x02, 0x22, 0xfe, 0x8c, 0xff, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00,
        0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x73, 0x91, 0x81, 0x65, 0x6e, 0x67,
        0x81, 0x7f, 0xff, 0x74, 0x00, 0x00, 0x1c,
    ];

    let expected_output = format!("File Format=MacCaption_MCC V1.0\r\n\
\r\n\
///////////////////////////////////////////////////////////////////////////////////\r\n\
// Computer Prompting and Captioning Company\r\n\
// Ancillary Data Packet Transfer File\r\n\
//\r\n\
// Permission to generate this format is granted provided that\r\n\
//   1. This ANC Transfer file format is used on an as-is basis and no warranty is given, and\r\n\
//   2. This entire descriptive information text is included in a generated .mcc file.\r\n\
//\r\n\
// General file format:\r\n\
//   HH:MM:SS:FF(tab)[Hexadecimal ANC data in groups of 2 characters]\r\n\
//     Hexadecimal data starts with the Ancillary Data Packet DID (Data ID defined in S291M)\r\n\
//       and concludes with the Check Sum following the User Data Words.\r\n\
//     Each time code line must contain at most one complete ancillary data packet.\r\n\
//     To transfer additional ANC Data successive lines may contain identical time code.\r\n\
//     Time Code Rate=[24, 25, 30, 30DF, 50, 60]\r\n\
//\r\n\
//   ANC data bytes may be represented by one ASCII character according to the following schema:\r\n\
//     G  FAh 00h 00h\r\n\
//     H  2 x (FAh 00h 00h)\r\n\
//     I  3 x (FAh 00h 00h)\r\n\
//     J  4 x (FAh 00h 00h)\r\n\
//     K  5 x (FAh 00h 00h)\r\n\
//     L  6 x (FAh 00h 00h)\r\n\
//     M  7 x (FAh 00h 00h)\r\n\
//     N  8 x (FAh 00h 00h)\r\n\
//     O  9 x (FAh 00h 00h)\r\n\
//     P  FBh 80h 80h\r\n\
//     Q  FCh 80h 80h\r\n\
//     R  FDh 80h 80h\r\n\
//     S  96h 69h\r\n\
//     T  61h 01h\r\n\
//     U  E1h 00h 00h 00h\r\n\
//     Z  00h\r\n\
//\r\n\
///////////////////////////////////////////////////////////////////////////////////\r\n\
\r\n\
UUID=14720C04-857D-40E2-86FC-F080DE44CE74\r\n\
Creation Program=GStreamer MCC Encoder {}\r\n\
Creation Date=Thursday, December 27, 2018\r\n\
Creation Time=17:34:47\r\n\
Time Code Rate=30DF\r\n\
\r\n\
11:12:13;14	T52S524F67ZZ72F4QRFF0222FE8CFFOM739181656E67817FFF74ZZ1CZ\r\n", env!("CARGO_PKG_VERSION"));

    let mut h = gst_check::Harness::new("mccenc");
    {
        let enc = h.element().expect("could not create encoder");
        enc.set_property("uuid", &"14720C04-857D-40E2-86FC-F080DE44CE74")
            .unwrap();
        enc.set_property(
            "creation-date",
            &glib::DateTime::new_utc(2018, 12, 27, 17, 34, 47.0).unwrap(),
        )
        .unwrap();
    }

    h.set_src_caps_str(
        "closedcaption/x-cea-708, format=(string)cdp, framerate=(fraction)30000/1001",
    );

    let tc = gst_video::ValidVideoTimeCode::new(
        gst::Fraction::new(30000, 1001),
        None,
        gst_video::VideoTimeCodeFlags::DROP_FRAME,
        11,
        12,
        13,
        14,
        0,
    )
    .unwrap();

    let buf = {
        let mut buf = gst::Buffer::from_mut_slice(Vec::from(input));
        let buf_ref = buf.get_mut().unwrap();
        gst_video::VideoTimeCodeMeta::add(buf_ref, &tc);
        buf_ref.set_pts(gst::ClockTime::from_seconds(0));
        buf
    };

    assert_eq!(h.push(buf), Ok(gst::FlowSuccess::Ok));
    h.push_event(gst::event::Eos::new());

    let buf = h.pull().expect("Couldn't pull buffer");
    let timecode = buf
        .meta::<gst_video::VideoTimeCodeMeta>()
        .expect("No timecode for buffer")
        .tc();
    assert_eq!(timecode, tc);

    let pts = buf.pts().unwrap();
    assert_eq!(pts, gst::ClockTime::ZERO);

    let map = buf.map_readable().expect("Couldn't map buffer readable");
    assert_eq!(
        std::str::from_utf8(map.as_ref()),
        Ok(expected_output.as_str()),
    );
}
<|endoftext|>


<commit_before><commit_msg>Added error.rs file<commit_after>use std::error::Error;
use std::convert::From;
use std::fmt;
use diesel::result::Error as DieselError;
use rocket::http::Status;
use rocket::response::{Response, Responder};
use rocket::Request;

#[derive(Debug)]
pub enum ApiError {
    NotFound,
    InternalServerError,
}

impl fmt:Display for ApiError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ApiError::NotFound => f.write_str("NotFound"),
            ApiError::InternalServerError => f.write_str("InternalServerError"),

        }
    }
}

impl From<DieselError> for ApiError {
    fn from(e: DieselError) -> Self {
        match e {
            DieselError::NotFound => ApiError::NotFound,
            _ => ApiError::InternalServerError,
        }
    }
}

impl Error for ApiError {
    fn description(&self) -> &str {
        match *self {
            ApiError::NotFound => "Record not found",
            ApiError::InternalServerError => "Internal server error",
        }
    }
}

impl<'r> Responder<'r> for ApiError {
    fn respond_to(self, _request: &Request) -> Result<Response<'r>, Status> {
        match self {
            ApiError::NotFound => Err(Status::NotFound),
            _=> Err(Status::InternalServerError),
        }
    }
}
<|endoftext|>


<commit_before>use clap::crate_version;

use std::env;
use std::path::{Path, PathBuf};

use clap::{App, AppSettings, ArgMatches, SubCommand};

use mdbook::errors::Result as Result3;
use mdbook::MDBook;

use failure::Error;
#[cfg(feature = "linkcheck")]
use mdbook::renderer::RenderContext;
#[cfg(feature = "linkcheck")]
use mdbook_linkcheck::{self, errors::BrokenLinks};

fn main() {
    let d_message = "-d, --dest-dir=[dest-dir]
'The output directory for your book{n}(Defaults to ./book when omitted)'";
    let dir_message = "[dir]
'A directory for your book{n}(Defaults to Current Directory when omitted)'";

    let matches = App::new("rustbook")
        .about("Build a book with mdBook")
        .author("Steve Klabnik <steve@steveklabnik.com>")
        .version(&*format!("v{}", crate_version!()))
        .setting(AppSettings::SubcommandRequired)
        .subcommand(
            SubCommand::with_name("build")
                .about("Build the book from the markdown files")
                .arg_from_usage(d_message)
                .arg_from_usage(dir_message),
        )
        .subcommand(
            SubCommand::with_name("linkcheck")
                .about("Run linkcheck with mdBook 3")
                .arg_from_usage(dir_message),
        )
        .get_matches();

    // Check which subcomamnd the user ran...
    match matches.subcommand() {
        ("build", Some(sub_matches)) => {
            if let Err(e) = build(sub_matches) {
                eprintln!("Error: {}", e);

                for cause in e.iter().skip(1) {
                    eprintln!("\tCaused By: {}", cause);
                }

                ::std::process::exit(101);
            }
        }
        ("linkcheck", Some(sub_matches)) => {
            if let Err(err) = linkcheck(sub_matches) {
                eprintln!("Error: {}", err);

                // HACK: ignore timeouts
                let actually_broken = {
                    #[cfg(feature = "linkcheck")]
                    {
                        err.downcast::<BrokenLinks>()
                            .unwrap_or(false)
                            .links()
                            .iter()
                            .inspect(|cause| eprintln!("\tCaused By: {}", cause))
                            .any(|cause| !cause.contains("timed out"));
                    }

                    #[cfg(not(feature = "linkcheck"))]
                    {
                        false
                    }
                };

                if actually_broken {
                    std::process::exit(101);
                } else {
                    std::process::exit(0);
                }
            }
        }
        (_, _) => unreachable!(),
    };
}

#[cfg(feature = "linkcheck")]
pub fn linkcheck(args: &ArgMatches<'_>) -> Result<(), Error> {
    let book_dir = get_book_dir(args);
    let book = MDBook::load(&book_dir).unwrap();
    let cfg = book.config;
    let render_ctx = RenderContext::new(&book_dir, book.book, cfg, &book_dir);

    mdbook_linkcheck::check_links(&render_ctx)
}

#[cfg(not(feature = "linkcheck"))]
pub fn linkcheck(_args: &ArgMatches<'_>) -> Result<(), Error> {
    println!("mdbook-linkcheck is disabled.");
    Ok(())
}

// Build command implementation
pub fn build(args: &ArgMatches<'_>) -> Result3<()> {
    let book_dir = get_book_dir(args);
    let mut book = MDBook::load(&book_dir)?;

    // Set this to allow us to catch bugs in advance.
    book.config.build.create_missing = false;

    if let Some(dest_dir) = args.value_of("dest-dir") {
        book.config.build.build_dir = PathBuf::from(dest_dir);
    }

    book.build()?;

    Ok(())
}

fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}
<commit_msg>actually compiles now?<commit_after>use clap::crate_version;

use std::env;
use std::path::{Path, PathBuf};

use clap::{App, AppSettings, ArgMatches, SubCommand};

use mdbook::errors::Result as Result3;
use mdbook::MDBook;

use failure::Error;
#[cfg(feature = "linkcheck")]
use mdbook::renderer::RenderContext;
#[cfg(feature = "linkcheck")]
use mdbook_linkcheck::{self, errors::BrokenLinks};

fn main() {
    let d_message = "-d, --dest-dir=[dest-dir]
'The output directory for your book{n}(Defaults to ./book when omitted)'";
    let dir_message = "[dir]
'A directory for your book{n}(Defaults to Current Directory when omitted)'";

    let matches = App::new("rustbook")
        .about("Build a book with mdBook")
        .author("Steve Klabnik <steve@steveklabnik.com>")
        .version(&*format!("v{}", crate_version!()))
        .setting(AppSettings::SubcommandRequired)
        .subcommand(
            SubCommand::with_name("build")
                .about("Build the book from the markdown files")
                .arg_from_usage(d_message)
                .arg_from_usage(dir_message),
        )
        .subcommand(
            SubCommand::with_name("linkcheck")
                .about("Run linkcheck with mdBook 3")
                .arg_from_usage(dir_message),
        )
        .get_matches();

    // Check which subcomamnd the user ran...
    match matches.subcommand() {
        ("build", Some(sub_matches)) => {
            if let Err(e) = build(sub_matches) {
                eprintln!("Error: {}", e);

                for cause in e.iter().skip(1) {
                    eprintln!("\tCaused By: {}", cause);
                }

                ::std::process::exit(101);
            }
        }
        ("linkcheck", Some(sub_matches)) => {
            if let Err(err) = linkcheck(sub_matches) {
                eprintln!("Error: {}", err);

                // HACK: ignore timeouts
                let actually_broken = {
                    #[cfg(feature = "linkcheck")]
                    {
                        err.downcast::<BrokenLinks>()
                            .map(|broken_links| {
                                broken_links
                                    .links()
                                    .iter()
                                    .inspect(|cause| eprintln!("\tCaused By: {}", cause))
                                    .any(|cause| !format!("{}", cause).contains("timed out"))
                            })
                            .unwrap_or(false)
                    }

                    #[cfg(not(feature = "linkcheck"))]
                    {
                        false
                    }
                };

                if actually_broken {
                    std::process::exit(101);
                } else {
                    std::process::exit(0);
                }
            }
        }
        (_, _) => unreachable!(),
    };
}

#[cfg(feature = "linkcheck")]
pub fn linkcheck(args: &ArgMatches<'_>) -> Result<(), Error> {
    let book_dir = get_book_dir(args);
    let book = MDBook::load(&book_dir).unwrap();
    let cfg = book.config;
    let render_ctx = RenderContext::new(&book_dir, book.book, cfg, &book_dir);

    mdbook_linkcheck::check_links(&render_ctx)
}

#[cfg(not(feature = "linkcheck"))]
pub fn linkcheck(_args: &ArgMatches<'_>) -> Result<(), Error> {
    println!("mdbook-linkcheck is disabled.");
    Ok(())
}

// Build command implementation
pub fn build(args: &ArgMatches<'_>) -> Result3<()> {
    let book_dir = get_book_dir(args);
    let mut book = MDBook::load(&book_dir)?;

    // Set this to allow us to catch bugs in advance.
    book.config.build.create_missing = false;

    if let Some(dest_dir) = args.value_of("dest-dir") {
        book.config.build.build_dir = PathBuf::from(dest_dir);
    }

    book.build()?;

    Ok(())
}

fn get_book_dir(args: &ArgMatches<'_>) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}
<|endoftext|>


<commit_before>// Uncomment the following line to disable unstable warnings:
// #![allow(unstable)]

extern crate regex;
use regex::Regex;
use std::ascii::AsciiExt;
// use std::collections;
use std::io;
use std::io::{BufferedReader, File, IoResult};

pub mod config;
pub mod btree_map;

fn do_work(cfg: &config::Config) -> IoResult<()> {
    // Open input and output files
    let mut readers = vec![];
    if cfg.input.is_empty() {
        readers.push(BufferedReader::new(Box::new(io::stdin()) as Box<Reader>));
    } else {
        for name in cfg.input.iter() {
            let file = try!(File::open(&Path::new(name.as_slice())));
            readers.push(BufferedReader::new(Box::new(file) as Box<Reader>));
        }
    }
    let mut writer = match cfg.output {
        Some(ref x) => { Box::new(try!(File::create(&Path::new(x.as_slice())))) as Box<Writer> }
        None => { Box::new(io::stdout()) as Box<Writer> }
    };

    // Parse words
    // let mut map = collections::BTreeMap::<String, u32>::new();
    let mut map = btree_map::BTreeMap::<String, u32>::new();
    let re = Regex::new(r"\w+").unwrap();
    for reader in readers.iter_mut() {
        // let mut buf_reader = BufferedReader::new(*reader);
        for line in reader.lines() {
            for caps in re.captures_iter(line.unwrap().as_slice()) {
                if let Some(cap) = caps.at(0) {
                    let word = match cfg.ignore_case {
                        true  => cap.to_ascii_lowercase(),
                        false => cap.to_string(),
                    };
                    let found = match map.get_mut(&word) {
                        Some(count) => { *count = *count + 1; true }
                        None => false,
                    };
                    if !found {
                        map.insert(word, 1);
                    }
                }
            }
        }
    }

    // Write counts
    for (k, v) in map.iter() {
        let line = format!("{}\t{}\n", v, k);
        try!(writer.write(line.as_bytes()));
    }
    Ok(())
}

#[cfg(not(test))]
fn main() {
    let cfg = match config::get_config(std::os::args()) {
        Ok(c) => c,
        Err(usage) => {
            println!("{}", usage);
            return;
        }
    };
    do_work(&cfg).unwrap();
}

#[test]
fn test_do_work() {
    let bad_filename = "/..no-such-file..";
    let cfg = config::Config {
        input: vec![bad_filename.to_string()],
        output: None,
        ignore_case: false,
    };
    match do_work(&cfg) {
        // should fail for bad input file (couldn't open file ... path=/..no-such-file..; ...)
        Err(e) => assert!(format!("{}", e).as_slice().contains(bad_filename), format!("{}", e)),
        _ => panic!()
    }
}

<commit_msg>Remove unnecessary comments<commit_after>// Uncomment the following line to disable unstable warnings:
// #![allow(unstable)]

extern crate regex;
use regex::Regex;
use std::ascii::AsciiExt;
// use std::collections;
use std::io;
use std::io::{BufferedReader, File, IoResult};

pub mod config;
pub mod btree_map;

fn do_work(cfg: &config::Config) -> IoResult<()> {
    // Open input and output files
    let mut readers = vec![];
    if cfg.input.is_empty() {
        readers.push(BufferedReader::new(Box::new(io::stdin()) as Box<Reader>));
    } else {
        for name in cfg.input.iter() {
            let file = try!(File::open(&Path::new(name.as_slice())));
            readers.push(BufferedReader::new(Box::new(file) as Box<Reader>));
        }
    }
    let mut writer = match cfg.output {
        Some(ref x) => { Box::new(try!(File::create(&Path::new(x.as_slice())))) as Box<Writer> }
        None => { Box::new(io::stdout()) as Box<Writer> }
    };

    // Parse words
    // let mut map = collections::BTreeMap::<String, u32>::new();
    let mut map = btree_map::BTreeMap::<String, u32>::new();
    let re = Regex::new(r"\w+").unwrap();
    for reader in readers.iter_mut() {
        for line in reader.lines() {
            for caps in re.captures_iter(line.unwrap().as_slice()) {
                if let Some(cap) = caps.at(0) {
                    let word = match cfg.ignore_case {
                        true  => cap.to_ascii_lowercase(),
                        false => cap.to_string(),
                    };
                    let found = match map.get_mut(&word) {
                        Some(count) => { *count = *count + 1; true }
                        None => false,
                    };
                    if !found {
                        map.insert(word, 1);
                    }
                }
            }
        }
    }

    // Write counts
    for (k, v) in map.iter() {
        let line = format!("{}\t{}\n", v, k);
        try!(writer.write(line.as_bytes()));
    }
    Ok(())
}

#[cfg(not(test))]
fn main() {
    let cfg = match config::get_config(std::os::args()) {
        Ok(c) => c,
        Err(usage) => {
            println!("{}", usage);
            return;
        }
    };
    do_work(&cfg).unwrap();
}

#[test]
fn test_do_work() {
    let bad_filename = "/..no-such-file..";
    let cfg = config::Config {
        input: vec![bad_filename.to_string()],
        output: None,
        ignore_case: false,
    };
    match do_work(&cfg) {
        // should fail for bad input file (couldn't open file ... path=/..no-such-file..; ...)
        Err(e) => assert!(format!("{}", e).as_slice().contains(bad_filename), format!("{}", e)),
        _ => panic!()
    }
}

<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use WindowWrapper;
use base64;
use image::load as load_piston_image;
use image::png::PNGEncoder;
use image::{ColorType, ImageFormat};
use parse_function::parse_function;
use png::save_flipped;
use std::cmp;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::sync::mpsc::{channel, Receiver, Sender};
use webrender_traits::*;
use wrench::{Wrench, WrenchThing};
use yaml_frame_reader::YamlFrameReader;

pub struct ReftestOptions {
    // These override values that are lower.
    pub allow_max_difference: usize,
    pub allow_num_differences: usize,
}

impl ReftestOptions {
    pub fn default() -> Self {
        ReftestOptions {
            allow_max_difference: 0,
            allow_num_differences: 0,
        }
    }
}

pub enum ReftestOp {
    Equal,
    NotEqual,
}
pub struct Reftest {
    op: ReftestOp,
    test: PathBuf,
    reference: PathBuf,
    max_difference: usize,
    num_differences: usize,
}

struct ReftestImage {
    data: Vec<u8>,
    size: DeviceUintSize,
}
enum ReftestImageComparison {
    Equal,
    NotEqual { max_difference: usize, count_different: usize },
}

impl ReftestImage {
    fn compare(&self, other: &ReftestImage) -> ReftestImageComparison {
        assert_eq!(self.size, other.size);
        assert_eq!(self.data.len(), other.data.len());
        assert_eq!(self.data.len() % 4, 0);

        let mut count = 0;
        let mut max = 0;

        for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) {
            if a != b {
                let pixel_max = a.iter()
                                 .zip(b.iter())
                                 .map(|(x, y)| (*x as isize - *y as isize).abs() as usize)
                                 .max().unwrap();

                count += 1;
                max = cmp::max(max, pixel_max);
            }
        }

        if count != 0 {
            ReftestImageComparison::NotEqual {
                max_difference: max,
                count_different: count,
            }
        } else {
            ReftestImageComparison::Equal
        }
    }

    fn create_data_uri(mut self) -> String {
        let width = self.size.width;
        let height = self.size.height;

        // flip image vertically (texture is upside down)
        let orig_pixels = self.data.clone();
        let stride = width as usize * 4;
        for y in 0..height as usize {
            let dst_start = y * stride;
            let src_start = (height as usize - y - 1) * stride;
            let src_slice = &orig_pixels[src_start .. src_start + stride];
            (&mut self.data[dst_start .. dst_start + stride]).clone_from_slice(&src_slice[..stride]);
        }

        let mut png: Vec<u8> = vec![];
        {
            let encoder = PNGEncoder::new(&mut png);
            encoder.encode(&self.data[..],
                            width,
                            height,
                            ColorType::RGBA(8)).expect("Unable to encode PNG!");
        }
        let png_base64 = base64::encode(&png);
        format!("data:image/png;base64,{}", png_base64)
    }
}

struct ReftestManifest {
    reftests: Vec<Reftest>,
}
impl ReftestManifest {
    fn new(manifest: &Path, options: &ReftestOptions) -> ReftestManifest {
        let dir = manifest.parent().unwrap();
        let f = File::open(manifest).expect(&format!("couldn't open manifest: {}", manifest.display()));
        let file = BufReader::new(&f);

        let mut reftests = Vec::new();

        for line in file.lines() {
            let l = line.unwrap();

            // strip the comments
            let s = &l[0..l.find('#').unwrap_or(l.len())];
            let s = s.trim();
            if s.is_empty() {
                continue;
            }

            let items: Vec<&str> = s.split_whitespace().collect();

            match items[0] {
                "include" => {
                    let include = dir.join(items[1]);

                    reftests.append(&mut ReftestManifest::new(include.as_path(), options).reftests);
                }
                item_str => {
                    // If the first item is "fuzzy(<val>,<count>)" the positions of the operator
                    // and paths in the array are offset.
                    // TODO: This is simple but not great because it does not support having spaces
                    // in the fuzzy syntax, like between the arguments.
                    let (max, count, offset) =  if item_str.starts_with("fuzzy(") {
                        let (_, args, _) = parse_function(item_str);
                        (args[0].parse().unwrap(),  args[1].parse().unwrap(), 1)
                    } else {
                        (0, 0, 0)
                    };

                    reftests.push(Reftest {
                        op: parse_operator(items[offset]).expect("unexpected match operator"),
                        test: dir.join(items[offset + 1]),
                        reference: dir.join(items[offset + 2]),
                        max_difference: cmp::max(max, options.allow_max_difference),
                        num_differences: cmp::max(count, options.allow_num_differences),
                    });
                }
            };
        }

        ReftestManifest {
            reftests: reftests
        }
    }

    fn find(&self, prefix: &Path) -> Vec<&Reftest> {
        self.reftests.iter().filter(|x| {
            x.test.starts_with(prefix) || x.reference.starts_with(prefix)
        }).collect()
    }
}

fn parse_operator(op_str: &str) -> Option<ReftestOp> {
    match op_str {
        "==" => Some(ReftestOp::Equal),
        "!=" => Some(ReftestOp::NotEqual),
        _ => None,
    }
}

pub struct ReftestHarness<'a> {
    wrench: &'a mut Wrench,
    window: &'a mut WindowWrapper,
    rx: Receiver<()>,
}
impl<'a> ReftestHarness<'a> {
    pub fn new(wrench: &'a mut Wrench,
               window: &'a mut WindowWrapper) -> ReftestHarness<'a>
    {
        // setup a notifier so we can wait for frames to be finished
        struct Notifier {
            tx: Sender<()>,
        };
        impl RenderNotifier for Notifier {
            fn new_frame_ready(&mut self) {
                self.tx.send(()).unwrap();
            }
            fn new_scroll_frame_ready(&mut self, _composite_needed: bool) {}
        }
        let (tx, rx) = channel();
        wrench.renderer.set_render_notifier(Box::new(Notifier { tx: tx }));

        ReftestHarness {
            wrench,
            window,
            rx,
        }
    }

    pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) {
        let manifest = ReftestManifest::new(base_manifest, options);
        let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new()));

        let mut total_passing = 0;
        let mut total_failing = 0;

        for t in reftests {
            if self.run_reftest(t) {
                total_passing += 1;
            } else {
                total_failing += 1;
            }
        }

        println!("REFTEST INFO | {} passing, {} failing", total_passing, total_failing);

        // panic here so that we fail CI
        assert!(total_failing <= 0);
    }

    fn run_reftest(&mut self, t: &Reftest) -> bool {
        let name = match t.op {
            ReftestOp::Equal => format!("{} == {}", t.test.display(), t.reference.display()),
            ReftestOp::NotEqual => format!("{} != {}", t.test.display(), t.reference.display()),
        };

        println!("REFTEST {}", name);

        let window_size = DeviceUintSize::new(self.window.get_inner_size_pixels().0,
                                              self.window.get_inner_size_pixels().1);
        let reference = match t.reference.extension().unwrap().to_str().unwrap() {
            "yaml" => self.render_yaml(t.reference.as_path(), window_size),
            "png" => self.load_image(t.reference.as_path(), ImageFormat::PNG),
            other => panic!("Unknown reftest extension: {}", other),
        };
        // the reference can be smaller than the window size,
        // in which case we only compare the intersection
        let test = self.render_yaml(t.test.as_path(), reference.size);
        let comparison = test.compare(&reference);

        match (&t.op, comparison) {
            (&ReftestOp::Equal, ReftestImageComparison::Equal) => true,
            (&ReftestOp::Equal,
              ReftestImageComparison::NotEqual { max_difference, count_different }) => {
                if max_difference > t.max_difference || count_different > t.num_differences {
                    println!("{} | {} | {}: {}, {}: {}",
                             "REFTEST TEST-UNEXPECTED-FAIL", name,
                             "image comparison, max difference", max_difference,
                             "number of differing pixels", count_different);
                    println!("REFTEST   IMAGE 1 (TEST): {}", test.create_data_uri());
                    println!("REFTEST   IMAGE 2 (REFERENCE): {}", reference.create_data_uri());
                    println!("REFTEST TEST-END | {}", name);

                    false
                } else {
                    true
                }
            },
            (&ReftestOp::NotEqual, ReftestImageComparison::Equal) => {
                println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", name);
                println!("REFTEST TEST-END | {}", name);

                false
            },
            (&ReftestOp::NotEqual, ReftestImageComparison::NotEqual { .. }) => true,
        }
    }

    fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage {
        let file = BufReader::new(File::open(filename).unwrap());
        let img_raw = load_piston_image(file, format).unwrap();
        let img = img_raw.flipv().to_rgba();
        let size = img.dimensions();
        ReftestImage {
            data: img.into_raw(),
            size: DeviceUintSize::new(size.0, size.1)
        }
    }

    fn render_yaml(&mut self, filename: &Path, size: DeviceUintSize) -> ReftestImage {
        let mut reader = YamlFrameReader::new(filename);
        reader.do_frame(self.wrench);

        // wait for the frame
        self.rx.recv().unwrap();
        self.wrench.render();

        let window_size = self.window.get_inner_size_pixels();
        assert!(size.width <= window_size.0 && size.height <= window_size.1);

        // taking the bottom left sub-rectangle
        let rect = DeviceUintRect::new(DeviceUintPoint::new(0, window_size.1 - size.height),
                                       size);
        let pixels = self.wrench.renderer.read_pixels_rgba8(rect);
        self.window.swap_buffers();

        let write_debug_images = false;
        if write_debug_images {
            let debug_path = filename.with_extension("yaml.png");
            save_flipped(debug_path, &pixels, size);
        }

        ReftestImage {
            data: pixels,
            size,
        }
    }
}
<commit_msg>Auto merge of #1414 - mrobinson:wrench-failing-tests, r=kvark<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use WindowWrapper;
use base64;
use image::load as load_piston_image;
use image::png::PNGEncoder;
use image::{ColorType, ImageFormat};
use parse_function::parse_function;
use png::save_flipped;
use std::cmp;
use std::fmt::{Display, Error, Formatter};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::sync::mpsc::{channel, Receiver, Sender};
use webrender_traits::*;
use wrench::{Wrench, WrenchThing};
use yaml_frame_reader::YamlFrameReader;

pub struct ReftestOptions {
    // These override values that are lower.
    pub allow_max_difference: usize,
    pub allow_num_differences: usize,
}

impl ReftestOptions {
    pub fn default() -> Self {
        ReftestOptions {
            allow_max_difference: 0,
            allow_num_differences: 0,
        }
    }
}

pub enum ReftestOp {
    Equal,
    NotEqual,
}

impl Display for ReftestOp {
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        write!(f, "{}",
               match *self {
                   ReftestOp::Equal => "==".to_owned(),
                   ReftestOp::NotEqual => "!=".to_owned(),
                })
    }
}

pub struct Reftest {
    op: ReftestOp,
    test: PathBuf,
    reference: PathBuf,
    max_difference: usize,
    num_differences: usize,
}

impl Display for Reftest {
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        write!(f, "{} {} {}", self.test.display(), self.op, self.reference.display())
    }
}

struct ReftestImage {
    data: Vec<u8>,
    size: DeviceUintSize,
}
enum ReftestImageComparison {
    Equal,
    NotEqual { max_difference: usize, count_different: usize },
}

impl ReftestImage {
    fn compare(&self, other: &ReftestImage) -> ReftestImageComparison {
        assert_eq!(self.size, other.size);
        assert_eq!(self.data.len(), other.data.len());
        assert_eq!(self.data.len() % 4, 0);

        let mut count = 0;
        let mut max = 0;

        for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) {
            if a != b {
                let pixel_max = a.iter()
                                 .zip(b.iter())
                                 .map(|(x, y)| (*x as isize - *y as isize).abs() as usize)
                                 .max().unwrap();

                count += 1;
                max = cmp::max(max, pixel_max);
            }
        }

        if count != 0 {
            ReftestImageComparison::NotEqual {
                max_difference: max,
                count_different: count,
            }
        } else {
            ReftestImageComparison::Equal
        }
    }

    fn create_data_uri(mut self) -> String {
        let width = self.size.width;
        let height = self.size.height;

        // flip image vertically (texture is upside down)
        let orig_pixels = self.data.clone();
        let stride = width as usize * 4;
        for y in 0..height as usize {
            let dst_start = y * stride;
            let src_start = (height as usize - y - 1) * stride;
            let src_slice = &orig_pixels[src_start .. src_start + stride];
            (&mut self.data[dst_start .. dst_start + stride]).clone_from_slice(&src_slice[..stride]);
        }

        let mut png: Vec<u8> = vec![];
        {
            let encoder = PNGEncoder::new(&mut png);
            encoder.encode(&self.data[..],
                            width,
                            height,
                            ColorType::RGBA(8)).expect("Unable to encode PNG!");
        }
        let png_base64 = base64::encode(&png);
        format!("data:image/png;base64,{}", png_base64)
    }
}

struct ReftestManifest {
    reftests: Vec<Reftest>,
}
impl ReftestManifest {
    fn new(manifest: &Path, options: &ReftestOptions) -> ReftestManifest {
        let dir = manifest.parent().unwrap();
        let f = File::open(manifest).expect(&format!("couldn't open manifest: {}", manifest.display()));
        let file = BufReader::new(&f);

        let mut reftests = Vec::new();

        for line in file.lines() {
            let l = line.unwrap();

            // strip the comments
            let s = &l[0..l.find('#').unwrap_or(l.len())];
            let s = s.trim();
            if s.is_empty() {
                continue;
            }

            let items: Vec<&str> = s.split_whitespace().collect();

            match items[0] {
                "include" => {
                    let include = dir.join(items[1]);

                    reftests.append(&mut ReftestManifest::new(include.as_path(), options).reftests);
                }
                item_str => {
                    // If the first item is "fuzzy(<val>,<count>)" the positions of the operator
                    // and paths in the array are offset.
                    // TODO: This is simple but not great because it does not support having spaces
                    // in the fuzzy syntax, like between the arguments.
                    let (max, count, offset) =  if item_str.starts_with("fuzzy(") {
                        let (_, args, _) = parse_function(item_str);
                        (args[0].parse().unwrap(),  args[1].parse().unwrap(), 1)
                    } else {
                        (0, 0, 0)
                    };

                    reftests.push(Reftest {
                        op: parse_operator(items[offset]).expect("unexpected match operator"),
                        test: dir.join(items[offset + 1]),
                        reference: dir.join(items[offset + 2]),
                        max_difference: cmp::max(max, options.allow_max_difference),
                        num_differences: cmp::max(count, options.allow_num_differences),
                    });
                }
            };
        }

        ReftestManifest {
            reftests: reftests
        }
    }

    fn find(&self, prefix: &Path) -> Vec<&Reftest> {
        self.reftests.iter().filter(|x| {
            x.test.starts_with(prefix) || x.reference.starts_with(prefix)
        }).collect()
    }
}

fn parse_operator(op_str: &str) -> Option<ReftestOp> {
    match op_str {
        "==" => Some(ReftestOp::Equal),
        "!=" => Some(ReftestOp::NotEqual),
        _ => None,
    }
}

pub struct ReftestHarness<'a> {
    wrench: &'a mut Wrench,
    window: &'a mut WindowWrapper,
    rx: Receiver<()>,
}
impl<'a> ReftestHarness<'a> {
    pub fn new(wrench: &'a mut Wrench,
               window: &'a mut WindowWrapper) -> ReftestHarness<'a>
    {
        // setup a notifier so we can wait for frames to be finished
        struct Notifier {
            tx: Sender<()>,
        };
        impl RenderNotifier for Notifier {
            fn new_frame_ready(&mut self) {
                self.tx.send(()).unwrap();
            }
            fn new_scroll_frame_ready(&mut self, _composite_needed: bool) {}
        }
        let (tx, rx) = channel();
        wrench.renderer.set_render_notifier(Box::new(Notifier { tx: tx }));

        ReftestHarness {
            wrench,
            window,
            rx,
        }
    }

    pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) {
        let manifest = ReftestManifest::new(base_manifest, options);
        let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new()));

        let mut total_passing = 0;
        let mut failing = Vec::new();

        for t in reftests {
            if self.run_reftest(t) {
                total_passing += 1;
            } else {
                failing.push(t);
            }
        }

        println!("REFTEST INFO | {} passing, {} failing", total_passing, failing.len());

        if !failing.is_empty() {
            println!("\nReftests with unexpected results:");

            for reftest in &failing {
                println!("\t{}", reftest);
            }
        }

        // panic here so that we fail CI
        assert!(failing.is_empty());
    }

    fn run_reftest(&mut self, t: &Reftest) -> bool {
        println!("REFTEST {}", t);

        let window_size = DeviceUintSize::new(self.window.get_inner_size_pixels().0,
                                              self.window.get_inner_size_pixels().1);
        let reference = match t.reference.extension().unwrap().to_str().unwrap() {
            "yaml" => self.render_yaml(t.reference.as_path(), window_size),
            "png" => self.load_image(t.reference.as_path(), ImageFormat::PNG),
            other => panic!("Unknown reftest extension: {}", other),
        };
        // the reference can be smaller than the window size,
        // in which case we only compare the intersection
        let test = self.render_yaml(t.test.as_path(), reference.size);
        let comparison = test.compare(&reference);

        match (&t.op, comparison) {
            (&ReftestOp::Equal, ReftestImageComparison::Equal) => true,
            (&ReftestOp::Equal,
              ReftestImageComparison::NotEqual { max_difference, count_different }) => {
                if max_difference > t.max_difference || count_different > t.num_differences {
                    println!("{} | {} | {}: {}, {}: {}",
                             "REFTEST TEST-UNEXPECTED-FAIL", t,
                             "image comparison, max difference", max_difference,
                             "number of differing pixels", count_different);
                    println!("REFTEST   IMAGE 1 (TEST): {}", test.create_data_uri());
                    println!("REFTEST   IMAGE 2 (REFERENCE): {}", reference.create_data_uri());
                    println!("REFTEST TEST-END | {}", t);

                    false
                } else {
                    true
                }
            },
            (&ReftestOp::NotEqual, ReftestImageComparison::Equal) => {
                println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", t);
                println!("REFTEST TEST-END | {}", t);

                false
            },
            (&ReftestOp::NotEqual, ReftestImageComparison::NotEqual { .. }) => true,
        }
    }

    fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage {
        let file = BufReader::new(File::open(filename).unwrap());
        let img_raw = load_piston_image(file, format).unwrap();
        let img = img_raw.flipv().to_rgba();
        let size = img.dimensions();
        ReftestImage {
            data: img.into_raw(),
            size: DeviceUintSize::new(size.0, size.1)
        }
    }

    fn render_yaml(&mut self, filename: &Path, size: DeviceUintSize) -> ReftestImage {
        let mut reader = YamlFrameReader::new(filename);
        reader.do_frame(self.wrench);

        // wait for the frame
        self.rx.recv().unwrap();
        self.wrench.render();

        let window_size = self.window.get_inner_size_pixels();
        assert!(size.width <= window_size.0 && size.height <= window_size.1);

        // taking the bottom left sub-rectangle
        let rect = DeviceUintRect::new(DeviceUintPoint::new(0, window_size.1 - size.height),
                                       size);
        let pixels = self.wrench.renderer.read_pixels_rgba8(rect);
        self.window.swap_buffers();

        let write_debug_images = false;
        if write_debug_images {
            let debug_path = filename.with_extension("yaml.png");
            save_flipped(debug_path, &pixels, size);
        }

        ReftestImage {
            data: pixels,
            size,
        }
    }
}
<|endoftext|>


<commit_before>#![warn(clippy::trailing_zero_sized_array_without_repr)]
#![feature(const_generics_defaults)]

// Do lint:

struct RarelyUseful {
    field: i32,
    last: [usize; 0],
}

struct OnlyField {
    first_and_last: [usize; 0],
}

struct GenericArrayType<T> {
    field: i32,
    last: [T; 0],
}

#[must_use]
struct OnlyAnotherAttributeMustUse {
    field: i32,
    last: [usize; 0],
}

// NOTE: Unfortunately the attribute isn't included in the lint output. I'm not sure how to make it
// show up.
#[derive(Debug)]
struct OnlyAnotherAttributeDerive {
    field: i32,
    last: [usize; 0],
}

const ZERO: usize = 0;
struct ZeroSizedWithConst {
    field: i32,
    last: [usize; ZERO],
}

#[allow(clippy::eq_op)]
const fn compute_zero() -> usize {
    (4 + 6) - (2 * 5)
}
struct ZeroSizedWithConstFunction {
    field: i32,
    last: [usize; compute_zero()],
}

struct ZeroSizedArrayWrapper([usize; 0]);

struct TupleStruct(i32, [usize; 0]);

struct LotsOfFields {
    f1: u32,
    f2: u32,
    f3: u32,
    f4: u32,
    f5: u32,
    f6: u32,
    f7: u32,
    f8: u32,
    f9: u32,
    f10: u32,
    f11: u32,
    f12: u32,
    f13: u32,
    f14: u32,
    f15: u32,
    f16: u32,
    last: [usize; 0],
}

// Don't lint

#[repr(C)]
struct GoodReason {
    field: i32,
    last: [usize; 0],
}

#[repr(C)]
struct OnlyFieldWithReprC {
    first_and_last: [usize; 0],
}

struct NonZeroSizedArray {
    field: i32,
    last: [usize; 1],
}

struct NotLastField {
    f1: u32,
    zero_sized: [usize; 0],
    last: i32,
}

const ONE: usize = 1;
struct NonZeroSizedWithConst {
    field: i32,
    last: [usize; ONE],
}

#[derive(Debug)]
#[repr(C)]
struct OtherAttributesDerive {
    field: i32,
    last: [usize; 0],
}

#[must_use]
#[repr(C)]
struct OtherAttributesMustUse {
    field: i32,
    last: [usize; 0],
}

#[repr(packed)]
struct ReprPacked {
    field: i32,
    last: [usize; 0],
}

#[repr(C, packed)]
struct ReprCPacked {
    field: i32,
    last: [usize; 0],
}

#[repr(align(64))]
struct ReprAlign {
    field: i32,
    last: [usize; 0],
}
#[repr(C, align(64))]
struct ReprCAlign {
    field: i32,
    last: [usize; 0],
}

// NOTE: because of https://doc.rust-lang.org/stable/reference/type-layout.html#primitive-representation-of-enums-with-fields and I'm not sure when in the compilation pipeline that would happen
#[repr(C)]
enum DontLintAnonymousStructsFromDesuraging {
    A(u32),
    B(f32, [u64; 0]),
    C { x: u32, y: [u64; 0] },
}

#[repr(C)]
struct TupleStructReprC(i32, [usize; 0]);

type NamedTuple = (i32, [usize; 0]);

#[rustfmt::skip] // [rustfmt#4995](https://github.com/rust-lang/rustfmt/issues/4995)
struct ConstParamZeroDefault<const N: usize = 0> {
    field: i32,
    last: [usize; N],
}

struct ConstParamNoDefault<const N: usize> {
    field: i32,
    last: [usize; N],
}

#[rustfmt::skip]
struct ConstParamNonZeroDefault<const N: usize = 1> {
    field: i32,
    last: [usize; N],
}

type A = ConstParamZeroDefault;
type B = ConstParamZeroDefault<0>;
type C = ConstParamNoDefault<0>;
type D = ConstParamNonZeroDefault<0>;

fn main() {}
<commit_msg>One more test<commit_after>#![warn(clippy::trailing_zero_sized_array_without_repr)]
#![feature(const_generics_defaults)]

// Do lint:

struct RarelyUseful {
    field: i32,
    last: [usize; 0],
}

struct OnlyField {
    first_and_last: [usize; 0],
}

struct GenericArrayType<T> {
    field: i32,
    last: [T; 0],
}

#[must_use]
struct OnlyAnotherAttributeMustUse {
    field: i32,
    last: [usize; 0],
}

// NOTE: Unfortunately the attribute isn't included in the lint output. I'm not sure how to make it
// show up.
#[derive(Debug)]
struct OnlyAnotherAttributeDerive {
    field: i32,
    last: [usize; 0],
}

const ZERO: usize = 0;
struct ZeroSizedWithConst {
    field: i32,
    last: [usize; ZERO],
}

#[allow(clippy::eq_op)]
const fn compute_zero() -> usize {
    (4 + 6) - (2 * 5)
}
struct ZeroSizedWithConstFunction {
    field: i32,
    last: [usize; compute_zero()],
}

struct ZeroSizedArrayWrapper([usize; 0]);

struct TupleStruct(i32, [usize; 0]);

struct LotsOfFields {
    f1: u32,
    f2: u32,
    f3: u32,
    f4: u32,
    f5: u32,
    f6: u32,
    f7: u32,
    f8: u32,
    f9: u32,
    f10: u32,
    f11: u32,
    f12: u32,
    f13: u32,
    f14: u32,
    f15: u32,
    f16: u32,
    last: [usize; 0],
}

// Don't lint

#[repr(C)]
struct GoodReason {
    field: i32,
    last: [usize; 0],
}

#[repr(C)]
struct OnlyFieldWithReprC {
    first_and_last: [usize; 0],
}

struct NonZeroSizedArray {
    field: i32,
    last: [usize; 1],
}

struct NotLastField {
    f1: u32,
    zero_sized: [usize; 0],
    last: i32,
}

const ONE: usize = 1;
struct NonZeroSizedWithConst {
    field: i32,
    last: [usize; ONE],
}

#[derive(Debug)]
#[repr(C)]
struct OtherAttributesDerive {
    field: i32,
    last: [usize; 0],
}

#[must_use]
#[repr(C)]
struct OtherAttributesMustUse {
    field: i32,
    last: [usize; 0],
}

#[repr(packed)]
struct ReprPacked {
    field: i32,
    last: [usize; 0],
}

#[repr(C, packed)]
struct ReprCPacked {
    field: i32,
    last: [usize; 0],
}

#[repr(align(64))]
struct ReprAlign {
    field: i32,
    last: [usize; 0],
}
#[repr(C, align(64))]
struct ReprCAlign {
    field: i32,
    last: [usize; 0],
}

// NOTE: because of https://doc.rust-lang.org/stable/reference/type-layout.html#primitive-representation-of-enums-with-fields and I'm not sure when in the compilation pipeline that would happen
#[repr(C)]
enum DontLintAnonymousStructsFromDesuraging {
    A(u32),
    B(f32, [u64; 0]),
    C { x: u32, y: [u64; 0] },
}

#[repr(C)]
struct TupleStructReprC(i32, [usize; 0]);

type NamedTuple = (i32, [usize; 0]);

#[rustfmt::skip] // [rustfmt#4995](https://github.com/rust-lang/rustfmt/issues/4995)
struct ConstParamZeroDefault<const N: usize = 0> {
    field: i32,
    last: [usize; N],
}

struct ConstParamNoDefault<const N: usize> {
    field: i32,
    last: [usize; N],
}

#[rustfmt::skip]
struct ConstParamNonZeroDefault<const N: usize = 1> {
    field: i32,
    last: [usize; N],
}

struct TwoGenericParams<T, const N: usize> {
    field: i32,
    last: [T; N],
}

type A = ConstParamZeroDefault;
type B = ConstParamZeroDefault<0>;
type C = ConstParamNoDefault<0>;
type D = ConstParamNonZeroDefault<0>;

fn main() {}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate test;
extern crate scribe;

use test::Bencher;
use scribe::Workspace;
use std::path::PathBuf;

#[bench]
fn bench_tokens(b: &mut Bencher) {
    // Create a workspace with this benchmark test as a buffer.
    let mut workspace = Workspace::new(Path::new("."));
    let path = Path::new("benches/buffer_tokens.rs");
    workspace.open_buffer(path);

    // Benchmark the buffer's tokens method.
    let buffer = workspace.current_buffer().unwrap();
    b.iter(|| {
        let mut tokens = buffer.tokens().unwrap();

        // Exhaust the token iterator.
        for _ in tokens.iter() {}
    });
}
<commit_msg>Update benchmark test to work with workspace result<commit_after>#![feature(test)]

extern crate test;
extern crate scribe;

use test::Bencher;
use scribe::Workspace;
use std::path::Path;

#[bench]
fn bench_tokens(b: &mut Bencher) {
    // Create a workspace with this benchmark test as a buffer.
    let mut workspace = Workspace::new(Path::new(".")).unwrap();
    let path = Path::new("benches/buffer_tokens.rs");
    workspace.open_buffer(path);

    // Benchmark the buffer's tokens method.
    let buffer = workspace.current_buffer().unwrap();
    b.iter(|| {
        let tokens = buffer.tokens().unwrap();

        // Exhaust the token iterator.
        for _ in tokens.iter() {}
    });
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! This module provides algorithms for accessing and updating a Merkle Accumulator structure
//! persisted in a key-value store. Note that this doesn't write to the storage directly, rather,
//! it reads from it via the `HashReader` trait and yields writes via an in memory `HashMap`.
//!
//! # Merkle Accumulator
//! Given an ever growing (append only) series of "leaf" hashes, we construct an evolving Merkle
//! Tree for which proofs of inclusion/exclusion of a leaf hash at a leaf index in a snapshot
//! of the tree (represented by root hash) can be given.
//!
//! # Leaf Nodes
//! Leaf nodes carry hash values to be stored and proved. They are only appended to the tree but
//! never deleted or updated.
//!
//! # Internal Nodes
//! A non-leaf node carries the hash value derived from both its left and right children.
//!
//! # Placeholder Nodes
//! To make sure each Leaf node has a Merkle Proof towards the root, placeholder nodes are added so
//! that along the route from a leaf to the root, each node has a sibling. Placeholder nodes have
//! the hash value `ACCUMULATOR_PLACEHOLDER_HASH`
//!
//! A placeholder node can appear as either a Leaf node or a non-Leaf node, but there is at most one
//! placeholder leaf at any time.
//!
//! # Frozen Nodes & Non-frozen Nodes
//! As leaves are added to the tree, placeholder nodes get replaced by non-placeholder nodes, and
//! when a node has all its descendants being non-placeholder, it becomes "Frozen" -- its hash value
//! won't change again in the event of new leaves being added. All leaves appended (not counting the
//! one possible placeholder leaf) are by definition Frozen.
//!
//! Other nodes, which have one or more placeholder descendants are Non-Frozen. As new elements are
//! appended to the accumulator the hash value of these nodes will change.
//!
//! # Leaf Count
//! Given a count of the number of leaves in a Merkle Accumulator it is possible to determine the
//! shape of the accumulator -- which nodes are filled and which nodes are placeholder nodes.
//!
//! Example:
//! Logical view of a Merkle Accumulator with 5 leaves:
//! ```text
//!            Non-fzn
//!           /       \
//!          /         \
//!         /           \
//!       Fzn2         Non-fzn
//!      /   \           /   \
//!     /     \         /     \
//!    Fzn1    Fzn3  Non-fzn  [Placeholder]
//!   /  \    /  \    /    \
//!  L0  L1  L2  L3 L4   [Placeholder]
//! ```
//!
//! # Position and Physical Representation
//! As a Merkle Accumulator tree expands to the right and upwards, we number newly frozen nodes
//! monotonically. One way to do it is simply to use in-order index of nodes, and this is what
//! we do for the in-memory representation. We call the stated numbers identifying nodes below
//! simply "Position", and unless otherwise stated, this is the in-order position.
//!
//! For writing to disk however, we write all the children of a node before the parent.
//! Thus for disk write order, it is more convenient to use the post-order position as an index.
//! And with that we can map a Merkle Accumulator into a key-value storage: key is the post-order
//! position of a node, and the value is hash value it carries.
//!
//! We store only Frozen nodes, and generate non-Frozen nodes on the fly when accessing the tree.
//! This way, the physical representation of the tree is append-only, i.e. once written to physical
//! storage, nodes won't be either modified or deleted.
//!
//! Here is what we persist for the logical tree in the above example:
//!
//! ```text
//!          Fzn2(6)
//!         /      \
//!        /        \
//!    Fzn1(2)       Fzn3(5)
//!   /     \       /     \
//!  L0(0)  L1(1)  L2(3)  L3(4)  L4(7)
//! ```
//!
//! When the next leaf node is persisted, the physical representation will be:
//!
//! ```text
//!          Fzn2(6)
//!         /      \
//!        /        \
//!    Fzn1(2)       Fzn3(5)       Fzn4(9)
//!   /     \       /     \       /      \
//!  L0(0)  L1(1)  L2(3)  L3(4)  L4(7)   L5(8)
//! ```
//!
//! The numbering corresponds to the post-order traversal of the tree.
//!
//! To think in key-value pairs:
//! ```text
//! |<-key->|<--value-->|
//! |   0   | hash_L0   |
//! |   1   | hash_L1   |
//! |   2   | hash_Fzn1 |
//! |  ...  |   ...     |
//! ```

use crypto::hash::{CryptoHash, CryptoHasher, HashValue, ACCUMULATOR_PLACEHOLDER_HASH};
use failure::prelude::*;
use std::marker::PhantomData;
use types::proof::{
    position::{FrozenSubTreeIterator, FrozenSubtreeSiblingIterator, Position},
    AccumulatorConsistencyProof, AccumulatorProof, MerkleTreeInternalNode,
};

/// Defines the interface between `MerkleAccumulator` and underlying storage.
pub trait HashReader {
    /// Return `HashValue` carried by the node at `Position`.
    fn get(&self, position: Position) -> Result<HashValue>;
}

/// A `Node` in a `MerkleAccumulator` tree is a `HashValue` at a `Position`
type Node = (Position, HashValue);

/// In this live Merkle Accumulator algorithms.
pub struct MerkleAccumulator<R, H> {
    reader: PhantomData<R>,
    hasher: PhantomData<H>,
}

impl<R, H> MerkleAccumulator<R, H>
where
    R: HashReader,
    H: CryptoHasher,
{
    /// Given an existing Merkle Accumulator (represented by `num_existing_leaves` and a `reader`
    /// that is able to fetch all existing frozen nodes), and a list of leaves to be appended,
    /// returns the result root hash and new nodes to be frozen.
    pub fn append(
        reader: &R,
        num_existing_leaves: usize,
        new_leaves: &[HashValue],
    ) -> Result<(HashValue, Vec<Node>)> {
        MerkleAccumulatorView::<R, H>::new(reader, num_existing_leaves).append(new_leaves)
    }

    /// Get proof of inclusion of the leaf at `leaf_index` in this Merkle Accumulator of
    /// `num_leaves` leaves in total. Siblings are read via `reader` (or generated dynamically
    /// if they are non-frozen).
    ///
    /// See [`types::proof::AccumulatorProof`] for proof format.
    pub fn get_proof(reader: &R, num_leaves: usize, leaf_index: u64) -> Result<AccumulatorProof> {
        MerkleAccumulatorView::<R, H>::new(reader, num_leaves).get_proof(leaf_index)
    }

    /// Gets a proof that shows the full accumulator is consistent with a smaller accumulator.
    ///
    /// See [`types::proof::AccumulatorConsistencyProof`] for proof format.
    pub fn get_consistency_proof(
        reader: &R,
        full_acc_leaves: usize,
        sub_acc_leaves: usize,
    ) -> Result<AccumulatorConsistencyProof> {
        MerkleAccumulatorView::<R, H>::new(reader, full_acc_leaves)
            .get_consistency_proof(sub_acc_leaves)
    }

    /// From left to right, gets frozen subtree root hashes of the accumulator. For example, if the
    /// accumulator has 5 leaves, `x` and `e` are returned.
    /// ```text
    ///                 root
    ///                /    \
    ///              /        \
    ///            /            \
    ///           x              o
    ///         /   \           / \
    ///        /     \         /   \
    ///       o       o       o     placeholder
    ///      / \     / \     / \
    ///     a   b   c   d   e   placeholder
    /// ```
    pub fn get_frozen_subtree_hashes(reader: &R, num_leaves: usize) -> Result<Vec<HashValue>> {
        MerkleAccumulatorView::<R, H>::new(reader, num_leaves).get_frozen_subtree_hashes()
    }
}

/// Actual implementation of Merkle Accumulator algorithms, which carries the `reader` and
/// `num_leaves` on an instance for convenience
struct MerkleAccumulatorView<'a, R, H> {
    reader: &'a R,
    num_leaves: usize,
    hasher: PhantomData<H>,
}

impl<'a, R, H> MerkleAccumulatorView<'a, R, H>
where
    R: HashReader,
    H: CryptoHasher,
{
    fn new(reader: &'a R, num_leaves: usize) -> Self {
        Self {
            reader,
            num_leaves,
            hasher: PhantomData,
        }
    }

    /// implementation for pub interface `MerkleAccumulator::append`
    fn append(&self, new_leaves: &[HashValue]) -> Result<(HashValue, Vec<Node>)> {
        // Deal with the case where new_leaves is empty
        if new_leaves.is_empty() {
            if self.num_leaves == 0 {
                return Ok((*ACCUMULATOR_PLACEHOLDER_HASH, Vec::new()));
            } else {
                let root_hash = self.get_hash(Position::root_from_leaf_count(self.num_leaves))?;
                return Ok((root_hash, Vec::new()));
            }
        }

        let num_new_leaves = new_leaves.len();
        let last_new_leaf_count = self.num_leaves + num_new_leaves;
        let root_level = Position::root_level_from_leaf_count(last_new_leaf_count);
        let mut to_freeze = Vec::with_capacity(Self::max_to_freeze(num_new_leaves, root_level));

        // create one new node for each new leaf hash
        let mut current_level = self.gen_leaf_level(new_leaves);
        Self::record_to_freeze(
            &mut to_freeze,
            &current_level,
            false, /* has_non_frozen */
        );

        // loop starting from leaf level, upwards till root_level - 1,
        // making new nodes of parent level and recording frozen ones.
        let mut has_non_frozen = false;
        for _ in 0..root_level {
            let (parent_level, placeholder_used) = self.gen_parent_level(&current_level)?;

            // If a placeholder node is used to generate the right most node of a certain level,
            // such level and all its parent levels have a non-frozen right most node.
            has_non_frozen |= placeholder_used;
            Self::record_to_freeze(&mut to_freeze, &parent_level, has_non_frozen);

            current_level = parent_level;
        }

        assert_eq!(current_level.len(), 1, "must conclude in single root node");
        Ok((current_level.first().expect("unexpected None").1, to_freeze))
    }

    /// upper bound of num of frozen nodes:
    ///     new leaves and resulting frozen internal nodes forming a complete binary subtree
    ///         num_new_leaves * 2 - 1 < num_new_leaves * 2
    ///     and the full route from root of that subtree to the accumulator root turns frozen
    ///         height - (log2(num_new_leaves) + 1) < height - 1 = root_level
    fn max_to_freeze(num_new_leaves: usize, root_level: usize) -> usize {
        num_new_leaves * 2 + root_level
    }

    fn hash_internal_node(left: HashValue, right: HashValue) -> HashValue {
        MerkleTreeInternalNode::<H>::new(left, right).hash()
    }

    fn rightmost_leaf_index(&self) -> u64 {
        (self.num_leaves - 1) as u64
    }

    /// Given leaf level hashes, create leaf level nodes
    fn gen_leaf_level(&self, new_leaves: &[HashValue]) -> Vec<Node> {
        new_leaves
            .iter()
            .enumerate()
            .map(|(i, hash)| {
                (
                    Position::from_leaf_index((self.num_leaves + i) as u64),
                    *hash,
                )
            })
            .collect()
    }

    /// Given a level of new nodes (frozen or not), return new nodes on its parent level, and
    /// a boolean value indicating whether a placeholder node is used to construct the last node
    fn gen_parent_level(&self, current_level: &[Node]) -> Result<((Vec<Node>, bool))> {
        let mut parent_level: Vec<Node> = Vec::with_capacity(current_level.len() / 2 + 1);
        let mut iter = current_level.iter().peekable();

        // first node may be a right child, in that case pair it with its existing sibling
        let (first_pos, first_hash) = iter.peek().expect("Current level is empty");
        if !first_pos.is_left_child() {
            parent_level.push((
                first_pos.parent(),
                Self::hash_internal_node(self.reader.get(first_pos.sibling())?, *first_hash),
            ));
            iter.next();
        }

        // walk through in pairs of siblings, use placeholder as last right sibling if necessary
        let mut placeholder_used = false;
        while let Some((left_pos, left_hash)) = iter.next() {
            let right_hash = match iter.next() {
                Some((_, h)) => h,
                None => {
                    placeholder_used = true;
                    &ACCUMULATOR_PLACEHOLDER_HASH
                }
            };

            parent_level.push((
                left_pos.parent(),
                Self::hash_internal_node(*left_hash, *right_hash),
            ));
        }

        Ok((parent_level, placeholder_used))
    }

    /// append a level of new nodes into output vector, skip the last one if it's a non-frozen node
    fn record_to_freeze(to_freeze: &mut Vec<Node>, level: &[Node], has_non_frozen: bool) {
        to_freeze.extend(
            level
                .iter()
                .take(level.len() - has_non_frozen as usize)
                .cloned(),
        )
    }

    fn get_hash(&self, position: Position) -> Result<HashValue> {
        let idx = self.rightmost_leaf_index();
        if position.is_placeholder(idx) {
            Ok(*ACCUMULATOR_PLACEHOLDER_HASH)
        } else if position.is_freezable(idx) {
            self.reader.get(position)
        } else {
            // non-frozen non-placeholder node
            Ok(Self::hash_internal_node(
                self.get_hash(position.left_child())?,
                self.get_hash(position.right_child())?,
            ))
        }
    }

    /// implementation for pub interface `MerkleAccumulator::get_proof`
    fn get_proof(&self, leaf_index: u64) -> Result<AccumulatorProof> {
        ensure!(
            leaf_index < self.num_leaves as u64,
            "invalid leaf_index {}, num_leaves {}",
            leaf_index,
            self.num_leaves
        );

        let leaf_pos = Position::from_leaf_index(leaf_index);
        let root_pos = Position::root_from_leaf_count(self.num_leaves);

        let siblings: Vec<HashValue> = leaf_pos
            .iter_ancestor_sibling()
            .take(root_pos.level() as usize)
            .map(|p| self.get_hash(p))
            .collect::<Result<Vec<HashValue>>>()?
            .into_iter()
            .rev()
            .collect();

        Ok(AccumulatorProof::new(siblings))
    }

    /// Implementation for public interface `MerkleAccumulator::get_consistency_proof`.
    fn get_consistency_proof(&self, sub_acc_leaves: usize) -> Result<AccumulatorConsistencyProof> {
        ensure!(
            sub_acc_leaves <= self.num_leaves,
            "The other accumulator is bigger than this one. self.num_leaves: {}. \
             sub_acc_leaves: {}.",
            self.num_leaves,
            sub_acc_leaves,
        );

        let subtrees = FrozenSubtreeSiblingIterator::new(sub_acc_leaves, self.num_leaves)
            .map(|p| self.reader.get(p))
            .collect::<Result<Vec<_>>>()?;

        Ok(AccumulatorConsistencyProof::new(subtrees))
    }

    /// Implementation for public interface `MerkleAccumulator::get_frozen_subtree_hashes`.
    fn get_frozen_subtree_hashes(&self) -> Result<Vec<HashValue>> {
        FrozenSubTreeIterator::new(self.num_leaves)
            .map(|p| self.reader.get(p))
            .collect::<Result<Vec<_>>>()
    }
}

#[cfg(test)]
mod tests;
<commit_msg>[storage] Simplify accumulator append<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! This module provides algorithms for accessing and updating a Merkle Accumulator structure
//! persisted in a key-value store. Note that this doesn't write to the storage directly, rather,
//! it reads from it via the `HashReader` trait and yields writes via an in memory `HashMap`.
//!
//! # Merkle Accumulator
//! Given an ever growing (append only) series of "leaf" hashes, we construct an evolving Merkle
//! Tree for which proofs of inclusion/exclusion of a leaf hash at a leaf index in a snapshot
//! of the tree (represented by root hash) can be given.
//!
//! # Leaf Nodes
//! Leaf nodes carry hash values to be stored and proved. They are only appended to the tree but
//! never deleted or updated.
//!
//! # Internal Nodes
//! A non-leaf node carries the hash value derived from both its left and right children.
//!
//! # Placeholder Nodes
//! To make sure each Leaf node has a Merkle Proof towards the root, placeholder nodes are added so
//! that along the route from a leaf to the root, each node has a sibling. Placeholder nodes have
//! the hash value `ACCUMULATOR_PLACEHOLDER_HASH`
//!
//! A placeholder node can appear as either a Leaf node or a non-Leaf node, but there is at most one
//! placeholder leaf at any time.
//!
//! # Frozen Nodes & Non-frozen Nodes
//! As leaves are added to the tree, placeholder nodes get replaced by non-placeholder nodes, and
//! when a node has all its descendants being non-placeholder, it becomes "Frozen" -- its hash value
//! won't change again in the event of new leaves being added. All leaves appended (not counting the
//! one possible placeholder leaf) are by definition Frozen.
//!
//! Other nodes, which have one or more placeholder descendants are Non-Frozen. As new elements are
//! appended to the accumulator the hash value of these nodes will change.
//!
//! # Leaf Count
//! Given a count of the number of leaves in a Merkle Accumulator it is possible to determine the
//! shape of the accumulator -- which nodes are filled and which nodes are placeholder nodes.
//!
//! Example:
//! Logical view of a Merkle Accumulator with 5 leaves:
//! ```text
//!            Non-fzn
//!           /       \
//!          /         \
//!         /           \
//!       Fzn2         Non-fzn
//!      /   \           /   \
//!     /     \         /     \
//!    Fzn1    Fzn3  Non-fzn  [Placeholder]
//!   /  \    /  \    /    \
//!  L0  L1  L2  L3 L4   [Placeholder]
//! ```
//!
//! # Position and Physical Representation
//! As a Merkle Accumulator tree expands to the right and upwards, we number newly frozen nodes
//! monotonically. One way to do it is simply to use in-order index of nodes, and this is what
//! we do for the in-memory representation. We call the stated numbers identifying nodes below
//! simply "Position", and unless otherwise stated, this is the in-order position.
//!
//! For writing to disk however, we write all the children of a node before the parent.
//! Thus for disk write order, it is more convenient to use the post-order position as an index.
//! And with that we can map a Merkle Accumulator into a key-value storage: key is the post-order
//! position of a node, and the value is hash value it carries.
//!
//! We store only Frozen nodes, and generate non-Frozen nodes on the fly when accessing the tree.
//! This way, the physical representation of the tree is append-only, i.e. once written to physical
//! storage, nodes won't be either modified or deleted.
//!
//! Here is what we persist for the logical tree in the above example:
//!
//! ```text
//!          Fzn2(6)
//!         /      \
//!        /        \
//!    Fzn1(2)       Fzn3(5)
//!   /     \       /     \
//!  L0(0)  L1(1)  L2(3)  L3(4)  L4(7)
//! ```
//!
//! When the next leaf node is persisted, the physical representation will be:
//!
//! ```text
//!          Fzn2(6)
//!         /      \
//!        /        \
//!    Fzn1(2)       Fzn3(5)       Fzn4(9)
//!   /     \       /     \       /      \
//!  L0(0)  L1(1)  L2(3)  L3(4)  L4(7)   L5(8)
//! ```
//!
//! The numbering corresponds to the post-order traversal of the tree.
//!
//! To think in key-value pairs:
//! ```text
//! |<-key->|<--value-->|
//! |   0   | hash_L0   |
//! |   1   | hash_L1   |
//! |   2   | hash_Fzn1 |
//! |  ...  |   ...     |
//! ```

use crypto::hash::{CryptoHash, CryptoHasher, HashValue, ACCUMULATOR_PLACEHOLDER_HASH};
use failure::prelude::*;
use std::marker::PhantomData;
use types::proof::{
    position::{FrozenSubTreeIterator, FrozenSubtreeSiblingIterator, Position},
    AccumulatorConsistencyProof, AccumulatorProof, MerkleTreeInternalNode,
};

/// Defines the interface between `MerkleAccumulator` and underlying storage.
pub trait HashReader {
    /// Return `HashValue` carried by the node at `Position`.
    fn get(&self, position: Position) -> Result<HashValue>;
}

/// A `Node` in a `MerkleAccumulator` tree is a `HashValue` at a `Position`
type Node = (Position, HashValue);

/// In this live Merkle Accumulator algorithms.
pub struct MerkleAccumulator<R, H> {
    reader: PhantomData<R>,
    hasher: PhantomData<H>,
}

impl<R, H> MerkleAccumulator<R, H>
where
    R: HashReader,
    H: CryptoHasher,
{
    /// Given an existing Merkle Accumulator (represented by `num_existing_leaves` and a `reader`
    /// that is able to fetch all existing frozen nodes), and a list of leaves to be appended,
    /// returns the result root hash and new nodes to be frozen.
    pub fn append(
        reader: &R,
        num_existing_leaves: usize,
        new_leaves: &[HashValue],
    ) -> Result<(HashValue, Vec<Node>)> {
        MerkleAccumulatorView::<R, H>::new(reader, num_existing_leaves).append(new_leaves)
    }

    /// Get proof of inclusion of the leaf at `leaf_index` in this Merkle Accumulator of
    /// `num_leaves` leaves in total. Siblings are read via `reader` (or generated dynamically
    /// if they are non-frozen).
    ///
    /// See [`types::proof::AccumulatorProof`] for proof format.
    pub fn get_proof(reader: &R, num_leaves: usize, leaf_index: u64) -> Result<AccumulatorProof> {
        MerkleAccumulatorView::<R, H>::new(reader, num_leaves).get_proof(leaf_index)
    }

    /// Gets a proof that shows the full accumulator is consistent with a smaller accumulator.
    ///
    /// See [`types::proof::AccumulatorConsistencyProof`] for proof format.
    pub fn get_consistency_proof(
        reader: &R,
        full_acc_leaves: usize,
        sub_acc_leaves: usize,
    ) -> Result<AccumulatorConsistencyProof> {
        MerkleAccumulatorView::<R, H>::new(reader, full_acc_leaves)
            .get_consistency_proof(sub_acc_leaves)
    }

    /// From left to right, gets frozen subtree root hashes of the accumulator. For example, if the
    /// accumulator has 5 leaves, `x` and `e` are returned.
    /// ```text
    ///                 root
    ///                /    \
    ///              /        \
    ///            /            \
    ///           x              o
    ///         /   \           / \
    ///        /     \         /   \
    ///       o       o       o     placeholder
    ///      / \     / \     / \
    ///     a   b   c   d   e   placeholder
    /// ```
    pub fn get_frozen_subtree_hashes(reader: &R, num_leaves: usize) -> Result<Vec<HashValue>> {
        MerkleAccumulatorView::<R, H>::new(reader, num_leaves).get_frozen_subtree_hashes()
    }
}

/// Actual implementation of Merkle Accumulator algorithms, which carries the `reader` and
/// `num_leaves` on an instance for convenience
struct MerkleAccumulatorView<'a, R, H> {
    reader: &'a R,
    num_leaves: usize,
    hasher: PhantomData<H>,
}

impl<'a, R, H> MerkleAccumulatorView<'a, R, H>
where
    R: HashReader,
    H: CryptoHasher,
{
    fn new(reader: &'a R, num_leaves: usize) -> Self {
        Self {
            reader,
            num_leaves,
            hasher: PhantomData,
        }
    }

    /// implementation for pub interface `MerkleAccumulator::append`
    fn append(&self, new_leaves: &[HashValue]) -> Result<(HashValue, Vec<Node>)> {
        // Deal with the case where new_leaves is empty
        if new_leaves.is_empty() {
            if self.num_leaves == 0 {
                return Ok((*ACCUMULATOR_PLACEHOLDER_HASH, Vec::new()));
            } else {
                let root_hash = self.get_hash(Position::root_from_leaf_count(self.num_leaves))?;
                return Ok((root_hash, Vec::new()));
            }
        }

        let num_new_leaves = new_leaves.len();
        let last_new_leaf_count = self.num_leaves + num_new_leaves;
        let root_level = Position::root_level_from_leaf_count(last_new_leaf_count);
        let mut to_freeze = Vec::with_capacity(Self::max_to_freeze(num_new_leaves, root_level));

        // Iterate over the new leaves, adding them to to_freeze and then adding any frozen parents
        // when right children are encountered.  This has the effect of creating frozen nodes in
        // perfect post-order, which can be used as a strictly increasing append only index for
        // the underlying storage.
        //
        // We will track newly created left siblings while iterating so we can pair them with their
        // right sibling, if and when it becomes frozen.  If the frozen left sibling is not created
        // in this iteration, it must already exist in storage.
        let mut left_siblings: Vec<(_, _)> = Vec::new();
        for (leaf_offset, leaf) in new_leaves.iter().enumerate() {
            let leaf_pos = Position::from_leaf_index((self.num_leaves + leaf_offset) as u64);
            let mut hash = *leaf;
            to_freeze.push((leaf_pos, hash));
            let mut pos = leaf_pos;
            while pos.is_right_child() {
                let sibling = pos.sibling();
                hash = match left_siblings.pop() {
                    Some((x, left_hash)) => {
                        assert_eq!(x, sibling);
                        Self::hash_internal_node(left_hash, hash)
                    }
                    None => Self::hash_internal_node(self.reader.get(sibling)?, hash),
                };
                pos = pos.parent();
                to_freeze.push((pos, hash));
            }
            // The node remaining must be a left child, possibly the root of a complete binary tree.
            left_siblings.push((pos, hash));
        }

        // Now reconstruct the final root hash by walking up to root level and adding
        // placeholder hash nodes as needed on the right, and left siblings that have either
        // been newly created or read from storage.
        let (mut pos, mut hash) = left_siblings.pop().expect("Must have at least one node");
        for _ in pos.level()..root_level as u32 {
            hash = if pos.is_left_child() {
                Self::hash_internal_node(hash, *ACCUMULATOR_PLACEHOLDER_HASH)
            } else {
                let sibling = pos.sibling();
                match left_siblings.pop() {
                    Some((x, left_hash)) => {
                        assert_eq!(x, sibling);
                        Self::hash_internal_node(left_hash, hash)
                    }
                    None => Self::hash_internal_node(self.reader.get(sibling)?, hash),
                }
            };
            pos = pos.parent();
        }
        assert!(left_siblings.is_empty());

        Ok((hash, to_freeze))
    }

    /// upper bound of num of frozen nodes:
    ///     new leaves and resulting frozen internal nodes forming a complete binary subtree
    ///         num_new_leaves * 2 - 1 < num_new_leaves * 2
    ///     and the full route from root of that subtree to the accumulator root turns frozen
    ///         height - (log2(num_new_leaves) + 1) < height - 1 = root_level
    fn max_to_freeze(num_new_leaves: usize, root_level: usize) -> usize {
        num_new_leaves * 2 + root_level
    }

    fn hash_internal_node(left: HashValue, right: HashValue) -> HashValue {
        MerkleTreeInternalNode::<H>::new(left, right).hash()
    }

    fn rightmost_leaf_index(&self) -> u64 {
        (self.num_leaves - 1) as u64
    }

    fn get_hash(&self, position: Position) -> Result<HashValue> {
        let idx = self.rightmost_leaf_index();
        if position.is_placeholder(idx) {
            Ok(*ACCUMULATOR_PLACEHOLDER_HASH)
        } else if position.is_freezable(idx) {
            self.reader.get(position)
        } else {
            // non-frozen non-placeholder node
            Ok(Self::hash_internal_node(
                self.get_hash(position.left_child())?,
                self.get_hash(position.right_child())?,
            ))
        }
    }

    /// implementation for pub interface `MerkleAccumulator::get_proof`
    fn get_proof(&self, leaf_index: u64) -> Result<AccumulatorProof> {
        ensure!(
            leaf_index < self.num_leaves as u64,
            "invalid leaf_index {}, num_leaves {}",
            leaf_index,
            self.num_leaves
        );

        let leaf_pos = Position::from_leaf_index(leaf_index);
        let root_pos = Position::root_from_leaf_count(self.num_leaves);

        let siblings: Vec<HashValue> = leaf_pos
            .iter_ancestor_sibling()
            .take(root_pos.level() as usize)
            .map(|p| self.get_hash(p))
            .collect::<Result<Vec<HashValue>>>()?
            .into_iter()
            .rev()
            .collect();

        Ok(AccumulatorProof::new(siblings))
    }

    /// Implementation for public interface `MerkleAccumulator::get_consistency_proof`.
    fn get_consistency_proof(&self, sub_acc_leaves: usize) -> Result<AccumulatorConsistencyProof> {
        ensure!(
            sub_acc_leaves <= self.num_leaves,
            "The other accumulator is bigger than this one. self.num_leaves: {}. \
             sub_acc_leaves: {}.",
            self.num_leaves,
            sub_acc_leaves,
        );

        let subtrees = FrozenSubtreeSiblingIterator::new(sub_acc_leaves, self.num_leaves)
            .map(|p| self.reader.get(p))
            .collect::<Result<Vec<_>>>()?;

        Ok(AccumulatorConsistencyProof::new(subtrees))
    }

    /// Implementation for public interface `MerkleAccumulator::get_frozen_subtree_hashes`.
    fn get_frozen_subtree_hashes(&self) -> Result<Vec<HashValue>> {
        FrozenSubTreeIterator::new(self.num_leaves)
            .map(|p| self.reader.get(p))
            .collect::<Result<Vec<_>>>()
    }
}

#[cfg(test)]
mod tests;
<|endoftext|>