<commit_before>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::field_descriptor_proto;
use protobuf::descriptor::DescriptorProto;
use protobuf::descriptor::EnumDescriptorProto;
use protobuf::descriptor::FieldDescriptorProto;
use protobuf::descriptor::FileDescriptorProto;
use protobuf::descriptor::FileDescriptorSet;
use protobuf::descriptor::OneofDescriptorProto;
use protobuf::text_format::lexer::float::parse_protobuf_float;
use protobuf::Message;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    FileDescriptorSet::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = codegen.file_descriptors;
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in &mut fds.file {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc)
    }

    for desc in &mut desc.extension {
        desc.options.mut_or_default();
    }
    desc.options.mut_or_default();

    // TODO: don't clear services.
    desc.service.clear();

    // for unittest_custom_options.proto where a custom option
    // is an extension. Probably nobody outside of Google uses it.
    desc.options
        .mut_or_default()
        .unknown_fields
        .remove(15478479);
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    desc.options.mut_or_default();

    for value in &mut desc.value {
        value.options.mut_or_default();
    }
}

fn normalize_oneof_descriptor(desc: &mut OneofDescriptorProto) {
    desc.options.mut_or_default();
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in &mut desc.nested_type {
        normalize_descriptor(desc);
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc);
    }
    for desc in &mut desc.oneof_decl {
        normalize_oneof_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();

    for field in &mut desc.field {
        normalize_field(field);
    }

    for ext in &mut desc.extension {
        ext.options.mut_or_default();
    }

    for ext in &mut desc.extension_range {
        // If ext range end exceeds max field number,
        // the actual upper limit does not matter.
        // protoc is not consistent in behavior thus flush
        // the value to some arbitrary compatible value.
        if ext.has_end() && ext.get_end() >= 0x20000000 {
            ext.set_end(0x20000000);
        }
    }
}

fn normalize_field(field: &mut FieldDescriptorProto) {
    field.options.mut_or_default();

    if field.has_default_value() {
        if field.get_field_type() == field_descriptor_proto::Type::TYPE_FLOAT {
            field.set_default_value(format!(
                "{}",
                parse_protobuf_float(field.get_default_value()).unwrap()
            ));
        }
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn descriptor_for_file<'a>(fds: &'a FileDescriptorSet, file_name: &str) -> &'a FileDescriptorProto {
    for file in &fds.file {
        if Path::new(file.get_name()).file_name().unwrap()
            == Path::new(file_name).file_name().unwrap()
        {
            return file;
        }
    }
    panic!(
        "file not found: {}; all files: {}",
        file_name,
        fds.file
            .iter()
            .map(|f| f.get_name())
            .collect::<Vec<_>>()
            .join(", ")
    );
}

fn test_diff_in<F>(root: &str, sources_dir: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let mut include_root = Path::new(root).to_path_buf();
    include_root.push(include);

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, sources_dir);

    let inputs_glob = format!("{}/*.proto", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
    let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
    normalize_descriptor_set(&mut protoc_descriptors);
    normalize_descriptor_set(&mut pure_descriptors);

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_file_name = input.strip_prefix(&include_root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let protoc_descriptor_for_file = descriptor_for_file(&protoc_descriptors, &proto_file_name);
        let pure_descriptor_for_file = descriptor_for_file(&pure_descriptors, &proto_file_name);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents
            && protoc_descriptor_for_file == pure_descriptor_for_file
        {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(protoc_descriptor_for_file),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(pure_descriptor_for_file),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| false);
}
<commit_msg>Another place where we should use vendored protoc<commit_after>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::field_descriptor_proto;
use protobuf::descriptor::DescriptorProto;
use protobuf::descriptor::EnumDescriptorProto;
use protobuf::descriptor::FieldDescriptorProto;
use protobuf::descriptor::FileDescriptorProto;
use protobuf::descriptor::FileDescriptorSet;
use protobuf::descriptor::OneofDescriptorProto;
use protobuf::text_format::lexer::float::parse_protobuf_float;
use protobuf::Message;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    FileDescriptorSet::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = codegen.file_descriptors;
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in &mut fds.file {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc)
    }

    for desc in &mut desc.extension {
        desc.options.mut_or_default();
    }
    desc.options.mut_or_default();

    // TODO: don't clear services.
    desc.service.clear();

    // for unittest_custom_options.proto where a custom option
    // is an extension. Probably nobody outside of Google uses it.
    desc.options
        .mut_or_default()
        .unknown_fields
        .remove(15478479);
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    desc.options.mut_or_default();

    for value in &mut desc.value {
        value.options.mut_or_default();
    }
}

fn normalize_oneof_descriptor(desc: &mut OneofDescriptorProto) {
    desc.options.mut_or_default();
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in &mut desc.nested_type {
        normalize_descriptor(desc);
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc);
    }
    for desc in &mut desc.oneof_decl {
        normalize_oneof_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();

    for field in &mut desc.field {
        normalize_field(field);
    }

    for ext in &mut desc.extension {
        ext.options.mut_or_default();
    }

    for ext in &mut desc.extension_range {
        // If ext range end exceeds max field number,
        // the actual upper limit does not matter.
        // protoc is not consistent in behavior thus flush
        // the value to some arbitrary compatible value.
        if ext.has_end() && ext.get_end() >= 0x20000000 {
            ext.set_end(0x20000000);
        }
    }
}

fn normalize_field(field: &mut FieldDescriptorProto) {
    field.options.mut_or_default();

    if field.has_default_value() {
        if field.get_field_type() == field_descriptor_proto::Type::TYPE_FLOAT {
            field.set_default_value(format!(
                "{}",
                parse_protobuf_float(field.get_default_value()).unwrap()
            ));
        }
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn descriptor_for_file<'a>(fds: &'a FileDescriptorSet, file_name: &str) -> &'a FileDescriptorProto {
    for file in &fds.file {
        if Path::new(file.get_name()).file_name().unwrap()
            == Path::new(file_name).file_name().unwrap()
        {
            return file;
        }
    }
    panic!(
        "file not found: {}; all files: {}",
        file_name,
        fds.file
            .iter()
            .map(|f| f.get_name())
            .collect::<Vec<_>>()
            .join(", ")
    );
}

fn test_diff_in<F>(root: &str, sources_dir: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let mut include_root = Path::new(root).to_path_buf();
    include_root.push(include);

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, sources_dir);

    let inputs_glob = format!("{}/*.proto", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .protoc_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
    let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
    normalize_descriptor_set(&mut protoc_descriptors);
    normalize_descriptor_set(&mut pure_descriptors);

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_file_name = input.strip_prefix(&include_root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let protoc_descriptor_for_file = descriptor_for_file(&protoc_descriptors, &proto_file_name);
        let pure_descriptor_for_file = descriptor_for_file(&pure_descriptors, &proto_file_name);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents
            && protoc_descriptor_for_file == pure_descriptor_for_file
        {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(protoc_descriptor_for_file),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(pure_descriptor_for_file),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| false);
}
<|endoftext|>


<commit_before>//! A library to read binary protobuf files
//!
//! This reader is developed similarly to a pull reader

#![deny(missing_docs)]
#![allow(dead_code)]

extern crate byteorder;
extern crate failure;
#[macro_use]
extern crate failure_derive;

pub mod errors;
pub mod message;
pub mod reader;
pub mod sizeofs;
pub mod writer;

pub use errors::{Error, Result};
pub use message::{MessageRead, MessageWrite};
pub use reader::{deserialize_from_slice, BytesReader, Reader};
pub use writer::{serialize_into_vec, Writer};
<commit_msg>Removed #[macro_use] to clear warning.<commit_after>//! A library to read binary protobuf files
//!
//! This reader is developed similarly to a pull reader

#![deny(missing_docs)]
#![allow(dead_code)]

extern crate byteorder;
extern crate failure;
extern crate failure_derive;

pub mod errors;
pub mod message;
pub mod reader;
pub mod sizeofs;
pub mod writer;

pub use errors::{Error, Result};
pub use message::{MessageRead, MessageWrite};
pub use reader::{deserialize_from_slice, BytesReader, Reader};
pub use writer::{serialize_into_vec, Writer};
<|endoftext|>


<commit_before>//! Tests for various types of file (video, image, compressed, etc).
//!
//! Currently this is dependent on the file’s name and extension, because
//! those are the only metadata that we have access to without reading the
//! file’s contents.

use ansi_term::Style;

use crate::fs::File;
use crate::output::icons::FileIcon;
use crate::theme::FileColours;

#[derive(Debug, Default, PartialEq)]
pub struct FileExtensions;

impl FileExtensions {

    /// An “immediate” file is something that can be run or activated somehow
    /// in order to kick off the build of a project. It’s usually only present
    /// in directories full of source code.
    #[allow(clippy::case_sensitive_file_extension_comparisons)]
    fn is_immediate(&self, file: &File<'_>) -> bool {
        file.name.to_lowercase().starts_with("readme") ||
        file.name.ends_with(".ninja") ||
        file.name_is_one_of( &[
            "Makefile", "Cargo.toml", "SConstruct", "CMakeLists.txt",
            "build.gradle", "pom.xml", "Rakefile", "package.json", "Gruntfile.js",
            "Gruntfile.coffee", "BUILD", "BUILD.bazel", "WORKSPACE", "build.xml", "Podfile",
            "webpack.config.js", "meson.build", "composer.json", "RoboFile.php", "PKGBUILD",
            "Justfile", "Procfile", "Dockerfile", "Containerfile", "Vagrantfile", "Brewfile",
            "Gemfile", "Pipfile", "build.sbt", "mix.exs", "bsconfig.json", "tsconfig.json",
        ])
    }

    fn is_image(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "png", "jfi", "jfif", "jif", "jpe", "jpeg", "jpg", "gif", "bmp",
            "tiff", "tif", "ppm", "pgm", "pbm", "pnm", "webp", "raw", "arw",
            "svg", "stl", "eps", "dvi", "ps", "cbr", "jpf", "cbz", "xpm",
            "ico", "cr2", "orf", "nef", "heif",
        ])
    }

    fn is_video(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "avi", "flv", "m2v", "m4v", "mkv", "mov", "mp4", "mpeg",
            "mpg", "ogm", "ogv", "vob", "wmv", "webm", "m2ts", "heic",
        ])
    }

    fn is_music(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "aac", "m4a", "mp3", "ogg", "wma", "mka", "opus",
        ])
    }

    // Lossless music, rather than any other kind of data...
    fn is_lossless(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "alac", "ape", "flac", "wav",
        ])
    }

    fn is_crypto(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "asc", "enc", "gpg", "pgp", "sig", "signature", "pfx", "p12",
        ])
    }

    fn is_document(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "djvu", "doc", "docx", "dvi", "eml", "eps", "fotd", "key",
            "keynote", "numbers", "odp", "odt", "pages", "pdf", "ppt",
            "pptx", "rtf", "xls", "xlsx",
        ])
    }

    fn is_compressed(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "zip", "tar", "Z", "z", "gz", "bz2", "a", "ar", "7z",
            "iso", "dmg", "tc", "rar", "par", "tgz", "xz", "txz",
            "lz", "tlz", "lzma", "deb", "rpm", "zst",
        ])
    }

    fn is_temp(&self, file: &File<'_>) -> bool {
        file.name.ends_with('~')
            || (file.name.starts_with('#') && file.name.ends_with('#'))
            || file.extension_is_one_of( &[ "tmp", "swp", "swo", "swn", "bak", "bk" ])
    }

    fn is_compiled(&self, file: &File<'_>) -> bool {
        if file.extension_is_one_of( &[ "class", "elc", "hi", "o", "pyc", "zwc", "ko" ]) {
            true
        }
        else if let Some(dir) = file.parent_dir {
            file.get_source_files().iter().any(|path| dir.contains(path))
        }
        else {
            false
        }
    }
}

impl FileColours for FileExtensions {
    fn colour_file(&self, file: &File<'_>) -> Option<Style> {
        use ansi_term::Colour::*;

        Some(match file {
            f if self.is_temp(f)        => Fixed(244).normal(),
            f if self.is_immediate(f)   => Yellow.bold().underline(),
            f if self.is_image(f)       => Fixed(133).normal(),
            f if self.is_video(f)       => Fixed(135).normal(),
            f if self.is_music(f)       => Fixed(92).normal(),
            f if self.is_lossless(f)    => Fixed(93).normal(),
            f if self.is_crypto(f)      => Fixed(109).normal(),
            f if self.is_document(f)    => Fixed(105).normal(),
            f if self.is_compressed(f)  => Red.normal(),
            f if self.is_compiled(f)    => Fixed(137).normal(),
            _                           => return None,
        })
    }
}

impl FileIcon for FileExtensions {
    fn icon_file(&self, file: &File<'_>) -> Option<char> {
        use crate::output::icons::Icons;

        if self.is_music(file) || self.is_lossless(file) {
            Some(Icons::Audio.value())
        }
        else if self.is_image(file) {
            Some(Icons::Image.value())
        }
        else if self.is_video(file) {
            Some(Icons::Video.value())
        }
        else {
            None
        }
    }
}
<commit_msg>Add extension ".bkp" as temp file<commit_after>//! Tests for various types of file (video, image, compressed, etc).
//!
//! Currently this is dependent on the file’s name and extension, because
//! those are the only metadata that we have access to without reading the
//! file’s contents.

use ansi_term::Style;

use crate::fs::File;
use crate::output::icons::FileIcon;
use crate::theme::FileColours;

#[derive(Debug, Default, PartialEq)]
pub struct FileExtensions;

impl FileExtensions {

    /// An “immediate” file is something that can be run or activated somehow
    /// in order to kick off the build of a project. It’s usually only present
    /// in directories full of source code.
    #[allow(clippy::case_sensitive_file_extension_comparisons)]
    fn is_immediate(&self, file: &File<'_>) -> bool {
        file.name.to_lowercase().starts_with("readme") ||
        file.name.ends_with(".ninja") ||
        file.name_is_one_of( &[
            "Makefile", "Cargo.toml", "SConstruct", "CMakeLists.txt",
            "build.gradle", "pom.xml", "Rakefile", "package.json", "Gruntfile.js",
            "Gruntfile.coffee", "BUILD", "BUILD.bazel", "WORKSPACE", "build.xml", "Podfile",
            "webpack.config.js", "meson.build", "composer.json", "RoboFile.php", "PKGBUILD",
            "Justfile", "Procfile", "Dockerfile", "Containerfile", "Vagrantfile", "Brewfile",
            "Gemfile", "Pipfile", "build.sbt", "mix.exs", "bsconfig.json", "tsconfig.json",
        ])
    }

    fn is_image(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "png", "jfi", "jfif", "jif", "jpe", "jpeg", "jpg", "gif", "bmp",
            "tiff", "tif", "ppm", "pgm", "pbm", "pnm", "webp", "raw", "arw",
            "svg", "stl", "eps", "dvi", "ps", "cbr", "jpf", "cbz", "xpm",
            "ico", "cr2", "orf", "nef", "heif",
        ])
    }

    fn is_video(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "avi", "flv", "m2v", "m4v", "mkv", "mov", "mp4", "mpeg",
            "mpg", "ogm", "ogv", "vob", "wmv", "webm", "m2ts", "heic",
        ])
    }

    fn is_music(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "aac", "m4a", "mp3", "ogg", "wma", "mka", "opus",
        ])
    }

    // Lossless music, rather than any other kind of data...
    fn is_lossless(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "alac", "ape", "flac", "wav",
        ])
    }

    fn is_crypto(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "asc", "enc", "gpg", "pgp", "sig", "signature", "pfx", "p12",
        ])
    }

    fn is_document(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "djvu", "doc", "docx", "dvi", "eml", "eps", "fotd", "key",
            "keynote", "numbers", "odp", "odt", "pages", "pdf", "ppt",
            "pptx", "rtf", "xls", "xlsx",
        ])
    }

    fn is_compressed(&self, file: &File<'_>) -> bool {
        file.extension_is_one_of( &[
            "zip", "tar", "Z", "z", "gz", "bz2", "a", "ar", "7z",
            "iso", "dmg", "tc", "rar", "par", "tgz", "xz", "txz",
            "lz", "tlz", "lzma", "deb", "rpm", "zst",
        ])
    }

    fn is_temp(&self, file: &File<'_>) -> bool {
        file.name.ends_with('~')
            || (file.name.starts_with('#') && file.name.ends_with('#'))
            || file.extension_is_one_of( &[ "tmp", "swp", "swo", "swn", "bak", "bkp", "bk" ])
    }

    fn is_compiled(&self, file: &File<'_>) -> bool {
        if file.extension_is_one_of( &[ "class", "elc", "hi", "o", "pyc", "zwc", "ko" ]) {
            true
        }
        else if let Some(dir) = file.parent_dir {
            file.get_source_files().iter().any(|path| dir.contains(path))
        }
        else {
            false
        }
    }
}

impl FileColours for FileExtensions {
    fn colour_file(&self, file: &File<'_>) -> Option<Style> {
        use ansi_term::Colour::*;

        Some(match file {
            f if self.is_temp(f)        => Fixed(244).normal(),
            f if self.is_immediate(f)   => Yellow.bold().underline(),
            f if self.is_image(f)       => Fixed(133).normal(),
            f if self.is_video(f)       => Fixed(135).normal(),
            f if self.is_music(f)       => Fixed(92).normal(),
            f if self.is_lossless(f)    => Fixed(93).normal(),
            f if self.is_crypto(f)      => Fixed(109).normal(),
            f if self.is_document(f)    => Fixed(105).normal(),
            f if self.is_compressed(f)  => Red.normal(),
            f if self.is_compiled(f)    => Fixed(137).normal(),
            _                           => return None,
        })
    }
}

impl FileIcon for FileExtensions {
    fn icon_file(&self, file: &File<'_>) -> Option<char> {
        use crate::output::icons::Icons;

        if self.is_music(file) || self.is_lossless(file) {
            Some(Icons::Audio.value())
        }
        else if self.is_image(file) {
            Some(Icons::Image.value())
        }
        else if self.is_video(file) {
            Some(Icons::Video.value())
        }
        else {
            None
        }
    }
}
<|endoftext|>


<commit_before>use std::io::{self, Read};
use std::mem;

use futures::{Poll, Future};

/// A future which can be used to easily read the entire contents of a stream
/// into a vector.
///
/// Created by the `read_exact` function.
pub struct ReadExact<A, T> {
    state: State<A, T>,
}

enum State<A, T> {
    Reading {
        a: A,
        buf: T,
        pos: usize,
    },
    Empty,
}

/// Creates a future which will read exactly enough bytes to fill `buf`,
/// returning an error if EOF is hit sooner.
///
/// The returned future will resolve to both the I/O stream as well as the
/// buffer once the read operation is completed.
///
/// In the case of an error the buffer and the object will be discarded, with
/// the error yielded. In the case of success the object will be destroyed and
/// the buffer will be returned, with all data read from the stream appended to
/// the buffer.
pub fn read_exact<A, T>(a: A, buf: T) -> ReadExact<A, T>
    where A: Read,
          T: AsMut<[u8]>,
{
    ReadExact {
        state: State::Reading {
            a: a,
            buf: buf,
            pos: 0,
        },
    }
}

fn eof() -> io::Error {
    io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")
}

impl<A, T> Future for ReadExact<A, T>
    where A: Read,
          T: AsMut<[u8]>,
{
    type Item = (A, T);
    type Error = io::Error;

    fn poll(&mut self) -> Poll<(A, T), io::Error> {
        match self.state {
            State::Reading { ref mut a, ref mut buf, ref mut pos } => {
                let buf = buf.as_mut();
                while *pos < buf.len() {
                    let n = try_nb!(a.read(&mut buf[*pos..]));
                    *pos += n;
                    if n == 0 {
                        return Poll::Err(eof())
                    }
                }
            }
            State::Empty => panic!("poll a ReadExact after it's done"),
        }

        match mem::replace(&mut self.state, State::Empty) {
            State::Reading { a, buf, .. } => Poll::Ok((a, buf)),
            State::Empty => panic!(),
        }
    }
}
<commit_msg>docs: Fix copypasta in read_exact docs<commit_after>use std::io::{self, Read};
use std::mem;

use futures::{Poll, Future};

/// A future which can be used to easily read exactly enough bytes to fill
/// a buffer.
///
/// Created by the `read_exact` function.
pub struct ReadExact<A, T> {
    state: State<A, T>,
}

enum State<A, T> {
    Reading {
        a: A,
        buf: T,
        pos: usize,
    },
    Empty,
}

/// Creates a future which will read exactly enough bytes to fill `buf`,
/// returning an error if EOF is hit sooner.
///
/// The returned future will resolve to both the I/O stream as well as the
/// buffer once the read operation is completed.
///
/// In the case of an error the buffer and the object will be discarded, with
/// the error yielded. In the case of success the object will be destroyed and
/// the buffer will be returned, with all data read from the stream appended to
/// the buffer.
pub fn read_exact<A, T>(a: A, buf: T) -> ReadExact<A, T>
    where A: Read,
          T: AsMut<[u8]>,
{
    ReadExact {
        state: State::Reading {
            a: a,
            buf: buf,
            pos: 0,
        },
    }
}

fn eof() -> io::Error {
    io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")
}

impl<A, T> Future for ReadExact<A, T>
    where A: Read,
          T: AsMut<[u8]>,
{
    type Item = (A, T);
    type Error = io::Error;

    fn poll(&mut self) -> Poll<(A, T), io::Error> {
        match self.state {
            State::Reading { ref mut a, ref mut buf, ref mut pos } => {
                let buf = buf.as_mut();
                while *pos < buf.len() {
                    let n = try_nb!(a.read(&mut buf[*pos..]));
                    *pos += n;
                    if n == 0 {
                        return Poll::Err(eof())
                    }
                }
            }
            State::Empty => panic!("poll a ReadExact after it's done"),
        }

        match mem::replace(&mut self.state, State::Empty) {
            State::Reading { a, buf, .. } => Poll::Ok((a, buf)),
            State::Empty => panic!(),
        }
    }
}
<|endoftext|>


<commit_before>
use util;
use std;
use std::error::Error;

pub type Result<T> = std::result::Result<T,String>;

/// A list of symbols to be tokenized.
pub const SYMBOL_LIST: &'static [char] = &[
    ',', ':', '(', ')', '@', '%', '{', '}',
];

/// A token.
#[derive(Clone,Debug,PartialEq,Eq)]
pub enum Token
{
    /// A word.
    Word(String),
    /// A string literal.
    String(String),
    /// An integer literal.
    // TODO: use BigNum
    Integer(i64),
    /// A comment.
    ///
    /// If the comment is inline, it existed on the same line
    /// as the previous statement.
    ///
    /// For example
    ///
    /// ```
    /// add 2, 4 ; inline comment goes here
    /// ```
    Comment {
        inline: bool,
        text: String,
    },
    /// A symbol.
    Symbol(String),
    /// A new line.
    NewLine,
    /// End of file.
    /// Will always be yielded as the last
    /// token.
    EOF,
}

impl Token
{
    pub fn comma() -> Self { Token::symbol(",") }
    pub fn colon() -> Self { Token::symbol(":") }
    pub fn left_parenthesis() -> Self { Token::symbol("(") }
    pub fn right_parenthesis() -> Self { Token::symbol(")") }
    pub fn at_sign() -> Self { Token::symbol("@") }
    pub fn percent_sign() -> Self { Token::symbol("%") }
    pub fn left_curly_brace() -> Self { Token::symbol("{") }
    pub fn right_curly_brace() -> Self { Token::symbol("}") }

    pub fn word<S>(word: S) -> Self
        where S: Into<String> {
        Token::Word(word.into())
    }

    pub fn string<S>(string: S) -> Self
        where S: Into<String> {
        Token::String(string.into())
    }

    pub fn integer<I>(integer: I) -> Self
        where I: Into<i64> {
        Token::Integer(integer.into())
    }

    pub fn comment<S>(text: S) -> Self
        where S: Into<String> {
        Token::Comment {
            inline: false,
            text: text.into(),
        }
    }

    pub fn inline_comment<S>(text: S) -> Self
        where S: Into<String> {
        Token::Comment {
            inline: true,
            text: text.into(),
        }
    }

    pub fn symbol<S>(symbol: S) -> Self
        where S: Into<String> {
        Token::Symbol(symbol.into())
    }

    pub fn new_line() -> Self {
        Token::NewLine
    }

    pub fn eof() -> Self {
        Token::EOF
    }
}

impl std::fmt::Display for Token
{
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        match self {
            &Token::Word(ref w) => write!(fmt, "{}", w),
            &Token::String(ref s) => write!(fmt, "\"{}\"", s),
            &Token::Integer(ref i) => write!(fmt, "{}", i),
            &Token::Symbol(ref s) => write!(fmt, "{}", s),
            &Token::Comment { ref text, .. } => write!(fmt, " {}", text),
            &Token::NewLine => write!(fmt, "new line"),
            &Token::EOF => write!(fmt, "EOF"),
        }
    }
}

/// An iterator over a set of characters.
///
/// This type abstracts over another iterator of characters,
/// making sure that the yielded characters will have a specific
/// format.
///
/// * There will always be a new line (`\n`) before the end of file.
pub struct Characters<I: Iterator<Item=char>>
{
    it: std::iter::Peekable<I>,
    finished: bool,
    peek_buf: Option<char>,
}

impl<I> Characters<I>
    where I: Iterator<Item=char>
{
    pub fn new(it: I) -> Self {
        Characters {
            it: it.peekable(),
            finished: false,
            peek_buf: None,
        }
    }

    pub fn peek(&mut self) -> Option<char> {
        if self.peek_buf.is_none() {
            self.peek_buf = self.next();
        }

        self.peek_buf
    }

    pub fn eat(&mut self) {
        self.next();
    }

    pub fn eat_while<P>(&mut self, predicate: P)
        where P: FnMut(char) -> bool {
        self.consume_while(predicate);
    }

    pub fn consume_while<P>(&mut self, mut predicate: P)
        -> std::vec::IntoIter<char>
        where P: FnMut(char) -> bool {
        let mut result = Vec::new();

        loop {
            let c = match self.peek() {
                Some(c) => c,
                None => break,
            };

            if predicate(c) {
                result.push(self.next().unwrap());
            } else {
                break;
            }
        }

        result.into_iter()
    }
}

impl<I> Iterator for Characters<I>
    where I: Iterator<Item=char>
{
    type Item = char;

    fn next(&mut self) -> Option<char> {
        if self.peek_buf.is_some() {
            let c = self.peek_buf.unwrap();
            self.peek_buf = None;
            return Some(c);
        }

        match self.it.next() {
            Some(c) => Some(c),
            None => {
                if self.finished {
                    None
                } else { // When we first reach the EOF
                    self.finished = true;
                    Some('\n')
                }
            },
        }
    }
}

pub struct Tokenizer<I: Iterator<Item=char>>
{
    chars: Characters<I>,

    finished: bool,
    preserve_comments: bool,
}

impl<I> Tokenizer<I>
    where I: Iterator<Item=char>
{
    pub fn new(chars: I) -> Self {
        Tokenizer {
            chars: Characters::new(chars),
            finished: false,
            preserve_comments: false,
        }
    }

    /// Allows the tokenizer to preserve comments it tokenizes.
    pub fn preserve_comments(mut self) -> Self {
        self.preserve_comments = true;
        self
    }

    fn eat_whitespace(&mut self) {
        self.chars.eat_while(|c| c.is_whitespace() && c != '\n')
    }

    fn next_string(&mut self) -> Option<Result<Token>> {
        self.assert('"');
        let string = self.chars.consume_while(|c| c != '\"');
        self.assert('"');

        Some(Ok(Token::String(string.collect())))
    }

    fn next_integer(&mut self) -> Option<Result<Token>> {
        let string: String = self.chars.consume_while(|c| c.is_numeric()).collect();

        let int = match i64::from_str_radix(&string, 10) {
            Ok(int) => int,
            Err(e) => return Some(Err(format!("Could not parse integer: {}",
                                              e.description()))),
        };

        Some(Ok(Token::Integer(int)))
    }

    fn next_word(&mut self) -> Option<Result<Token>> {
        let word: String = self.chars.consume_while(internal::can_word_contain).collect();

        Some(Ok(Token::Word(word)))
    }

    fn next_symbol(&mut self) -> Option<Result<Token>> {
        match self.expect_one_of(SYMBOL_LIST) {
            Ok(sym) => Some(Ok(Token::symbol(format!("{}", sym)))),
            // TODO: this will give a shitty error message
            // if the end of stream is reached.
            Err(..) => Some(Err("unknown token".into())),
        }
    }

    fn next_comment(&mut self) -> Option<Result<Token>> {
        self.assert(';');
        let text: String = self.chars.consume_while(|c| c != '\n').collect();

        Some(Ok(Token::comment(text)))
    }

    fn eat_next_comment(&mut self) -> Option<Result<()>> {
        self.next_comment().map(|result| result.map(|_| ()))
    }

    fn assert(&mut self, expected: char) -> char {
        self.expect(expected).expect("unexpected character")
    }

    fn expect(&mut self, expected: char) -> Result<char> {
        // TODO: more specific error message
        self.expect_one_of(&[expected])
    }

    fn expect_one_of(&mut self, expected: &[char]) -> Result<char> {
        let expected_str = util::comma_separated_values(expected.iter());

        let next = match self.chars.next() {
            Some(c) => c,
            None => {
                return Err(format!("expected one of {} but got nothing", expected_str))
            },
        };

        if expected.iter().any(|&c| c == next) {
            Ok(next)
        } else {
            Err(format!("Expected one of {} but got {}", expected_str, next))
        }
    }
}

impl<I> Iterator for Tokenizer<I>
    where I: Iterator<Item=char>
{
    type Item = Result<Token>;

    fn next(&mut self) -> Option<Result<Token>> {
        self.eat_whitespace();

        let first_char = match self.chars.peek() {
            Some(c) => c,

            // If there are no tokens left, return EOF if
            // we haven't already, or None otherwise.
            None => {
                if self.finished {
                    return None;
                } else {
                    self.finished = true;
                    return Some(Ok(Token::eof()));
                }
            },
        };

        if first_char == '"' {
            self.next_string()
        } else if first_char == '\n' {
            self.chars.eat();
            Some(Ok(Token::new_line()))
        } else if first_char == ';' {
            if self.preserve_comments {
                self.next_comment()
            } else {

                // TODO: This could be nicer
                match self.eat_next_comment() {
                    Some(Ok(..)) => (),
                    Some(Err(e)) => return Some(Err(e)),
                    None => panic!("expected a comment to be parsed"),
                }

                // TODO: fix needless recursion
                self.next()
            }
        } else if first_char.is_numeric() {
            self.next_integer()
        } else if internal::can_word_start_with(first_char) {
            self.next_word()
        } else {
            // Try to parse the token as a symbol.
            // This is our final fallback.
            self.next_symbol()
        }
    }
}

mod internal
{
    pub fn can_word_start_with(c: char) -> bool {
        c.is_alphabetic() || c == '_'
    }

    pub fn can_word_contain(c: char) -> bool {
        can_word_start_with(c) || c.is_numeric()
    }
}

mod test
{
    use super::{Token,Tokenizer};

    /// Expects a mapping from a string of tokens into
    /// a list of tokens with the default tokenizer.
    macro_rules! expect_mapping {
        ($input:expr => $( $output:expr ),* ) => {
            expect_mapping_with!(Tokenizer::new($input.chars()) =>
                                 $( $output ),*);
        }
    }

    /// Expects a mapping from a string of tokens into
    /// a list of tokens, given a specific tokenizer.
    macro_rules! expect_mapping_with {
        // TODO: make the macro take several arguments
        // for several outputs
        ($tokenizer:expr => $( $output:expr ),* ) => {
            {
                let mut tokenizer = $tokenizer;

                $(
                    let token = tokenizer.next().unwrap().unwrap();
                    assert!($output == token,
                            format!("expected {} but got {}",
                                    $output, token));
                )*
            }
        }
    }

    #[test]
    fn test_string() {
        expect_mapping!("\"hello\"" =>  Token::string("hello"),
                                        Token::new_line(),
                                        Token::eof());

        expect_mapping!("\"hello abc\"" => Token::string("hello abc"),
                                           Token::new_line(),
                                           Token::eof());

        expect_mapping!("\"hello world\"  \"it is me\"" =>
                        Token::string("hello world"),
                        Token::string("it is me"),
                        Token::new_line(),
                        Token::eof());
    }

    #[test]
    fn test_integer() {
        expect_mapping!("123" => Token::integer(123),
                                 Token::new_line(),
                                 Token::eof());

        expect_mapping!("0982" => Token::integer(982),
                                  Token::new_line(),
                                  Token::eof());

        expect_mapping!("333 662" => Token::integer(333),
                                     Token::integer(662),
                                     Token::new_line(),
                                     Token::eof());

        expect_mapping!("1 2 3 4" => Token::integer(1),
                                     Token::integer(2),
                                     Token::integer(3),
                                     Token::integer(4),
                                     Token::new_line(),
                                     Token::eof());
    }

    #[test]
    fn test_word() {
        expect_mapping!("hello" => Token::word("hello"));

        expect_mapping!("ab cd" => Token::word("ab"),
                                   Token::word("cd"),
                                   Token::new_line(),
                                   Token::eof());

        expect_mapping!("a b c d" => Token::word("a"),
                                     Token::word("b"),
                                     Token::word("c"),
                                     Token::word("d"),
                                     Token::new_line(),
                                     Token::eof());
    }

    #[test]
    fn test_comments_not_preserved_by_default() {
        expect_mapping!("hello ; there" => Token::word("hello"),
                                           Token::new_line(),
                                           Token::eof());

        expect_mapping!(";why hello" => Token::new_line(),
                                        Token::eof());
    }

    #[test]
    fn test_comments() {
        expect_mapping_with!(Tokenizer::new("hello world ; this is me".chars())
                             .preserve_comments()
                             => Token::word("hello"),
                                Token::word("world"),
                                Token::comment(" this is me"),
                                Token::new_line(),
                                Token::eof());

        expect_mapping_with!(Tokenizer::new(";this is a test".chars())
                             .preserve_comments()
                             => Token::comment("this is a test"),
                                Token::new_line(),
                                Token::eof());
    }

    #[test]
    fn test_symbols() {
        expect_mapping!("(" => Token::symbol("("),
                               Token::new_line(),
                               Token::eof());

        expect_mapping!(")" => Token::symbol(")"));
        expect_mapping!("{ }" => Token::symbol("{"), Token::symbol("}"));
        expect_mapping!(":" => Token::symbol(":"));
        expect_mapping!("," => Token::symbol(","));
        expect_mapping!("@%" => Token::symbol("@"), Token::symbol("%"));
    }

    #[test]
    fn test_multiple() {
        expect_mapping!("12 bark \"earth\"" => Token::integer(12),
                                               Token::word("bark"),
                                               Token::string("earth"),
                                               Token::new_line(),
                                               Token::eof());

        expect_mapping!("a      23 gg \"a\"" => Token::word("a"),
                                                Token::integer(23),
                                                Token::word("gg"),
                                                Token::string("a"),
                                                Token::new_line(),
                                                Token::eof());

        expect_mapping!("4:3{2\n}" => Token::integer(4),
                                      Token::symbol(":"),
                                      Token::integer(3),
                                      Token::symbol("{"),
                                      Token::integer(2),
                                      Token::new_line(),
                                      Token::symbol("}"),
                                      Token::new_line(),
                                      Token::eof());
    }

    #[test]
    fn test_whitespace() {
        expect_mapping!("a\tb\t123\n\"qwer\"\n" => Token::word("a"),
                                                   Token::word("b"),
                                                   Token::integer(123),
                                                   Token::new_line(),
                                                   Token::string("qwer"),
                                                   Token::new_line(),
                                                   Token::new_line(),
                                                   Token::eof());
    }
}
<commit_msg>Added peeking support to the tokenizer<commit_after>use util;
use std;
use std::error::Error;

pub type Result<T> = std::result::Result<T,String>;

/// A list of symbols to be tokenized.
pub const SYMBOL_LIST: &'static [char] = &[
    ',', ':', '(', ')', '@', '%', '{', '}',
];

/// A token.
#[derive(Clone,Debug,PartialEq,Eq)]
pub enum Token
{
    /// A word.
    Word(String),
    /// A string literal.
    String(String),
    /// An integer literal.
    // TODO: use BigNum
    Integer(i64),
    /// A comment.
    ///
    /// If the comment is inline, it existed on the same line
    /// as the previous statement.
    ///
    /// For example
    ///
    /// ```
    /// add 2, 4 ; inline comment goes here
    /// ```
    Comment {
        inline: bool,
        text: String,
    },
    /// A symbol.
    Symbol(String),
    /// A new line.
    NewLine,
    /// End of file.
    /// Will always be yielded as the last
    /// token.
    EOF,
}

impl Token
{
    pub fn comma() -> Self { Token::symbol(",") }
    pub fn colon() -> Self { Token::symbol(":") }
    pub fn left_parenthesis() -> Self { Token::symbol("(") }
    pub fn right_parenthesis() -> Self { Token::symbol(")") }
    pub fn at_sign() -> Self { Token::symbol("@") }
    pub fn percent_sign() -> Self { Token::symbol("%") }
    pub fn left_curly_brace() -> Self { Token::symbol("{") }
    pub fn right_curly_brace() -> Self { Token::symbol("}") }

    pub fn word<S>(word: S) -> Self
        where S: Into<String> {
        Token::Word(word.into())
    }

    pub fn string<S>(string: S) -> Self
        where S: Into<String> {
        Token::String(string.into())
    }

    pub fn integer<I>(integer: I) -> Self
        where I: Into<i64> {
        Token::Integer(integer.into())
    }

    pub fn comment<S>(text: S) -> Self
        where S: Into<String> {
        Token::Comment {
            inline: false,
            text: text.into(),
        }
    }

    pub fn inline_comment<S>(text: S) -> Self
        where S: Into<String> {
        Token::Comment {
            inline: true,
            text: text.into(),
        }
    }

    pub fn symbol<S>(symbol: S) -> Self
        where S: Into<String> {
        Token::Symbol(symbol.into())
    }

    pub fn new_line() -> Self {
        Token::NewLine
    }

    pub fn eof() -> Self {
        Token::EOF
    }
}

impl std::fmt::Display for Token
{
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        match self {
            &Token::Word(ref w) => write!(fmt, "{}", w),
            &Token::String(ref s) => write!(fmt, "\"{}\"", s),
            &Token::Integer(ref i) => write!(fmt, "{}", i),
            &Token::Symbol(ref s) => write!(fmt, "{}", s),
            &Token::Comment { ref text, .. } => write!(fmt, " {}", text),
            &Token::NewLine => write!(fmt, "new line"),
            &Token::EOF => write!(fmt, "EOF"),
        }
    }
}

/// An iterator over a set of characters.
///
/// This type abstracts over another iterator of characters,
/// making sure that the yielded characters will have a specific
/// format.
///
/// * There will always be a new line (`\n`) before the end of file.
pub struct Characters<I: Iterator<Item=char>>
{
    it: std::iter::Peekable<I>,
    finished: bool,
    peek_buf: Option<char>,
}

impl<I> Characters<I>
    where I: Iterator<Item=char>
{
    pub fn new(it: I) -> Self {
        Characters {
            it: it.peekable(),
            finished: false,
            peek_buf: None,
        }
    }

    pub fn peek(&mut self) -> Option<char> {
        if self.peek_buf.is_none() {
            self.peek_buf = self.next();
        }

        self.peek_buf
    }

    pub fn eat(&mut self) {
        self.next();
    }

    pub fn eat_while<P>(&mut self, predicate: P)
        where P: FnMut(char) -> bool {
        self.consume_while(predicate);
    }

    pub fn consume_while<P>(&mut self, mut predicate: P)
        -> std::vec::IntoIter<char>
        where P: FnMut(char) -> bool {
        let mut result = Vec::new();

        loop {
            let c = match self.peek() {
                Some(c) => c,
                None => break,
            };

            if predicate(c) {
                result.push(self.next().unwrap());
            } else {
                break;
            }
        }

        result.into_iter()
    }
}

impl<I> Iterator for Characters<I>
    where I: Iterator<Item=char>
{
    type Item = char;

    fn next(&mut self) -> Option<char> {
        if self.peek_buf.is_some() {
            let c = self.peek_buf.unwrap();
            self.peek_buf = None;
            return Some(c);
        }

        match self.it.next() {
            Some(c) => Some(c),
            None => {
                if self.finished {
                    None
                } else { // When we first reach the EOF
                    self.finished = true;
                    Some('\n')
                }
            },
        }
    }
}

pub struct Tokenizer<I: Iterator<Item=char>>
{
    chars: Characters<I>,

    finished: bool,
    preserve_comments: bool,

    peek_buf: Option<Token>,
}

impl<I> Tokenizer<I>
    where I: Iterator<Item=char>
{
    pub fn new(chars: I) -> Self {
        Tokenizer {
            chars: Characters::new(chars),
            finished: false,
            preserve_comments: false,
            peek_buf: None,
        }
    }

    /// Allows the tokenizer to preserve comments it tokenizes.
    pub fn preserve_comments(mut self) -> Self {
        self.preserve_comments = true;
        self
    }

    pub fn peek(&mut self) -> Option<Result<Token>> {
        let token = match self.next() {
            Some(Ok(token)) => token,
            Some(Err(token)) => return Some(Err(token)),
            None => return None,
        };

        assert!(self.peek_buf.is_none());

        self.peek_buf = Some(token.clone());
        Some(Ok(token))
    }

    fn eat_whitespace(&mut self) {
        self.chars.eat_while(|c| c.is_whitespace() && c != '\n')
    }

    fn next_string(&mut self) -> Option<Result<Token>> {
        self.assert('"');
        let string = self.chars.consume_while(|c| c != '\"');
        self.assert('"');

        Some(Ok(Token::String(string.collect())))
    }

    fn next_integer(&mut self) -> Option<Result<Token>> {
        let string: String = self.chars.consume_while(|c| c.is_numeric()).collect();

        let int = match i64::from_str_radix(&string, 10) {
            Ok(int) => int,
            Err(e) => return Some(Err(format!("Could not parse integer: {}",
                                              e.description()))),
        };

        Some(Ok(Token::Integer(int)))
    }

    fn next_word(&mut self) -> Option<Result<Token>> {
        let word: String = self.chars.consume_while(internal::can_word_contain).collect();

        Some(Ok(Token::Word(word)))
    }

    fn next_symbol(&mut self) -> Option<Result<Token>> {
        match self.expect_one_of(SYMBOL_LIST) {
            Ok(sym) => Some(Ok(Token::symbol(format!("{}", sym)))),
            // TODO: this will give a shitty error message
            // if the end of stream is reached.
            Err(..) => Some(Err("unknown token".into())),
        }
    }

    fn next_comment(&mut self) -> Option<Result<Token>> {
        self.assert(';');
        let text: String = self.chars.consume_while(|c| c != '\n').collect();

        Some(Ok(Token::comment(text)))
    }

    fn eat_next_comment(&mut self) -> Option<Result<()>> {
        self.next_comment().map(|result| result.map(|_| ()))
    }

    fn assert(&mut self, expected: char) -> char {
        self.expect(expected).expect("unexpected character")
    }

    fn expect(&mut self, expected: char) -> Result<char> {
        // TODO: more specific error message
        self.expect_one_of(&[expected])
    }

    fn expect_one_of(&mut self, expected: &[char]) -> Result<char> {
        let expected_str = util::comma_separated_values(expected.iter());

        let next = match self.chars.next() {
            Some(c) => c,
            None => {
                return Err(format!("expected one of {} but got nothing", expected_str))
            },
        };

        if expected.iter().any(|&c| c == next) {
            Ok(next)
        } else {
            Err(format!("Expected one of {} but got {}", expected_str, next))
        }
    }
}

impl<I> Iterator for Tokenizer<I>
    where I: Iterator<Item=char>
{
    type Item = Result<Token>;

    fn next(&mut self) -> Option<Result<Token>> {
        if self.peek_buf.is_some() {
            let token = self.peek_buf.clone().unwrap();
            self.peek_buf = None;
            return Some(Ok(token));
        }

        self.eat_whitespace();

        let first_char = match self.chars.peek() {
            Some(c) => c,

            // If there are no tokens left, return EOF if
            // we haven't already, or None otherwise.
            None => {
                if self.finished {
                    return None;
                } else {
                    self.finished = true;
                    return Some(Ok(Token::eof()));
                }
            },
        };

        if first_char == '"' {
            self.next_string()
        } else if first_char == '\n' {
            self.chars.eat();
            Some(Ok(Token::new_line()))
        } else if first_char == ';' {
            if self.preserve_comments {
                self.next_comment()
            } else {

                // TODO: This could be nicer
                match self.eat_next_comment() {
                    Some(Ok(..)) => (),
                    Some(Err(e)) => return Some(Err(e)),
                    None => panic!("expected a comment to be parsed"),
                }

                // TODO: fix needless recursion
                self.next()
            }
        } else if first_char.is_numeric() {
            self.next_integer()
        } else if internal::can_word_start_with(first_char) {
            self.next_word()
        } else {
            // Try to parse the token as a symbol.
            // This is our final fallback.
            self.next_symbol()
        }
    }
}

mod internal
{
    pub fn can_word_start_with(c: char) -> bool {
        c.is_alphabetic() || c == '_'
    }

    pub fn can_word_contain(c: char) -> bool {
        can_word_start_with(c) || c.is_numeric()
    }
}

#[cfg(test)]
mod test
{
    use super::{Token,Tokenizer};

    /// Expects a mapping from a string of tokens into
    /// a list of tokens with the default tokenizer.
    macro_rules! expect_mapping {
        ($input:expr => $( $output:expr ),* ) => {
            expect_mapping_with!(Tokenizer::new($input.chars()) =>
                                 $( $output ),*);
        }
    }

    /// Expects a mapping from a string of tokens into
    /// a list of tokens, given a specific tokenizer.
    macro_rules! expect_mapping_with {
        // TODO: make the macro take several arguments
        // for several outputs
        ($tokenizer:expr => $( $output:expr ),* ) => {
            {
                let mut tokenizer = $tokenizer;

                $(
                    let token = tokenizer.next().unwrap().unwrap();
                    assert!($output == token,
                            format!("expected {} but got {}",
                                    $output, token));
                )*
            }
        }
    }

    #[test]
    fn test_string() {
        expect_mapping!("\"hello\"" =>  Token::string("hello"),
                                        Token::new_line(),
                                        Token::eof());

        expect_mapping!("\"hello abc\"" => Token::string("hello abc"),
                                           Token::new_line(),
                                           Token::eof());

        expect_mapping!("\"hello world\"  \"it is me\"" =>
                        Token::string("hello world"),
                        Token::string("it is me"),
                        Token::new_line(),
                        Token::eof());
    }

    #[test]
    fn test_integer() {
        expect_mapping!("123" => Token::integer(123),
                                 Token::new_line(),
                                 Token::eof());

        expect_mapping!("0982" => Token::integer(982),
                                  Token::new_line(),
                                  Token::eof());

        expect_mapping!("333 662" => Token::integer(333),
                                     Token::integer(662),
                                     Token::new_line(),
                                     Token::eof());

        expect_mapping!("1 2 3 4" => Token::integer(1),
                                     Token::integer(2),
                                     Token::integer(3),
                                     Token::integer(4),
                                     Token::new_line(),
                                     Token::eof());
    }

    #[test]
    fn test_word() {
        expect_mapping!("hello" => Token::word("hello"));

        expect_mapping!("ab cd" => Token::word("ab"),
                                   Token::word("cd"),
                                   Token::new_line(),
                                   Token::eof());

        expect_mapping!("a b c d" => Token::word("a"),
                                     Token::word("b"),
                                     Token::word("c"),
                                     Token::word("d"),
                                     Token::new_line(),
                                     Token::eof());
    }

    #[test]
    fn test_comments_not_preserved_by_default() {
        expect_mapping!("hello ; there" => Token::word("hello"),
                                           Token::new_line(),
                                           Token::eof());

        expect_mapping!(";why hello" => Token::new_line(),
                                        Token::eof());
    }

    #[test]
    fn test_comments() {
        expect_mapping_with!(Tokenizer::new("hello world ; this is me".chars())
                             .preserve_comments()
                             => Token::word("hello"),
                                Token::word("world"),
                                Token::comment(" this is me"),
                                Token::new_line(),
                                Token::eof());

        expect_mapping_with!(Tokenizer::new(";this is a test".chars())
                             .preserve_comments()
                             => Token::comment("this is a test"),
                                Token::new_line(),
                                Token::eof());
    }

    #[test]
    fn test_symbols() {
        expect_mapping!("(" => Token::symbol("("),
                               Token::new_line(),
                               Token::eof());

        expect_mapping!(")" => Token::symbol(")"));
        expect_mapping!("{ }" => Token::symbol("{"), Token::symbol("}"));
        expect_mapping!(":" => Token::symbol(":"));
        expect_mapping!("," => Token::symbol(","));
        expect_mapping!("@%" => Token::symbol("@"), Token::symbol("%"));
    }

    #[test]
    fn test_multiple() {
        expect_mapping!("12 bark \"earth\"" => Token::integer(12),
                                               Token::word("bark"),
                                               Token::string("earth"),
                                               Token::new_line(),
                                               Token::eof());

        expect_mapping!("a      23 gg \"a\"" => Token::word("a"),
                                                Token::integer(23),
                                                Token::word("gg"),
                                                Token::string("a"),
                                                Token::new_line(),
                                                Token::eof());

        expect_mapping!("4:3{2\n}" => Token::integer(4),
                                      Token::symbol(":"),
                                      Token::integer(3),
                                      Token::symbol("{"),
                                      Token::integer(2),
                                      Token::new_line(),
                                      Token::symbol("}"),
                                      Token::new_line(),
                                      Token::eof());
    }

    #[test]
    fn test_whitespace() {
        expect_mapping!("a\tb\t123\n\"qwer\"\n" => Token::word("a"),
                                                   Token::word("b"),
                                                   Token::integer(123),
                                                   Token::new_line(),
                                                   Token::string("qwer"),
                                                   Token::new_line(),
                                                   Token::new_line(),
                                                   Token::eof());
    }

    #[test]
    fn test_peek() {
        let mut tokenizer = Tokenizer::new("a b c".chars());

        assert_eq!(tokenizer.peek().unwrap().unwrap(),
                   Token::word("a"));

        assert_eq!(tokenizer.peek().unwrap().unwrap(),
                   Token::word("a"));

        assert_eq!(tokenizer.next().unwrap().unwrap(),
                   Token::word("a"));

        assert_eq!(tokenizer.peek().unwrap().unwrap(),
                   Token::word("b"));

        assert_eq!(tokenizer.next().unwrap().unwrap(),
                   Token::word("b"));

        assert_eq!(tokenizer.next().unwrap().unwrap(),
                   Token::word("c"));

        assert_eq!(tokenizer.next().unwrap().unwrap(),
                   Token::new_line());
    }
}
<|endoftext|>


<commit_before>use std::io::prelude::*;
use std::io::BufReader;
use std::str;

//1 group = 512 bits = 16 * 32 bits
//512 bits = 64 bytes = 16 * 4 bytes
//448 bits = 56 bytes
//32 bits = 4 bytes
//MD5 give Hex chars out
/*
fn read_group<R>(reader: R) -> Vec<u8>
where
    R: Read,
{
    let mut buf = vec![];

    let mut chunk = reader.take(64);
    // Do appropriate error handling for your situation
    let n = chunk.read_to_end(&mut buf).expect("Read issue error:");

    if n < 64 {
        if n < 56 {
            buf.push(0x10u8);
            for _ in 0..(55 - n) {
                buf.push(0x00u8)
            }
        } else if n > 56 {
            buf.push(0x10u8);
            for _ in 0..(63 - n) {
                buf.push(0x00u8)
            }
            for _ in 0..56 {
                buf.push(0x00u8)
            }
        }
        return buf;
    }

    buf
}*/

fn read_group<R>(reader: R) -> Vec<u8>
where
    R: Read,
{
    let mut buf = vec![];
    let bytes = reader.bytes();
    let mut time = 0;

    let mut origin_len = 0;
    for byte in bytes {
        time += 1;
        origin_len += 1;
        buf.push(byte.unwrap());
        if time == 64 {
            time = 0;
        }
    }

    if time < 64 {
        if time < 56 {
            buf.push(0x10u8);
            for _ in 0..(55 - time) {
                buf.push(0x00u8)
            }
        } else if time > 56 {
            buf.push(0x10u8);
            for _ in 0..(63 - time) {
                buf.push(0x00u8)
            }
            for _ in 0..56 {
                buf.push(0x00u8)
            }
        }
    }

    //length of orginal data tail 64 bits
    //let mut len_data = (origin_len * 8) % u64::max_value();
    //len_data & 0xff00000000000000

    //let mut bit_cache = 0xff00000000000000;
    //for i in 0..8 {
    //    buf.push(((len_data & bit_cache) >> (64 - i * 8)) as u8);
    //    len_data >>= (64 - i * 8)
    //}

    buf
}

fn last_64_bits(length: &u64) -> Vec<u8> {
    let mut len_last64 = length % u64::max_value();
    let mut bit_cache = 0xff00000000000000;
    let mut buf: Vec<u8> = vec![];

    for i in 0..8 {
        buf.push(((len_last64 & bit_cache) >> (56 - i * 8)) as u8);
        bit_cache >>= 8;
    }

    return buf;
}

fn main() {
    let input_data = b"123456789A123456789B123456789C123456789D123456789E123456789F";
    let mut reader = BufReader::new(&input_data[..]);

    let word = read_group(&mut reader);
    println!("{}", word.len());
    println!("{:?}", str::from_utf8(&word));

    println!("{}", 18446744073709551615 % u64::max_value());
    println!("{}", 184 % u64::max_value());
    println!("{}", 0xafu8 >> 2); //10101111 -> 00101011

    println!("{}", 12249790986447749120 & 0xff00000000000000 as u64);

    println!("{:?}", last_64_bits(&(0x1200130014001500 as u64)));
}
<commit_msg>a bit confuse about length to last 64 bits..<commit_after>use std::io::prelude::*;
use std::io::BufReader;
use std::str;

//1 group = 512 bits = 16 * 32 bits
//512 bits = 64 bytes = 16 * 4 bytes
//448 bits = 56 bytes
//32 bits = 4 bytes
//MD5 give Hex chars out

fn read_group<R>(reader: R) -> Vec<u8>
where
    R: Read,
{
    let mut buf = vec![];
    let bytes = reader.bytes();
    let mut time = 0;

    let mut origin_len = 0;
    for byte in bytes {
        time += 1;
        origin_len += 1;
        buf.push(byte.unwrap());
        if time == 64 {
            time = 0;
        }
    }

    if time < 64 {
        if time < 56 {
            buf.push(0x10u8);
            for _ in 0..(55 - time) {
                buf.push(0x00u8)
            }
        } else if time > 56 {
            buf.push(0x10u8);
            for _ in 0..(63 - time) {
                buf.push(0x00u8)
            }
            for _ in 0..56 {
                buf.push(0x00u8)
            }
        }
    }

    //println!("{:?}", last_64_bits(&origin_len));

    buf.append(&mut last_64_bits(&origin_len));

    buf
}

//Put length in and return last 64 bits
fn last_64_bits(length: &u64) -> Vec<u8> {
    let mut len_last64 = length % u64::max_value();
    let mut bit_cache = 0xff00000000000000;
    let mut buf: Vec<u8> = vec![];

    for i in 0..8 {
        buf.push(((len_last64 & bit_cache) >> (56 - i * 8)) as u8);
        bit_cache >>= 8;
    }

    return buf;
}

//A=0x67452301, B=0xefcdab89, C=0x98badcfe, D=0x10325476

fn main() {
    let input_data = b"123456789A123456789B123456789C123456789D123456789E123456789F";
    let mut reader = BufReader::new(&input_data[..]);

    let word = read_group(&mut reader);
    println!("{}", word.len());
    println!("{:?}", str::from_utf8(&word));

    println!("{}", 18446744073709551615 % u64::max_value());
    println!("{}", 184 % u64::max_value());
    println!("{}", 0xafu8 >> 2); //10101111 -> 00101011

    println!("{}", 12249790986447749120 & 0xff00000000000000 as u64);

    println!("{:?}", last_64_bits(&(0x1200130014001500 as u64)));
}
<|endoftext|>


<commit_before>//! A module for user processes
// So far it just contains some test code

use core::fmt::Write;
use core::ptr;

use concurrency::{Semaphore, StaticSemaphore};
use io::stream::InputStream;
use vga::rectangle::{Rectangle, Color};
use vga::input::{InputElement, TextBox};
use super::Process;
use super::focus::focus;
use super::ready_queue;
use super::proc_table::PROCESS_TABLE;

// useful constants
const ROWS: usize = 25;
const COLS: usize = 80;
const NUM_LOOP: usize = (ROWS*2 + COLS*2 - 4) * 3;

// Some test semaphores
static mut current: (usize, usize) = (0,0);
static mut s1: StaticSemaphore = StaticSemaphore::new(1);
static mut s2: StaticSemaphore = StaticSemaphore::new(1);

// Some test routines
pub fn run(this: &Process) -> usize {

    // grab focus
    focus(None);

    let mut w0 = Rectangle::new(COLS, ROWS, (0, 0));
    let mut msg_sem = unsafe {
        let s = 0xD000_0000 as *mut Semaphore<Rectangle>;
        ptr::write(s, Semaphore::new(Rectangle::new(60, 7, (1,1)), 1));
        &mut *s
    };

    unsafe { *(0xF000_0000 as *mut usize) = this.pid; }

    w0.set_bg(Color::LightBlue);
    w0.paint();

    let mut msg = msg_sem.down();
    msg.set_cursor((0,0));
    msg.set_bg(Color::LightGray);
    msg.set_fg(Color::Black);

    msg.put_str("<-- If semaphores work correctly, then only this block \
                should be red when all loop_procs finish running.");

    for _ in 0..NUM_LOOP {
        ready_queue::make_ready(Process::new("loop_proc", self::run2));
        unsafe { s1.down(); }

        // test vm
        if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
            panic!("Oh no! *0xF000_0000 should be {} but is {}",
                   this.pid, unsafe { *(0xF000_0000 as *mut usize) });
        }
    }

    msg.put_str("\n\nYay! :)");

    // test share-page IPC
    unsafe {
        // create another process
        ready_queue::make_ready(Process::new("semaphore_test", self::run3));

        // share the semaphore
        let me = &mut *PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");
        if !me.addr_space.request_share(this.pid + NUM_LOOP + 1, 0xD000_0000) {
            panic!("Share request failed!");
        }
    }

    msg.put_str("\n\nNow test IPC... ");

    0
}

fn get_prev((r,c): (usize, usize)) -> (usize, usize) {
    if r == 0 && c > 0 { // top
        (0, c-1)
    } else if c == COLS-1 { // right
        (r-1, c)
    } else if r == ROWS-1 { // bottom
        (r, c+1)
    } else { // left
        (r+1, c)
    }
}

fn get_next((r,c): (usize, usize)) -> (usize, usize) {
    if r == 0 && c < COLS-1 { // top
        (0, c+1)
    } else if c == COLS-1 && r < ROWS-1 { // right
        (r+1, c)
    } else if r == ROWS-1 && c > 0 { // bottom
        (r, c-1)
    } else { // left
        (r-1, c)
    }
}

fn run2(this: &Process) -> usize {

    unsafe { *(0xF000_0000 as *mut usize) = this.pid; }

    // test vm
    if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
        panic!("Oh no! *0xF000_0000 should be {} but is {}",
               this.pid, unsafe { *(0xF000_0000 as *mut usize) });
    }

    unsafe { s2.down(); }

    let mut w = Rectangle::new(COLS,ROWS, (0,0));

    let me = unsafe { current };
    let prev = get_prev(me);

    unsafe {
        current = get_next(me);
    }

    printf!("Erase ({},{}) ", prev.0, prev.1);
    w.set_bg(Color::LightBlue);
    w.set_cursor(prev);
    w.put_char(' ');

    printf!("Draw ({},{})\n", me.0,me.1);
    w.set_bg(Color::Red);
    w.set_cursor(me);
    w.put_char(' ');

    unsafe { s2.up(); }

    // test vm
    if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
        panic!("Oh no! *0xF000_0000 should be {} but is {}",
               this.pid, unsafe { *(0xF000_0000 as *mut usize) });
    }

    // test process table
    unsafe {
        let me = PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");

        if (me as *const Process) != (this as *const Process) {
            panic!("Oh no! me = 0x{:X}, but should be 0x{:X}",
                   me as usize, this as *const Process as usize);
        }
    }

    unsafe { s1.up(); }

    0
}

fn run3(this: &Process) -> usize {
    unsafe {
        let me = &mut *PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");

        // accept share from the parent process
        if !me.addr_space.accept_share(3, 0xF000_0000) {
            panic!("Share accept failed");
        }

        let mut msg_sem = {
            let s = 0xF000_0000 as *mut Semaphore<Rectangle>;
            &mut *s
        };

        let mut msg = msg_sem.down();

        msg.put_str("Success! :D");
    }

    ready_queue::make_ready(Process::new("kbd_proc", self::run4));

    0
}

fn run4(_: &Process) -> usize {
    // test the keyboard

    let mut label = Rectangle::new(17,1,(8,1));
    label.set_bg(Color::Black);
    label.set_fg(Color::Yellow);
    label.set_cursor((0,0));
    label.put_str("Enter your name: ");

    let mut input_box = TextBox::new(40, (8,18));
    let string = input_box.get_str();

    let mut b = Rectangle::new(70, 1, (10, 1));
    b.set_bg(Color::LightGray);
    b.set_fg(Color::Black);
    b.set_cursor((0,0));

    let _ = write!(&mut b, "Hello, {}!\n", string);

    0
}
<commit_msg>Make output box larger at the request of my sister<commit_after>//! A module for user processes
// So far it just contains some test code

use core::fmt::Write;
use core::ptr;

use concurrency::{Semaphore, StaticSemaphore};
use io::stream::InputStream;
use vga::rectangle::{Rectangle, Color};
use vga::input::{InputElement, TextBox};
use super::Process;
use super::focus::focus;
use super::ready_queue;
use super::proc_table::PROCESS_TABLE;

// useful constants
const ROWS: usize = 25;
const COLS: usize = 80;
const NUM_LOOP: usize = (ROWS*2 + COLS*2 - 4) * 3;

// Some test semaphores
static mut current: (usize, usize) = (0,0);
static mut s1: StaticSemaphore = StaticSemaphore::new(1);
static mut s2: StaticSemaphore = StaticSemaphore::new(1);

// Some test routines
pub fn run(this: &Process) -> usize {

    // grab focus
    focus(None);

    let mut w0 = Rectangle::new(COLS, ROWS, (0, 0));
    let mut msg_sem = unsafe {
        let s = 0xD000_0000 as *mut Semaphore<Rectangle>;
        ptr::write(s, Semaphore::new(Rectangle::new(60, 7, (1,1)), 1));
        &mut *s
    };

    unsafe { *(0xF000_0000 as *mut usize) = this.pid; }

    w0.set_bg(Color::LightBlue);
    w0.paint();

    let mut msg = msg_sem.down();
    msg.set_cursor((0,0));
    msg.set_bg(Color::LightGray);
    msg.set_fg(Color::Black);

    msg.put_str("<-- If semaphores work correctly, then only this block \
                should be red when all loop_procs finish running.");

    for _ in 0..NUM_LOOP {
        ready_queue::make_ready(Process::new("loop_proc", self::run2));
        unsafe { s1.down(); }

        // test vm
        if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
            panic!("Oh no! *0xF000_0000 should be {} but is {}",
                   this.pid, unsafe { *(0xF000_0000 as *mut usize) });
        }
    }

    msg.put_str("\n\nYay! :)");

    // test share-page IPC
    unsafe {
        // create another process
        ready_queue::make_ready(Process::new("semaphore_test", self::run3));

        // share the semaphore
        let me = &mut *PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");
        if !me.addr_space.request_share(this.pid + NUM_LOOP + 1, 0xD000_0000) {
            panic!("Share request failed!");
        }
    }

    msg.put_str("\n\nNow test IPC... ");

    0
}

fn get_prev((r,c): (usize, usize)) -> (usize, usize) {
    if r == 0 && c > 0 { // top
        (0, c-1)
    } else if c == COLS-1 { // right
        (r-1, c)
    } else if r == ROWS-1 { // bottom
        (r, c+1)
    } else { // left
        (r+1, c)
    }
}

fn get_next((r,c): (usize, usize)) -> (usize, usize) {
    if r == 0 && c < COLS-1 { // top
        (0, c+1)
    } else if c == COLS-1 && r < ROWS-1 { // right
        (r+1, c)
    } else if r == ROWS-1 && c > 0 { // bottom
        (r, c-1)
    } else { // left
        (r-1, c)
    }
}

fn run2(this: &Process) -> usize {

    unsafe { *(0xF000_0000 as *mut usize) = this.pid; }

    // test vm
    if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
        panic!("Oh no! *0xF000_0000 should be {} but is {}",
               this.pid, unsafe { *(0xF000_0000 as *mut usize) });
    }

    unsafe { s2.down(); }

    let mut w = Rectangle::new(COLS,ROWS, (0,0));

    let me = unsafe { current };
    let prev = get_prev(me);

    unsafe {
        current = get_next(me);
    }

    printf!("Erase ({},{}) ", prev.0, prev.1);
    w.set_bg(Color::LightBlue);
    w.set_cursor(prev);
    w.put_char(' ');

    printf!("Draw ({},{})\n", me.0,me.1);
    w.set_bg(Color::Red);
    w.set_cursor(me);
    w.put_char(' ');

    unsafe { s2.up(); }

    // test vm
    if unsafe { *(0xF000_0000 as *mut usize) } != this.pid {
        panic!("Oh no! *0xF000_0000 should be {} but is {}",
               this.pid, unsafe { *(0xF000_0000 as *mut usize) });
    }

    // test process table
    unsafe {
        let me = PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");

        if (me as *const Process) != (this as *const Process) {
            panic!("Oh no! me = 0x{:X}, but should be 0x{:X}",
                   me as usize, this as *const Process as usize);
        }
    }

    unsafe { s1.up(); }

    0
}

fn run3(this: &Process) -> usize {
    unsafe {
        let me = &mut *PROCESS_TABLE.get(this.pid).expect("Oh no! expected Some(process)!");

        // accept share from the parent process
        if !me.addr_space.accept_share(3, 0xF000_0000) {
            panic!("Share accept failed");
        }

        let mut msg_sem = {
            let s = 0xF000_0000 as *mut Semaphore<Rectangle>;
            &mut *s
        };

        let mut msg = msg_sem.down();

        msg.put_str("Success! :D");
    }

    ready_queue::make_ready(Process::new("kbd_proc", self::run4));

    0
}

fn run4(_: &Process) -> usize {
    // test the keyboard

    let mut label = Rectangle::new(17,1,(8,1));
    label.set_bg(Color::Black);
    label.set_fg(Color::Yellow);
    label.set_cursor((0,0));
    label.put_str("Enter your name: ");

    let mut input_box = TextBox::new(40, (8,18));
    let string = input_box.get_str();

    let mut b = Rectangle::new(70, 10, (10, 1));
    b.set_bg(Color::LightGray);
    b.set_fg(Color::Black);
    b.set_cursor((0,0));

    let _ = write!(&mut b, "Hello, {}!\n", string);

    0
}
<|endoftext|>