<commit_before>use bio::io::fastq;
use rust_htslib::bam;
use rust_htslib::bam::Read;
use std::fs;
use std::process::Command;

/// Compare an output file to the expected output and delete the output file.
fn test_output(result: &str, expected: &str) {
    assert!(Command::new("cmp")
        .arg(result)
        .arg(expected)
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    fs::remove_file(result).unwrap();
}

/// Compare two fastq files, ignoring the name lines
/// Reads are sorted by their sequence, which is not 100% robust
/// if mutations/ sequencing errors are considered.
fn compare_fastq(result: &str, expected: &str) {
    let result_reader = fastq::Reader::from_file(result).unwrap();
    let mut result_recs: Vec<fastq::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().to_owned());
    let expected_reader = fastq::Reader::from_file(expected).unwrap();
    let mut expected_recs: Vec<fastq::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().to_owned());

    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq(), expected.seq());
        assert_eq!(result.qual(), expected.qual());
    }
}

fn compare_bam(result: &str, expected: &str) {
    let mut result_reader = bam::Reader::from_path(result).unwrap();
    let mut result_recs: Vec<bam::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    let mut expected_reader = bam::Reader::from_path(expected).unwrap();
    let mut expected_recs: Vec<bam::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq().as_bytes(), expected.seq().as_bytes());
        assert_eq!(result.qual(), expected.qual());
    }
}

#[test]
fn fastq_split() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt fastq-split tests/A.fastq tests/B.fastq < tests/test.fastq")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/A.fastq", "tests/expected/A.fastq");
    test_output("tests/B.fastq", "tests/expected/B.fastq");
}

#[test]
fn fastq_filter() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt fastq-filter tests/ids.txt < tests/test.fastq > tests/filtered.fastq"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/filtered.fastq", "tests/expected/B.fastq");
}

#[test]
fn bam_depth() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt bam-depth tests/test.bam < tests/pos.txt > tests/depth.txt")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/depth.txt", "tests/expected/depth.txt");
}

#[test]
fn vcf_to_txt() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-to-txt --genotypes --fmt S --info T X SOMATIC < tests/test.vcf > tests/variant-table.txt")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/variant-table.txt",
        "tests/expected/variant-table.txt",
    );
}

#[test]
fn vcf_match() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test3.vcf < tests/test2.vcf > tests/matching.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output("tests/matching.bcf", "tests/expected/matching.bcf");
}

#[test]
fn vcf_match_same() {
    assert!(Command::new("bash").arg("-c")
                                .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test4.vcf < tests/test4.vcf > tests/matching-same.bcf")
                                .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/matching-same.bcf",
        "tests/expected/matching-same.bcf",
    );
}

#[test]
fn vcf_fix_iupac_alleles() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt vcf-fix-iupac-alleles < tests/test-iupac.vcf > tests/iupac-fixed.bcf"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/iupac-fixed.bcf", "tests/expected/iupac-fixed.bcf");
}

#[test]
fn vcf_baf() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt vcf-baf < tests/test-freebayes.vcf > tests/baf.bcf")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/baf.bcf", "tests/expected/baf.bcf");
}

#[test]
fn test_vcf_report() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-report tests/ref.fa -v a=tests/report-test.vcf -v b=tests/report-test.vcf -b a=tests/test-report.bam -b b=tests/test-report.bam -- tests")
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success()
    );
    let files1 = vec![
        (
            "tests/indexes/index1.html",
            "tests/expected/report/indexes/index1.html",
        ),
        (
            "tests/genes/KRAS1.html",
            "tests/expected/report/genes/KRAS1.html",
        ),
    ];

    let files2 = vec![
        (
            "tests/details/a/KRAS.html",
            "tests/expected/report/details/a/KRAS.html",
        ),
        (
            "tests/details/b/KRAS.html",
            "tests/expected/report/details/b/KRAS.html",
        ),
    ];

    for (result, expected) in files1 {
        // delete line 22 with timestamp and 15 with version
        // this may fail on OS X due to the wrong sed being installed
        assert!(Command::new("bash")
            .arg("-c")
            .arg("sed -i '22d;15d' ".to_owned() + result)
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success());
        test_output(result, expected)
    }
    for (result, expected) in files2 {
        // delete line 29 with timestamp and 22 with version
        // this may fail on OS X due to the wrong sed being installed
        assert!(Command::new("bash")
            .arg("-c")
            .arg("sed -i '29d;22d' ".to_owned() + result)
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success());
        test_output(result, expected)
    }
    fs::remove_dir_all("tests/genes").unwrap();
    fs::remove_dir_all("tests/details").unwrap();
    fs::remove_dir_all("tests/js").unwrap();
    fs::remove_dir_all("tests/css").unwrap();
    fs::remove_dir_all("tests/indexes").unwrap();
    fs::remove_file("tests/index.html").unwrap();
}

#[test]
fn test_collapse_reads_to_fragments_two_cluster() {
    assert!(
        Command::new("bash")
                .arg("-c")
                .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 3 -u --max-umi-dist 0 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus.1.fastq /tmp/test-consensus.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus.1.fastq",
        "tests/expected/test-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus.2.fastq",
        "tests/expected/test-consensus.2.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_single_cluster() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 3 -u --max-umi-dist 2 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus_single.1.fastq /tmp/test-consensus_single.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus_single.1.fastq",
        "tests/expected/test-consensus_single.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus_single.2.fastq",
        "tests/expected/test-consensus_single.2.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_reads() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 10 --max-umi-dist 0 --max-seq-dist 8 --insert-size 450 --std-dev 50  tests/overlapping-consensus.1.fastq tests/overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.1.fastq /tmp/test_overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.3.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test_overlapping-consensus.1.fastq",
        "tests/expected/test_overlapping-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.2.fastq",
        "tests/expected/test_overlapping-consensus.2.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.3.fastq",
        "tests/expected/test_overlapping-consensus.3.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_from_bam() {
    assert!(
    Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt collapse-reads-to-fragments bam --max-seq-dist 8 tests/overlapping_consensus_marked.bam /tmp/overlapping_consensus_marked.bam")
        .spawn().unwrap().wait().unwrap().success());
    compare_bam(
        "/tmp/overlapping_consensus_marked.bam",
        "tests/expected/overlapping_consensus_marked.bam",
    );
}

#[test]
fn test_vcf_annotate_dgidb() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-annotate-dgidb tests/annotate_dgidb_test.vcf > /tmp/annotate_dgidb_test.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "/tmp/annotate_dgidb_test.bcf",
        "tests/expected/annotate_dgidb_test.bcf",
    );
}

#[test]
fn test_vcf_annotate_dgidb_drugbank() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-annotate-dgidb tests/annotate_dgidb_test.vcf -s DrugBank> /tmp/annotate_dgidb_drugbank_test.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "/tmp/annotate_dgidb_drugbank_test.bcf",
        "tests/expected/annotate_dgidb_drugbank_test.bcf",
    );
}

#[test]
fn test_stats_fasta_file() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt sequence-stats < tests/stats.fasta > /tmp/result.fasta.stats")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());

    test_output(
        "/tmp/result.fasta.stats",
        "tests/expected/result.fasta.stats",
    );
}

#[test]
fn test_stats_fastq_file() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt sequence-stats -q < tests/stats.fastq > /tmp/result.fastq.stats")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());

    test_output(
        "/tmp/result.fastq.stats",
        "tests/expected/result.fastq.stats",
    );
}
<commit_msg>add missing whitespace in test_vcf_annotate_dgidb_drugbank<commit_after>use bio::io::fastq;
use rust_htslib::bam;
use rust_htslib::bam::Read;
use std::fs;
use std::process::Command;

/// Compare an output file to the expected output and delete the output file.
fn test_output(result: &str, expected: &str) {
    assert!(Command::new("cmp")
        .arg(result)
        .arg(expected)
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    fs::remove_file(result).unwrap();
}

/// Compare two fastq files, ignoring the name lines
/// Reads are sorted by their sequence, which is not 100% robust
/// if mutations/ sequencing errors are considered.
fn compare_fastq(result: &str, expected: &str) {
    let result_reader = fastq::Reader::from_file(result).unwrap();
    let mut result_recs: Vec<fastq::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().to_owned());
    let expected_reader = fastq::Reader::from_file(expected).unwrap();
    let mut expected_recs: Vec<fastq::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().to_owned());

    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq(), expected.seq());
        assert_eq!(result.qual(), expected.qual());
    }
}

fn compare_bam(result: &str, expected: &str) {
    let mut result_reader = bam::Reader::from_path(result).unwrap();
    let mut result_recs: Vec<bam::Record> =
        result_reader.records().filter_map(Result::ok).collect();
    result_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    let mut expected_reader = bam::Reader::from_path(expected).unwrap();
    let mut expected_recs: Vec<bam::Record> =
        expected_reader.records().filter_map(Result::ok).collect();
    expected_recs.sort_by_key(|x| x.seq().as_bytes().to_owned());
    for (result, expected) in result_recs.iter().zip(expected_recs.iter()) {
        assert_eq!(result.seq().as_bytes(), expected.seq().as_bytes());
        assert_eq!(result.qual(), expected.qual());
    }
}

#[test]
fn fastq_split() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt fastq-split tests/A.fastq tests/B.fastq < tests/test.fastq")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/A.fastq", "tests/expected/A.fastq");
    test_output("tests/B.fastq", "tests/expected/B.fastq");
}

#[test]
fn fastq_filter() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt fastq-filter tests/ids.txt < tests/test.fastq > tests/filtered.fastq"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/filtered.fastq", "tests/expected/B.fastq");
}

#[test]
fn bam_depth() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt bam-depth tests/test.bam < tests/pos.txt > tests/depth.txt")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/depth.txt", "tests/expected/depth.txt");
}

#[test]
fn vcf_to_txt() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-to-txt --genotypes --fmt S --info T X SOMATIC < tests/test.vcf > tests/variant-table.txt")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/variant-table.txt",
        "tests/expected/variant-table.txt",
    );
}

#[test]
fn vcf_match() {
    assert!(Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test3.vcf < tests/test2.vcf > tests/matching.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output("tests/matching.bcf", "tests/expected/matching.bcf");
}

#[test]
fn vcf_match_same() {
    assert!(Command::new("bash").arg("-c")
                                .arg("target/debug/rbt vcf-match -d 50 -l 20 tests/test4.vcf < tests/test4.vcf > tests/matching-same.bcf")
                                .spawn().unwrap().wait().unwrap().success());
    test_output(
        "tests/matching-same.bcf",
        "tests/expected/matching-same.bcf",
    );
}

#[test]
fn vcf_fix_iupac_alleles() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg(
            "target/debug/rbt vcf-fix-iupac-alleles < tests/test-iupac.vcf > tests/iupac-fixed.bcf"
        )
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/iupac-fixed.bcf", "tests/expected/iupac-fixed.bcf");
}

#[test]
fn vcf_baf() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt vcf-baf < tests/test-freebayes.vcf > tests/baf.bcf")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());
    test_output("tests/baf.bcf", "tests/expected/baf.bcf");
}

#[test]
fn test_vcf_report() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-report tests/ref.fa -v a=tests/report-test.vcf -v b=tests/report-test.vcf -b a=tests/test-report.bam -b b=tests/test-report.bam -- tests")
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success()
    );
    let files1 = vec![
        (
            "tests/indexes/index1.html",
            "tests/expected/report/indexes/index1.html",
        ),
        (
            "tests/genes/KRAS1.html",
            "tests/expected/report/genes/KRAS1.html",
        ),
    ];

    let files2 = vec![
        (
            "tests/details/a/KRAS.html",
            "tests/expected/report/details/a/KRAS.html",
        ),
        (
            "tests/details/b/KRAS.html",
            "tests/expected/report/details/b/KRAS.html",
        ),
    ];

    for (result, expected) in files1 {
        // delete line 22 with timestamp and 15 with version
        // this may fail on OS X due to the wrong sed being installed
        assert!(Command::new("bash")
            .arg("-c")
            .arg("sed -i '22d;15d' ".to_owned() + result)
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success());
        test_output(result, expected)
    }
    for (result, expected) in files2 {
        // delete line 29 with timestamp and 22 with version
        // this may fail on OS X due to the wrong sed being installed
        assert!(Command::new("bash")
            .arg("-c")
            .arg("sed -i '29d;22d' ".to_owned() + result)
            .spawn()
            .unwrap()
            .wait()
            .unwrap()
            .success());
        test_output(result, expected)
    }
    fs::remove_dir_all("tests/genes").unwrap();
    fs::remove_dir_all("tests/details").unwrap();
    fs::remove_dir_all("tests/js").unwrap();
    fs::remove_dir_all("tests/css").unwrap();
    fs::remove_dir_all("tests/indexes").unwrap();
    fs::remove_file("tests/index.html").unwrap();
}

#[test]
fn test_collapse_reads_to_fragments_two_cluster() {
    assert!(
        Command::new("bash")
                .arg("-c")
                .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 3 -u --max-umi-dist 0 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus.1.fastq /tmp/test-consensus.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus.1.fastq",
        "tests/expected/test-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus.2.fastq",
        "tests/expected/test-consensus.2.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_single_cluster() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 3 -u --max-umi-dist 2 --max-seq-dist 2 tests/test-consensus.fastq tests/test-consensus.fastq /tmp/test-consensus_single.1.fastq /tmp/test-consensus_single.2.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test-consensus_single.1.fastq",
        "tests/expected/test-consensus_single.1.fastq",
    );
    compare_fastq(
        "/tmp/test-consensus_single.2.fastq",
        "tests/expected/test-consensus_single.2.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_reads() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt collapse-reads-to-fragments fastq --umi-len 10 --max-umi-dist 0 --max-seq-dist 8 --insert-size 450 --std-dev 50  tests/overlapping-consensus.1.fastq tests/overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.1.fastq /tmp/test_overlapping-consensus.2.fastq /tmp/test_overlapping-consensus.3.fastq")
            .spawn().unwrap().wait().unwrap().success());
    compare_fastq(
        "/tmp/test_overlapping-consensus.1.fastq",
        "tests/expected/test_overlapping-consensus.1.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.2.fastq",
        "tests/expected/test_overlapping-consensus.2.fastq",
    );
    compare_fastq(
        "/tmp/test_overlapping-consensus.3.fastq",
        "tests/expected/test_overlapping-consensus.3.fastq",
    );
}

#[test]
fn test_collapse_reads_to_fragments_from_bam() {
    assert!(
    Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt collapse-reads-to-fragments bam --max-seq-dist 8 tests/overlapping_consensus_marked.bam /tmp/overlapping_consensus_marked.bam")
        .spawn().unwrap().wait().unwrap().success());
    compare_bam(
        "/tmp/overlapping_consensus_marked.bam",
        "tests/expected/overlapping_consensus_marked.bam",
    );
}

#[test]
fn test_vcf_annotate_dgidb() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-annotate-dgidb tests/annotate_dgidb_test.vcf > /tmp/annotate_dgidb_test.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "/tmp/annotate_dgidb_test.bcf",
        "tests/expected/annotate_dgidb_test.bcf",
    );
}

#[test]
fn test_vcf_annotate_dgidb_drugbank() {
    assert!(
        Command::new("bash")
            .arg("-c")
            .arg("target/debug/rbt vcf-annotate-dgidb tests/annotate_dgidb_test.vcf -s DrugBank > /tmp/annotate_dgidb_drugbank_test.bcf")
            .spawn().unwrap().wait().unwrap().success());
    test_output(
        "/tmp/annotate_dgidb_drugbank_test.bcf",
        "tests/expected/annotate_dgidb_drugbank_test.bcf",
    );
}

#[test]
fn test_stats_fasta_file() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt sequence-stats < tests/stats.fasta > /tmp/result.fasta.stats")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());

    test_output(
        "/tmp/result.fasta.stats",
        "tests/expected/result.fasta.stats",
    );
}

#[test]
fn test_stats_fastq_file() {
    assert!(Command::new("bash")
        .arg("-c")
        .arg("target/debug/rbt sequence-stats -q < tests/stats.fastq > /tmp/result.fastq.stats")
        .spawn()
        .unwrap()
        .wait()
        .unwrap()
        .success());

    test_output(
        "/tmp/result.fastq.stats",
        "tests/expected/result.fastq.stats",
    );
}
<|endoftext|>


<commit_before>//! Test client and server TLS connected with TLS.

extern crate bytes;
extern crate futures;
extern crate tls_api;
extern crate tls_api_native_tls;
extern crate tokio_core;
extern crate tokio_tls_api;
extern crate httpbis;
extern crate log;
extern crate env_logger;

use bytes::Bytes;

use std::sync::Arc;

use futures::future::Future;

use httpbis::solicit::header::Headers;
use httpbis::*;
use httpbis::message::SimpleHttpMessage;

use httpbis::socket::AnySocketAddr;

use tls_api::Certificate;
use tls_api_native_tls::TlsAcceptor;
use tls_api_native_tls::TlsAcceptorBuilder;
use tls_api_native_tls::TlsConnector;
use tls_api::TlsAcceptorBuilder as tls_api_TlsAcceptorBuilder;
use tls_api::TlsConnector as tls_api_TlsConnector;
use tls_api::TlsConnectorBuilder;

fn test_tls_acceptor() -> TlsAcceptor {
    let pkcs12 = include_bytes!("identity.p12");
    let builder = TlsAcceptorBuilder::from_pkcs12(pkcs12, "mypass").unwrap();
    builder.build().unwrap()
}

fn test_tls_connector() -> TlsConnector {
    let root_ca = include_bytes!("root-ca.der");
    let root_ca = Certificate::from_der(root_ca.to_vec());

    let mut builder = TlsConnector::builder().unwrap();
    builder.add_root_certificate(root_ca).expect("add_root_certificate");
    builder.build().unwrap()
}

#[test]
fn tls() {
    struct ServiceImpl {
    }

    impl Service for ServiceImpl {
        fn start_request(&self, _headers: Headers, _req: HttpPartStream) -> Response {
            Response::headers_and_bytes(Headers::ok_200(), Bytes::from("hello"))
        }
    }

    let mut server = ServerBuilder::new();
    server.set_port(0);
    server.set_tls(test_tls_acceptor());
    server.service.set_service("/", Arc::new(ServiceImpl {}));
    let server = server.build().expect("server");

    let socket_addr = match server.local_addr() {
        &AnySocketAddr::Inet(ref sock_addr) => sock_addr,
        _ => panic!("Assumed server was an inet server")
    };

    let client: Client = Client::new_expl(
        socket_addr,
        ClientTlsOption::Tls("foobar.com".to_owned(), Arc::new(test_tls_connector())),
        Default::default())
            .expect("http client");

    let resp: SimpleHttpMessage = client.start_get("/hi", "localhost").collect().wait().unwrap();
    assert_eq!(200, resp.headers.status());
    assert_eq!(&b"hello"[..], &resp.body[..]);
}
<commit_msg>Bind another test on IPv4 because travis-ci doesn't have IPv6<commit_after>//! Test client and server TLS connected with TLS.

extern crate bytes;
extern crate futures;
extern crate tls_api;
extern crate tls_api_native_tls;
extern crate tokio_core;
extern crate tokio_tls_api;
extern crate httpbis;
#[macro_use]
extern crate log;
extern crate env_logger;
extern crate regex;

use bytes::Bytes;

use std::sync::Arc;

use futures::future::Future;

use httpbis::solicit::header::Headers;
use httpbis::*;
use httpbis::message::SimpleHttpMessage;

use httpbis::socket::AnySocketAddr;

use tls_api::Certificate;
use tls_api_native_tls::TlsAcceptor;
use tls_api_native_tls::TlsAcceptorBuilder;
use tls_api_native_tls::TlsConnector;
use tls_api::TlsAcceptorBuilder as tls_api_TlsAcceptorBuilder;
use tls_api::TlsConnector as tls_api_TlsConnector;
use tls_api::TlsConnectorBuilder;

mod test_misc;

use test_misc::BIND_HOST;

fn test_tls_acceptor() -> TlsAcceptor {
    let pkcs12 = include_bytes!("identity.p12");
    let builder = TlsAcceptorBuilder::from_pkcs12(pkcs12, "mypass").unwrap();
    builder.build().unwrap()
}

fn test_tls_connector() -> TlsConnector {
    let root_ca = include_bytes!("root-ca.der");
    let root_ca = Certificate::from_der(root_ca.to_vec());

    let mut builder = TlsConnector::builder().unwrap();
    builder.add_root_certificate(root_ca).expect("add_root_certificate");
    builder.build().unwrap()
}

#[test]
fn tls() {
    struct ServiceImpl {
    }

    impl Service for ServiceImpl {
        fn start_request(&self, _headers: Headers, _req: HttpPartStream) -> Response {
            Response::headers_and_bytes(Headers::ok_200(), Bytes::from("hello"))
        }
    }

    let mut server = ServerBuilder::new();
    server.set_addr((BIND_HOST, 0)).expect("set_addr");
    server.set_tls(test_tls_acceptor());
    server.service.set_service("/", Arc::new(ServiceImpl {}));
    let server = server.build().expect("server");

    let socket_addr = match server.local_addr() {
        &AnySocketAddr::Inet(ref sock_addr) => sock_addr,
        _ => panic!("Assumed server was an inet server")
    };

    let client: Client = Client::new_expl(
        socket_addr,
        ClientTlsOption::Tls("foobar.com".to_owned(), Arc::new(test_tls_connector())),
        Default::default())
            .expect("http client");

    let resp: SimpleHttpMessage = client.start_get("/hi", "localhost").collect().wait().unwrap();
    assert_eq!(200, resp.headers.status());
    assert_eq!(&b"hello"[..], &resp.body[..]);
}
<|endoftext|>


<commit_before>extern crate ktensor as k;
use std::sync::{Arc};

#[test]
fn xor() {
    // Variables
    let input_x = Arc::new(k::Variable::new("input_x".to_string(), k::Vec2(0, 2)));
    let target_y = Arc::new(k::Variable::new("target_y".to_string(), k::Vec2(0, 2)));

    // Initialize
    let mut variable_context = k::Context::<f64>::with_capacity(2);
    k::variable::init_f64(vec![input_x.clone(), target_y.clone()], &mut variable_context);

    // Graph
    let layers: usize = 2;
    let mut states = Vec::<Arc<k::State>>::with_capacity(2 * layers);
    let mut graph_head: Arc<k::Graph<f64>> = input_x.clone();

    let w = Arc::new(k::State::new(format!("weight_w_{}", 1), k::Vec2(2, 4)));
    let b = Arc::new(k::State::new(format!("weight_b_{}", 1), k::Vec2(1, 4)));

    let dot = Arc::new(k::op::dot::<f64>(format!("layer_{}_dot", 1), graph_head.clone(), w.clone()));
    let add = Arc::new(k::op::add_f64(format!("layer_{}_add", 1), dot.clone(), b.clone()));

    let relu = Arc::new(k::op::relu_f64(format!("layer_{}_relu", 1), add.clone()));

    graph_head = relu.clone();

    states.push(w.clone());
    states.push(b.clone());

    let w2 = Arc::new(k::State::new(format!("weight_w_{}", 2), k::Vec2(4, 2)));
    let b2 = Arc::new(k::State::new(format!("weight_b_{}", 2), k::Vec2(1, 2)));

    let dot2 = Arc::new(k::op::dot::<f64>(format!("layer_{}_dot", 2), graph_head.clone(), w2.clone()));
    let add2 = Arc::new(k::op::add_f64(format!("layer_{}_add", 2), dot2.clone(), b2.clone()));

    let relu2 = Arc::new(k::op::relu_f64(format!("layer_{}_relu", 2), add2.clone()));

    graph_head = relu2.clone();

    states.push(w2.clone());
    states.push(b2.clone());

    let softmax2 = Arc::new(k::op::softmax_f64(format!("layer_{}_softmax", 2), graph_head.clone()));
    let xentropy2 = Arc::new(k::cost::softmax_cross_entropy_f64(format!("layer_{}_xentropy", 2), softmax2.clone(), target_y.clone()));

    // initialize states
    let mut state_context = k::Context::<f64>::with_capacity(2 * layers);
    k::state::init_f64(states, &mut state_context);

    let training_set = (
        k::Tensor::from_vec(k::Vec2(4, 2), vec![
        0.0, 0.0,
        0.0, 1.0,
        1.0, 0.0,
        1.0, 1.0,
        ]),
        k::Tensor::from_vec(k::Vec2(4, 2), vec![
        0.0, 1.0,
        1.0, 0.0,
        1.0, 0.0,
        0.0, 1.0,
        ]),
    );

    let training_vec = vec![
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 0.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 1.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,])),
    ];

    // variable_context.set(input_x.get_id(), training_set.0.clone());
    // variable_context.set(target_y.get_id(), training_set.1.clone());
    // println!("\ninput x");
    // let final_test = k::execute(input_x.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nweight w 1");
    // let final_test = k::execute(w.clone(), &state_context, &variable_context);
    // for i in 0..2 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\ndot");
    // let final_test = k::execute(dot.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nweight b 1");
    // let final_test = k::execute(b.clone(), &state_context, &variable_context);
    // for i in 0..1 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nadd");
    // let final_test = k::execute(add.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nrelu");
    // let final_test = k::execute(relu.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\ntarget y");
    // let final_test = k::execute(target_y.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nxentropy");
    // let final_test = k::execute(xentropy.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }

    let batch = true;
    let print_rate = 4096;

    let mut history = k::Context::<f64>::with_capacity(5 * layers + 4);

    if batch {
        variable_context.set(input_x.get_id(), training_set.0.clone());
        variable_context.set(target_y.get_id(), training_set.1.clone());
    }
    for i in 0..16384 {
        let (ref a, ref b) = training_vec[i % 4];
        if !batch {
            variable_context.set(input_x.get_id(), a.clone());
            variable_context.set(target_y.get_id(), b.clone());
        }
        k::train(xentropy2.clone(), &mut state_context, &variable_context, &mut history, -0.1);
        if i % print_rate == 0 {
            variable_context.set(input_x.get_id(), training_set.0.clone());
            variable_context.set(target_y.get_id(), training_set.1.clone());
            println!("\nxentropy2");
            let final_test = k::execute(xentropy2.clone(), &state_context, &variable_context);
            for i in 0..4 {
                println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
            }
        }
    }

    // println!("\ninput x");
    // let final_test = k::execute(input_x.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nweight w 1");
    // let final_test = k::execute(w.clone(), &state_context, &variable_context);
    // for i in 0..2 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\ndot");
    // let final_test = k::execute(dot.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nweight b 1");
    // let final_test = k::execute(b.clone(), &state_context, &variable_context);
    // for i in 0..1 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nadd");
    // let final_test = k::execute(add.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nrelu");
    // let final_test = k::execute(relu.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\ntarget y");
    // let final_test = k::execute(target_y.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }
    // println!("\nxentropy");
    // let final_test = k::execute(xentropy.clone(), &state_context, &variable_context);
    // for i in 0..4 {
    //     println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    // }

    variable_context.set(input_x.get_id(), training_set.0.clone());
    variable_context.set(target_y.get_id(), training_set.1.clone());
    println!("\nxentropy2");
    let final_test = k::execute(xentropy2.clone(), &state_context, &variable_context);
    for i in 0..4 {
        println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    }
}
<commit_msg>improve xor test doc<commit_after>extern crate ktensor as k;
use std::sync::{Arc};

#[test]
fn xor() {
    ///////////////
    // Variables //
    ///////////////

    let input_x = Arc::new(k::Variable::new("input_x".to_string(), k::Vec2(0, 2)));
    let target_y = Arc::new(k::Variable::new("target_y".to_string(), k::Vec2(0, 2)));

    // Initialize
    let mut variable_context = k::Context::<f64>::with_capacity(2);
    k::variable::init_f64(vec![input_x.clone(), target_y.clone()], &mut variable_context);

    ///////////
    // Graph //
    ///////////

    let layers: usize = 2;
    let mut states = Vec::<Arc<k::State>>::with_capacity(2 * layers);
    let mut graph_head: Arc<k::Graph<f64>> = input_x.clone();

    let w = Arc::new(k::State::new(format!("weight_w_{}", 1), k::Vec2(2, 4)));
    let b = Arc::new(k::State::new(format!("weight_b_{}", 1), k::Vec2(1, 4)));

    let dot = Arc::new(k::op::dot::<f64>(format!("layer_{}_dot", 1), graph_head.clone(), w.clone()));
    let add = Arc::new(k::op::add_f64(format!("layer_{}_add", 1), dot.clone(), b.clone()));

    let relu = Arc::new(k::op::relu_f64(format!("layer_{}_relu", 1), add.clone()));

    graph_head = relu.clone();

    states.push(w.clone());
    states.push(b.clone());

    let w2 = Arc::new(k::State::new(format!("weight_w_{}", 2), k::Vec2(4, 2)));
    let b2 = Arc::new(k::State::new(format!("weight_b_{}", 2), k::Vec2(1, 2)));

    let dot2 = Arc::new(k::op::dot::<f64>(format!("layer_{}_dot", 2), graph_head.clone(), w2.clone()));
    let add2 = Arc::new(k::op::add_f64(format!("layer_{}_add", 2), dot2.clone(), b2.clone()));

    let relu2 = Arc::new(k::op::relu_f64(format!("layer_{}_relu", 2), add2.clone()));

    graph_head = relu2.clone();

    states.push(w2.clone());
    states.push(b2.clone());

    let softmax2 = Arc::new(k::op::softmax_f64(format!("layer_{}_softmax", 2), graph_head.clone()));
    let xentropy2 = Arc::new(k::cost::softmax_cross_entropy_f64(format!("layer_{}_xentropy", 2), softmax2.clone(), target_y.clone()));

    // initialize states
    let mut state_context = k::Context::<f64>::with_capacity(2 * layers);
    k::state::init_f64(states, &mut state_context);

    //////////////
    // training //
    //////////////

    let training_set = (
        k::Tensor::from_vec(k::Vec2(4, 2), vec![
        0.0, 0.0,
        0.0, 1.0,
        1.0, 0.0,
        1.0, 1.0,
        ]),
        k::Tensor::from_vec(k::Vec2(4, 2), vec![
        0.0, 1.0,
        1.0, 0.0,
        1.0, 0.0,
        0.0, 1.0,
        ]),
    );

    let training_vec = vec![
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 0.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 0.0,])),
        (k::Tensor::from_vec(k::Vec2(1, 2), vec![1.0, 1.0,]), k::Tensor::from_vec(k::Vec2(1, 2), vec![0.0, 1.0,])),
    ];

    let batch = true;
    let learning_rate = -0.1;
    let print_rate = 4096;

    let mut history = k::Context::<f64>::with_capacity(5 * layers + 4);

    if batch {
        variable_context.set(input_x.get_id(), training_set.0.clone());
        variable_context.set(target_y.get_id(), training_set.1.clone());
    }
    for i in 0..16384 {
        let (ref a, ref b) = training_vec[i % 4];
        if !batch {
            variable_context.set(input_x.get_id(), a.clone());
            variable_context.set(target_y.get_id(), b.clone());
        }
        k::train(xentropy2.clone(), &mut state_context, &variable_context, &mut history, learning_rate);

        // test print
        if i % print_rate == 0 {
            variable_context.set(input_x.get_id(), training_set.0.clone());
            variable_context.set(target_y.get_id(), training_set.1.clone());
            println!("\nxentropy2");
            let final_test = k::execute(xentropy2.clone(), &state_context, &variable_context);
            for i in 0..4 {
                println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
            }
        }
    }

    ////////////////
    // final test //
    ////////////////

    variable_context.set(input_x.get_id(), training_set.0.clone());
    variable_context.set(target_y.get_id(), training_set.1.clone());
    println!("\nxentropy2");
    let final_test = k::execute(xentropy2.clone(), &state_context, &variable_context);
    for i in 0..4 {
        println!("{} {}", final_test.get(k::Vec2(i, 0)), final_test.get(k::Vec2(i, 1)));
    }
}
<|endoftext|>


<commit_before><commit_msg>libheap: Add Buddy::log2<commit_after><|endoftext|>


<commit_before>pub use self::Sha256;
extern crate crypto_hash;
use crypto_hash::{Hasher,Algorithm};

pub struct Sha256(Hasher);

impl Sha256 {
    pub fn new() -> Sha256 {
        let hasher = Hasher::new(Algorithm::sha256());
        Sha256(hasher)
    }

    pub fn update(&mut self, bytes: &[u8]) {
        let _ = self.0.write_all(bytes);
    }

    pub fn finish(&mut self) -> [u8; 32] {
        let mut ret = [0u8; 32];
        let data = self.0.finish();
        ret.copy_from_slice(&data[..]);
        ret
    }
}
<commit_msg>Fix compilation error<commit_after>extern crate crypto_hash;
use self::crypto_hash::{Hasher,Algorithm};
use std::io::Write;

pub struct Sha256(Hasher);

impl Sha256 {
    pub fn new() -> Sha256 {
        let hasher = Hasher::new(Algorithm::SHA256);
        Sha256(hasher)
    }

    pub fn update(&mut self, bytes: &[u8]) {
        let _ = self.0.write_all(bytes);
    }

    pub fn finish(&mut self) -> [u8; 32] {
        let mut ret = [0u8; 32];
        let data = self.0.finish();
        ret.copy_from_slice(&data[..]);
        ret
    }
}
<|endoftext|>


<commit_before>use benching_utils::{BeaconBlockBencher, BeaconStateBencher};
use criterion::Criterion;
use criterion::{black_box, Benchmark};
use state_processing::{
    per_block_processing,
    per_block_processing::{
        process_attestations, process_attester_slashings, process_deposits, process_eth1_data,
        process_exits, process_proposer_slashings, process_randao, process_transfers,
        verify_block_signature,
    },
};
use types::*;

/// Run the benchmarking suite on a foundation spec with 16,384 validators.
pub fn block_processing_16k_validators(c: &mut Criterion) {
    let spec = ChainSpec::foundation();

    let validator_count = 16_384;

    let (mut state, keypairs) = build_state(validator_count, &spec);
    let block = build_block(&mut state, &keypairs, &spec);

    assert_eq!(
        block.body.proposer_slashings.len(),
        spec.max_proposer_slashings as usize,
        "The block should have the maximum possible proposer slashings"
    );

    assert_eq!(
        block.body.attester_slashings.len(),
        spec.max_attester_slashings as usize,
        "The block should have the maximum possible attester slashings"
    );

    for attester_slashing in &block.body.attester_slashings {
        let len_1 = attester_slashing
            .slashable_attestation_1
            .validator_indices
            .len();
        let len_2 = attester_slashing
            .slashable_attestation_1
            .validator_indices
            .len();
        assert!(
            (len_1 == len_2) && (len_2 == spec.max_indices_per_slashable_vote as usize),
            "Each attester slashing should have the maximum possible validator indices"
        );
    }

    assert_eq!(
        block.body.attestations.len(),
        spec.max_attestations as usize,
        "The block should have the maximum possible attestations."
    );

    assert_eq!(
        block.body.deposits.len(),
        spec.max_deposits as usize,
        "The block should have the maximum possible deposits."
    );

    assert_eq!(
        block.body.voluntary_exits.len(),
        spec.max_voluntary_exits as usize,
        "The block should have the maximum possible voluntary exits."
    );

    assert_eq!(
        block.body.transfers.len(),
        spec.max_transfers as usize,
        "The block should have the maximum possible transfers."
    );

    bench_block_processing(
        c,
        &block,
        &state,
        &spec,
        &format!("{}_validators", validator_count),
    );
}

fn build_state(validator_count: usize, spec: &ChainSpec) -> (BeaconState, Vec<Keypair>) {
    let mut builder = BeaconStateBencher::new(validator_count, &spec);

    // Set the state to be just before an epoch transition.
    let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
    builder.teleport_to_slot(target_slot, &spec);

    // Builds all caches; benches will not contain shuffling/committee building times.
    builder.build_caches(&spec).unwrap();

    builder.build()
}

fn build_block(state: &mut BeaconState, keypairs: &[Keypair], spec: &ChainSpec) -> BeaconBlock {
    let mut builder = BeaconBlockBencher::new(spec);

    builder.set_slot(state.slot);

    let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap();
    let keypair = &keypairs[proposer_index];

    builder.set_randao_reveal(&keypair.sk, &state.fork, spec);

    // Used as a stream of validator indices for use in slashings, exits, etc.
    let mut validators_iter = (0..keypairs.len() as u64).into_iter();

    // Insert the maximum possible number of `ProposerSlashing` objects.
    for _ in 0..spec.max_proposer_slashings {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        builder.insert_proposer_slashing(
            validator_index,
            &keypairs[validator_index as usize].sk,
            &state.fork,
            spec,
        );
    }

    // Insert the maximum possible number of `AttesterSlashing` objects
    for _ in 0..spec.max_attester_slashings {
        let mut attesters: Vec<u64> = vec![];
        let mut secret_keys: Vec<&SecretKey> = vec![];

        for _ in 0..spec.max_indices_per_slashable_vote {
            let validator_index = validators_iter.next().expect("Insufficient validators.");

            attesters.push(validator_index);
            secret_keys.push(&keypairs[validator_index as usize].sk);
        }

        builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec);
    }

    // Insert the maximum possible number of `Attestation` objects.
    let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect();
    builder
        .fill_with_attestations(state, &all_secret_keys, spec)
        .unwrap();

    // Insert the maximum possible number of `Deposit` objects.
    for i in 0..spec.max_deposits {
        builder.insert_deposit(32_000_000_000, state.deposit_index + i, spec);
    }

    // Insert the maximum possible number of `Exit` objects.
    for _ in 0..spec.max_voluntary_exits {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        builder.insert_exit(
            state,
            validator_index,
            &keypairs[validator_index as usize].sk,
            spec,
        );
    }

    // Insert the maximum possible number of `Transfer` objects.
    for _ in 0..spec.max_transfers {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        // Manually set the validator to be withdrawn.
        state.validator_registry[validator_index as usize].withdrawable_epoch =
            state.previous_epoch(spec);

        builder.insert_transfer(
            state,
            validator_index,
            validator_index,
            1,
            keypairs[validator_index as usize].clone(),
            spec,
        );
    }

    builder.build(&keypair.sk, &state.fork, spec)
}

/// Run the detailed benchmarking suite on the given `BeaconState`.
///
/// `desc` will be added to the title of each bench.
fn bench_block_processing(
    c: &mut Criterion,
    initial_block: &BeaconBlock,
    initial_state: &BeaconState,
    initial_spec: &ChainSpec,
    desc: &str,
) {
    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("verify_block_signature", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(verify_block_signature(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_randao", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(process_randao(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_eth1_data", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(process_eth1_data(&mut state, &block.eth1_data).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_proposer_slashings", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_proposer_slashings(
                            &mut state,
                            &block.body.proposer_slashings,
                            &spec,
                        )
                        .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_attester_slashings", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_attester_slashings(
                            &mut state,
                            &block.body.attester_slashings,
                            &spec,
                        )
                        .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_attestations", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_attestations(&mut state, &block.body.attestations, &spec).unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_deposits", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(process_deposits(&mut state, &block.body.deposits, &spec).unwrap())
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_exits", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_exits(&mut state, &block.body.voluntary_exits, &spec).unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_transfers", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(process_transfers(&mut state, &block.body.transfers, &spec).unwrap())
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("per_block_processing", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(per_block_processing(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let mut state = initial_state.clone();
    state.drop_cache(RelativeEpoch::Previous);
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("build_previous_state_epoch_cache", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        state
                            .build_epoch_cache(RelativeEpoch::Previous, &spec)
                            .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let mut state = initial_state.clone();
    state.drop_cache(RelativeEpoch::Current);
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("build_current_state_epoch_cache", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        state
                            .build_epoch_cache(RelativeEpoch::Current, &spec)
                            .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );
}
<commit_msg>Ensure epoch processing benches get new eth1 data<commit_after>use benching_utils::{BeaconBlockBencher, BeaconStateBencher};
use criterion::Criterion;
use criterion::{black_box, Benchmark};
use state_processing::{
    per_block_processing,
    per_block_processing::{
        process_attestations, process_attester_slashings, process_deposits, process_eth1_data,
        process_exits, process_proposer_slashings, process_randao, process_transfers,
        verify_block_signature,
    },
};
use types::*;

/// Run the benchmarking suite on a foundation spec with 16,384 validators.
pub fn block_processing_16k_validators(c: &mut Criterion) {
    let spec = ChainSpec::foundation();

    let validator_count = 16_384;

    let (mut state, keypairs) = build_state(validator_count, &spec);
    let block = build_block(&mut state, &keypairs, &spec);

    assert_eq!(
        block.body.proposer_slashings.len(),
        spec.max_proposer_slashings as usize,
        "The block should have the maximum possible proposer slashings"
    );

    assert_eq!(
        block.body.attester_slashings.len(),
        spec.max_attester_slashings as usize,
        "The block should have the maximum possible attester slashings"
    );

    for attester_slashing in &block.body.attester_slashings {
        let len_1 = attester_slashing
            .slashable_attestation_1
            .validator_indices
            .len();
        let len_2 = attester_slashing
            .slashable_attestation_1
            .validator_indices
            .len();
        assert!(
            (len_1 == len_2) && (len_2 == spec.max_indices_per_slashable_vote as usize),
            "Each attester slashing should have the maximum possible validator indices"
        );
    }

    assert_eq!(
        block.body.attestations.len(),
        spec.max_attestations as usize,
        "The block should have the maximum possible attestations."
    );

    assert_eq!(
        block.body.deposits.len(),
        spec.max_deposits as usize,
        "The block should have the maximum possible deposits."
    );

    assert_eq!(
        block.body.voluntary_exits.len(),
        spec.max_voluntary_exits as usize,
        "The block should have the maximum possible voluntary exits."
    );

    assert_eq!(
        block.body.transfers.len(),
        spec.max_transfers as usize,
        "The block should have the maximum possible transfers."
    );

    bench_block_processing(
        c,
        &block,
        &state,
        &spec,
        &format!("{}_validators", validator_count),
    );
}

fn build_state(validator_count: usize, spec: &ChainSpec) -> (BeaconState, Vec<Keypair>) {
    let mut builder = BeaconStateBencher::new(validator_count, &spec);

    // Set the state to be just before an epoch transition.
    let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
    builder.teleport_to_slot(target_slot, &spec);

    // Builds all caches; benches will not contain shuffling/committee building times.
    builder.build_caches(&spec).unwrap();

    builder.build()
}

fn build_block(state: &mut BeaconState, keypairs: &[Keypair], spec: &ChainSpec) -> BeaconBlock {
    let mut builder = BeaconBlockBencher::new(spec);

    builder.set_slot(state.slot);

    let proposer_index = state.get_beacon_proposer_index(state.slot, spec).unwrap();
    let keypair = &keypairs[proposer_index];

    builder.set_randao_reveal(&keypair.sk, &state.fork, spec);

    // Used as a stream of validator indices for use in slashings, exits, etc.
    let mut validators_iter = (0..keypairs.len() as u64).into_iter();

    // Insert the maximum possible number of `ProposerSlashing` objects.
    for _ in 0..spec.max_proposer_slashings {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        builder.insert_proposer_slashing(
            validator_index,
            &keypairs[validator_index as usize].sk,
            &state.fork,
            spec,
        );
    }

    // Insert the maximum possible number of `AttesterSlashing` objects
    for _ in 0..spec.max_attester_slashings {
        let mut attesters: Vec<u64> = vec![];
        let mut secret_keys: Vec<&SecretKey> = vec![];

        for _ in 0..spec.max_indices_per_slashable_vote {
            let validator_index = validators_iter.next().expect("Insufficient validators.");

            attesters.push(validator_index);
            secret_keys.push(&keypairs[validator_index as usize].sk);
        }

        builder.insert_attester_slashing(&attesters, &secret_keys, &state.fork, spec);
    }

    // Insert the maximum possible number of `Attestation` objects.
    let all_secret_keys: Vec<&SecretKey> = keypairs.iter().map(|keypair| &keypair.sk).collect();
    builder
        .fill_with_attestations(state, &all_secret_keys, spec)
        .unwrap();

    // Insert the maximum possible number of `Deposit` objects.
    for i in 0..spec.max_deposits {
        builder.insert_deposit(32_000_000_000, state.deposit_index + i, spec);
    }

    // Insert the maximum possible number of `Exit` objects.
    for _ in 0..spec.max_voluntary_exits {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        builder.insert_exit(
            state,
            validator_index,
            &keypairs[validator_index as usize].sk,
            spec,
        );
    }

    // Insert the maximum possible number of `Transfer` objects.
    for _ in 0..spec.max_transfers {
        let validator_index = validators_iter.next().expect("Insufficient validators.");

        // Manually set the validator to be withdrawn.
        state.validator_registry[validator_index as usize].withdrawable_epoch =
            state.previous_epoch(spec);

        builder.insert_transfer(
            state,
            validator_index,
            validator_index,
            1,
            keypairs[validator_index as usize].clone(),
            spec,
        );
    }

    let mut block = builder.build(&keypair.sk, &state.fork, spec);

    // Set the eth1 data to be different from the state.
    block.eth1_data.block_hash = Hash256::from_slice(&vec![42; 32]);

    block
}

/// Run the detailed benchmarking suite on the given `BeaconState`.
///
/// `desc` will be added to the title of each bench.
fn bench_block_processing(
    c: &mut Criterion,
    initial_block: &BeaconBlock,
    initial_state: &BeaconState,
    initial_spec: &ChainSpec,
    desc: &str,
) {
    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("verify_block_signature", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(verify_block_signature(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_randao", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(process_randao(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_eth1_data", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(process_eth1_data(&mut state, &block.eth1_data).unwrap()),
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_proposer_slashings", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_proposer_slashings(
                            &mut state,
                            &block.body.proposer_slashings,
                            &spec,
                        )
                        .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_attester_slashings", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_attester_slashings(
                            &mut state,
                            &block.body.attester_slashings,
                            &spec,
                        )
                        .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_attestations", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_attestations(&mut state, &block.body.attestations, &spec).unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_deposits", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(process_deposits(&mut state, &block.body.deposits, &spec).unwrap())
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_exits", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        process_exits(&mut state, &block.body.voluntary_exits, &spec).unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("process_transfers", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(process_transfers(&mut state, &block.body.transfers, &spec).unwrap())
                },
            )
        })
        .sample_size(10),
    );

    let state = initial_state.clone();
    let block = initial_block.clone();
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("per_block_processing", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| black_box(per_block_processing(&mut state, &block, &spec).unwrap()),
            )
        })
        .sample_size(10),
    );

    let mut state = initial_state.clone();
    state.drop_cache(RelativeEpoch::Previous);
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("build_previous_state_epoch_cache", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        state
                            .build_epoch_cache(RelativeEpoch::Previous, &spec)
                            .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );

    let mut state = initial_state.clone();
    state.drop_cache(RelativeEpoch::Current);
    let spec = initial_spec.clone();
    c.bench(
        &format!("block_processing_{}", desc),
        Benchmark::new("build_current_state_epoch_cache", move |b| {
            b.iter_with_setup(
                || state.clone(),
                |mut state| {
                    black_box(
                        state
                            .build_epoch_cache(RelativeEpoch::Current, &spec)
                            .unwrap(),
                    )
                },
            )
        })
        .sample_size(10),
    );
}
<|endoftext|>


<commit_before>//! Owned and borrowed OS handles.

#![unstable(feature = "io_safety", issue = "87074")]

use super::raw::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle};
use crate::convert::TryFrom;
use crate::ffi::c_void;
use crate::fmt;
use crate::fs;
use crate::marker::PhantomData;
use crate::mem::forget;
use crate::ptr::NonNull;
use crate::sys::c;
use crate::sys_common::{AsInner, FromInner, IntoInner};

/// A borrowed handle.
///
/// This has a lifetime parameter to tie it to the lifetime of something that
/// owns the handle.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a handle is passed as an argument,
/// it is not captured or consumed, and it is never null.
///
/// Note that it *may* have the value `INVALID_HANDLE_VALUE`. See [here] for
/// the full story.
///
/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
#[derive(Copy, Clone)]
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct BorrowedHandle<'handle> {
    raw: NonNull<c_void>,
    _phantom: PhantomData<&'handle OwnedHandle>,
}

/// An owned handle.
///
/// This closes the handle on drop.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a handle is passed as a consumed
/// argument or returned as an owned value, and is never null.
///
/// Note that it *may* have the value `INVALID_HANDLE_VALUE`. See [here] for
/// the full story. For APIs like `CreateFileW` which report errors with
/// `INVALID_HANDLE_VALUE` instead of null, use [`OptionFileHandle`] instead
/// of `Option<OwnedHandle>`.
///
/// `OwnedHandle` uses [`CloseHandle`] to close its handle on drop. As such,
/// it must not be used with handles to open registry keys which need to be
/// closed with [`RegCloseKey`] instead.
///
/// [`CloseHandle`]: https://docs.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle
/// [`RegCloseKey`]: https://docs.microsoft.com/en-us/windows/win32/api/winreg/nf-winreg-regclosekey
///
/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct OwnedHandle {
    raw: NonNull<c_void>,
}

/// Similar to `Option<OwnedHandle>`, but intended for use in FFI interfaces
/// where `INVALID_HANDLE_VALUE` is used as the sentry value, and null values
/// are not used at all, such as in the return value of `CreateFileW`.
///
/// The main thing you can do with an `OptionFileHandle` is to convert it into
/// an `OwnedHandle` using its [`TryFrom`] implementation, and this conversion
/// takes care of the check for `INVALID_HANDLE_VALUE`.
///
/// If this holds an owned handle, it closes the handle on drop.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a non-null handle is passed as a
/// consumed argument or returned as an owned value, or it is
/// `INVALID_HANDLE_VALUE` indicating an error or an otherwise absent value.
///
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct OptionFileHandle {
    raw: RawHandle,
}

// The Windows [`HANDLE`] type may be transferred across and shared between
// thread boundaries (despite containing a `*mut void`, which in general isn't
// `Send` or `Sync`).
//
// [`HANDLE`]: std::os::windows::raw::HANDLE
unsafe impl Send for OwnedHandle {}
unsafe impl Send for OptionFileHandle {}
unsafe impl Send for BorrowedHandle<'_> {}
unsafe impl Sync for OwnedHandle {}
unsafe impl Sync for OptionFileHandle {}
unsafe impl Sync for BorrowedHandle<'_> {}

impl BorrowedHandle<'_> {
    /// Return a `BorrowedHandle` holding the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must remain open for the duration of
    /// the returned `BorrowedHandle`, and it must not be null.
    #[inline]
    #[unstable(feature = "io_safety", issue = "87074")]
    pub unsafe fn borrow_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw: NonNull::new_unchecked(raw), _phantom: PhantomData }
    }
}

impl OptionFileHandle {
    /// Return an empty `OptionFileHandle` with no resource.
    #[inline]
    #[unstable(feature = "io_safety", issue = "87074")]
    pub const fn none() -> Self {
        Self { raw: c::INVALID_HANDLE_VALUE }
    }
}

impl TryFrom<OptionFileHandle> for OwnedHandle {
    type Error = ();

    #[inline]
    fn try_from(option: OptionFileHandle) -> Result<Self, ()> {
        let raw = option.raw;
        forget(option);
        if let Some(non_null) = NonNull::new(raw) {
            if non_null.as_ptr() != c::INVALID_HANDLE_VALUE {
                Ok(Self { raw: non_null })
            } else {
                Err(())
            }
        } else {
            // In theory, we ought to be able to assume that the pointer here
            // is never null, change `option.raw` to `NonNull`, and obviate the
            // the panic path here. Unfortunately, Win32 documentation doesn't
            // explicitly guarantee this anywhere.
            //
            // APIs like [`CreateFileW`] itself have `HANDLE` arguments where a
            // null handle indicates an absent value, which wouldn't work if
            // null were a valid handle value, so it seems very unlikely that
            // it could ever return null. But who knows?
            //
            // [`CreateFileW`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
            panic!("An `OptionFileHandle` was null!");
        }
    }
}

impl From<OwnedHandle> for OptionFileHandle {
    #[inline]
    fn from(owned: OwnedHandle) -> Self {
        let raw = owned.raw;
        forget(owned);
        Self { raw: raw.as_ptr() }
    }
}

impl AsRawHandle for BorrowedHandle<'_> {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.raw.as_ptr()
    }
}

impl AsRawHandle for OwnedHandle {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.raw.as_ptr()
    }
}

impl IntoRawHandle for OwnedHandle {
    #[inline]
    fn into_raw_handle(self) -> RawHandle {
        let raw = self.raw.as_ptr();
        forget(self);
        raw
    }
}

impl FromRawHandle for OwnedHandle {
    /// Constructs a new instance of `Self` from the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must be open and suitable for
    /// assuming ownership. The resource must not require any cleanup other
    /// than `CloseHandle`.
    ///
    /// In particular, it must not be used with handles to open registry
    /// keys which need to be closed with [`RegCloseKey`] instead.
    ///
    /// [`RegCloseKey`]: https://docs.microsoft.com/en-us/windows/win32/api/winreg/nf-winreg-regclosekey
    #[inline]
    unsafe fn from_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw: NonNull::new_unchecked(raw) }
    }
}

impl FromRawHandle for OptionFileHandle {
    /// Constructs a new instance of `Self` from the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must be either open and otherwise
    /// unowned, or equal to `INVALID_HANDLE_VALUE``. Note that not all Windows
    /// APIs use `INVALID_HANDLE_VALUE` for errors; see [here] for the full
    /// story.
    ///
    /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
    #[inline]
    unsafe fn from_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw }
    }
}

impl Drop for OwnedHandle {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let _ = c::CloseHandle(self.raw.as_ptr());
        }
    }
}

impl Drop for OptionFileHandle {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let _ = c::CloseHandle(self.raw);
        }
    }
}

impl fmt::Debug for BorrowedHandle<'_> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("BorrowedHandle").field("handle", &self.raw).finish()
    }
}

impl fmt::Debug for OwnedHandle {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("OwnedHandle").field("handle", &self.raw).finish()
    }
}

impl fmt::Debug for OptionFileHandle {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("OptionFileHandle").field("handle", &self.raw).finish()
    }
}

/// A trait to borrow the handle from an underlying object.
#[unstable(feature = "io_safety", issue = "87074")]
pub trait AsHandle {
    /// Borrows the handle.
    ///
    /// # Example
    ///
    /// ```rust,no_run
    /// use std::fs::File;
    /// # use std::io;
    /// use std::os::windows::{AsHandle, BorrowedHandle};
    ///
    /// let mut f = File::open("foo.txt")?;
    /// let borrowed_handle: BorrowedHandle<'_> = f.as_handle();
    /// # Ok::<(), io::Error>(())
    /// ```
    fn as_handle(&self) -> BorrowedHandle<'_>;
}

impl AsHandle for BorrowedHandle<'_> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        *self
    }
}

impl AsHandle for OwnedHandle {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for fs::File {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        self.as_inner().as_handle()
    }
}

impl From<fs::File> for OwnedHandle {
    #[inline]
    fn from(file: fs::File) -> OwnedHandle {
        file.into_inner().into_inner().into_inner().into()
    }
}

impl From<OwnedHandle> for fs::File {
    #[inline]
    fn from(owned: OwnedHandle) -> Self {
        Self::from_inner(FromInner::from_inner(FromInner::from_inner(owned)))
    }
}

impl AsHandle for crate::io::Stdin {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StdinLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::io::Stdout {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StdoutLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::io::Stderr {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StderrLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStdin {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStdin> for OwnedHandle {
    #[inline]
    fn from(child_stdin: crate::process::ChildStdin) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stdin.into_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStdout {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStdout> for OwnedHandle {
    #[inline]
    fn from(child_stdout: crate::process::ChildStdout) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stdout.into_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStderr {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStderr> for OwnedHandle {
    #[inline]
    fn from(child_stderr: crate::process::ChildStderr) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stderr.into_raw_handle()) }
    }
}

impl<T> AsHandle for crate::thread::JoinHandle<T> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<T> From<crate::thread::JoinHandle<T>> for OwnedHandle {
    #[inline]
    fn from(join_handle: crate::thread::JoinHandle<T>) -> OwnedHandle {
        join_handle.into_inner().into_handle().into_inner()
    }
}
<commit_msg>Delete a spurious empty comment line.<commit_after>//! Owned and borrowed OS handles.

#![unstable(feature = "io_safety", issue = "87074")]

use super::raw::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle};
use crate::convert::TryFrom;
use crate::ffi::c_void;
use crate::fmt;
use crate::fs;
use crate::marker::PhantomData;
use crate::mem::forget;
use crate::ptr::NonNull;
use crate::sys::c;
use crate::sys_common::{AsInner, FromInner, IntoInner};

/// A borrowed handle.
///
/// This has a lifetime parameter to tie it to the lifetime of something that
/// owns the handle.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a handle is passed as an argument,
/// it is not captured or consumed, and it is never null.
///
/// Note that it *may* have the value `INVALID_HANDLE_VALUE`. See [here] for
/// the full story.
///
/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
#[derive(Copy, Clone)]
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct BorrowedHandle<'handle> {
    raw: NonNull<c_void>,
    _phantom: PhantomData<&'handle OwnedHandle>,
}

/// An owned handle.
///
/// This closes the handle on drop.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a handle is passed as a consumed
/// argument or returned as an owned value, and is never null.
///
/// Note that it *may* have the value `INVALID_HANDLE_VALUE`. See [here] for
/// the full story. For APIs like `CreateFileW` which report errors with
/// `INVALID_HANDLE_VALUE` instead of null, use [`OptionFileHandle`] instead
/// of `Option<OwnedHandle>`.
///
/// `OwnedHandle` uses [`CloseHandle`] to close its handle on drop. As such,
/// it must not be used with handles to open registry keys which need to be
/// closed with [`RegCloseKey`] instead.
///
/// [`CloseHandle`]: https://docs.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle
/// [`RegCloseKey`]: https://docs.microsoft.com/en-us/windows/win32/api/winreg/nf-winreg-regclosekey
///
/// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct OwnedHandle {
    raw: NonNull<c_void>,
}

/// Similar to `Option<OwnedHandle>`, but intended for use in FFI interfaces
/// where `INVALID_HANDLE_VALUE` is used as the sentry value, and null values
/// are not used at all, such as in the return value of `CreateFileW`.
///
/// The main thing you can do with an `OptionFileHandle` is to convert it into
/// an `OwnedHandle` using its [`TryFrom`] implementation, and this conversion
/// takes care of the check for `INVALID_HANDLE_VALUE`.
///
/// If this holds an owned handle, it closes the handle on drop.
///
/// This uses `repr(transparent)` and has the representation of a host handle,
/// so it can be used in FFI in places where a non-null handle is passed as a
/// consumed argument or returned as an owned value, or it is
/// `INVALID_HANDLE_VALUE` indicating an error or an otherwise absent value.
#[repr(transparent)]
#[unstable(feature = "io_safety", issue = "87074")]
pub struct OptionFileHandle {
    raw: RawHandle,
}

// The Windows [`HANDLE`] type may be transferred across and shared between
// thread boundaries (despite containing a `*mut void`, which in general isn't
// `Send` or `Sync`).
//
// [`HANDLE`]: std::os::windows::raw::HANDLE
unsafe impl Send for OwnedHandle {}
unsafe impl Send for OptionFileHandle {}
unsafe impl Send for BorrowedHandle<'_> {}
unsafe impl Sync for OwnedHandle {}
unsafe impl Sync for OptionFileHandle {}
unsafe impl Sync for BorrowedHandle<'_> {}

impl BorrowedHandle<'_> {
    /// Return a `BorrowedHandle` holding the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must remain open for the duration of
    /// the returned `BorrowedHandle`, and it must not be null.
    #[inline]
    #[unstable(feature = "io_safety", issue = "87074")]
    pub unsafe fn borrow_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw: NonNull::new_unchecked(raw), _phantom: PhantomData }
    }
}

impl OptionFileHandle {
    /// Return an empty `OptionFileHandle` with no resource.
    #[inline]
    #[unstable(feature = "io_safety", issue = "87074")]
    pub const fn none() -> Self {
        Self { raw: c::INVALID_HANDLE_VALUE }
    }
}

impl TryFrom<OptionFileHandle> for OwnedHandle {
    type Error = ();

    #[inline]
    fn try_from(option: OptionFileHandle) -> Result<Self, ()> {
        let raw = option.raw;
        forget(option);
        if let Some(non_null) = NonNull::new(raw) {
            if non_null.as_ptr() != c::INVALID_HANDLE_VALUE {
                Ok(Self { raw: non_null })
            } else {
                Err(())
            }
        } else {
            // In theory, we ought to be able to assume that the pointer here
            // is never null, change `option.raw` to `NonNull`, and obviate the
            // the panic path here. Unfortunately, Win32 documentation doesn't
            // explicitly guarantee this anywhere.
            //
            // APIs like [`CreateFileW`] itself have `HANDLE` arguments where a
            // null handle indicates an absent value, which wouldn't work if
            // null were a valid handle value, so it seems very unlikely that
            // it could ever return null. But who knows?
            //
            // [`CreateFileW`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
            panic!("An `OptionFileHandle` was null!");
        }
    }
}

impl From<OwnedHandle> for OptionFileHandle {
    #[inline]
    fn from(owned: OwnedHandle) -> Self {
        let raw = owned.raw;
        forget(owned);
        Self { raw: raw.as_ptr() }
    }
}

impl AsRawHandle for BorrowedHandle<'_> {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.raw.as_ptr()
    }
}

impl AsRawHandle for OwnedHandle {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.raw.as_ptr()
    }
}

impl IntoRawHandle for OwnedHandle {
    #[inline]
    fn into_raw_handle(self) -> RawHandle {
        let raw = self.raw.as_ptr();
        forget(self);
        raw
    }
}

impl FromRawHandle for OwnedHandle {
    /// Constructs a new instance of `Self` from the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must be open and suitable for
    /// assuming ownership. The resource must not require any cleanup other
    /// than `CloseHandle`.
    ///
    /// In particular, it must not be used with handles to open registry
    /// keys which need to be closed with [`RegCloseKey`] instead.
    ///
    /// [`RegCloseKey`]: https://docs.microsoft.com/en-us/windows/win32/api/winreg/nf-winreg-regclosekey
    #[inline]
    unsafe fn from_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw: NonNull::new_unchecked(raw) }
    }
}

impl FromRawHandle for OptionFileHandle {
    /// Constructs a new instance of `Self` from the given raw handle.
    ///
    /// # Safety
    ///
    /// The resource pointed to by `raw` must be either open and otherwise
    /// unowned, or equal to `INVALID_HANDLE_VALUE``. Note that not all Windows
    /// APIs use `INVALID_HANDLE_VALUE` for errors; see [here] for the full
    /// story.
    ///
    /// [here]: https://devblogs.microsoft.com/oldnewthing/20040302-00/?p=40443
    #[inline]
    unsafe fn from_raw_handle(raw: RawHandle) -> Self {
        assert!(!raw.is_null());
        Self { raw }
    }
}

impl Drop for OwnedHandle {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let _ = c::CloseHandle(self.raw.as_ptr());
        }
    }
}

impl Drop for OptionFileHandle {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let _ = c::CloseHandle(self.raw);
        }
    }
}

impl fmt::Debug for BorrowedHandle<'_> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("BorrowedHandle").field("handle", &self.raw).finish()
    }
}

impl fmt::Debug for OwnedHandle {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("OwnedHandle").field("handle", &self.raw).finish()
    }
}

impl fmt::Debug for OptionFileHandle {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("OptionFileHandle").field("handle", &self.raw).finish()
    }
}

/// A trait to borrow the handle from an underlying object.
#[unstable(feature = "io_safety", issue = "87074")]
pub trait AsHandle {
    /// Borrows the handle.
    ///
    /// # Example
    ///
    /// ```rust,no_run
    /// use std::fs::File;
    /// # use std::io;
    /// use std::os::windows::{AsHandle, BorrowedHandle};
    ///
    /// let mut f = File::open("foo.txt")?;
    /// let borrowed_handle: BorrowedHandle<'_> = f.as_handle();
    /// # Ok::<(), io::Error>(())
    /// ```
    fn as_handle(&self) -> BorrowedHandle<'_>;
}

impl AsHandle for BorrowedHandle<'_> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        *self
    }
}

impl AsHandle for OwnedHandle {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for fs::File {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        self.as_inner().as_handle()
    }
}

impl From<fs::File> for OwnedHandle {
    #[inline]
    fn from(file: fs::File) -> OwnedHandle {
        file.into_inner().into_inner().into_inner().into()
    }
}

impl From<OwnedHandle> for fs::File {
    #[inline]
    fn from(owned: OwnedHandle) -> Self {
        Self::from_inner(FromInner::from_inner(FromInner::from_inner(owned)))
    }
}

impl AsHandle for crate::io::Stdin {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StdinLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::io::Stdout {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StdoutLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::io::Stderr {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<'a> AsHandle for crate::io::StderrLock<'a> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStdin {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStdin> for OwnedHandle {
    #[inline]
    fn from(child_stdin: crate::process::ChildStdin) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stdin.into_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStdout {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStdout> for OwnedHandle {
    #[inline]
    fn from(child_stdout: crate::process::ChildStdout) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stdout.into_raw_handle()) }
    }
}

impl AsHandle for crate::process::ChildStderr {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl From<crate::process::ChildStderr> for OwnedHandle {
    #[inline]
    fn from(child_stderr: crate::process::ChildStderr) -> OwnedHandle {
        unsafe { OwnedHandle::from_raw_handle(child_stderr.into_raw_handle()) }
    }
}

impl<T> AsHandle for crate::thread::JoinHandle<T> {
    #[inline]
    fn as_handle(&self) -> BorrowedHandle<'_> {
        unsafe { BorrowedHandle::borrow_raw_handle(self.as_raw_handle()) }
    }
}

impl<T> From<crate::thread::JoinHandle<T>> for OwnedHandle {
    #[inline]
    fn from(join_handle: crate::thread::JoinHandle<T>) -> OwnedHandle {
        join_handle.into_inner().into_handle().into_inner()
    }
}
<|endoftext|>