<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use self::infer::InferenceContext;
use rustc::ty::TypeFoldable;
use rustc::ty::subst::{Kind, Substs};
use rustc::ty::{Ty, TyCtxt, ClosureSubsts, RegionVid, RegionKind};
use rustc::mir::{Mir, Location, Rvalue, BasicBlock, Statement, StatementKind};
use rustc::mir::visit::{MutVisitor, Lookup};
use rustc::mir::transform::{MirPass, MirSource};
use rustc::infer::{self as rustc_infer, InferCtxt};
use rustc::util::nodemap::FxHashSet;
use rustc_data_structures::indexed_vec::{IndexVec, Idx};
use syntax_pos::DUMMY_SP;
use std::collections::HashMap;
use std::fmt;

use util as mir_util;
use self::mir_util::PassWhere;

mod infer;

#[allow(dead_code)]
struct NLLVisitor<'a, 'gcx: 'a + 'tcx, 'tcx: 'a> {
    lookup_map: HashMap<RegionVid, Lookup>,
    regions: IndexVec<RegionIndex, Region>,
    infcx: InferCtxt<'a, 'gcx, 'tcx>,
}

impl<'a, 'gcx, 'tcx> NLLVisitor<'a, 'gcx, 'tcx> {
    pub fn new(infcx: InferCtxt<'a, 'gcx, 'tcx>) -> Self {
        NLLVisitor {
            infcx,
            lookup_map: HashMap::new(),
            regions: IndexVec::new(),
        }
    }

    pub fn into_results(self) -> (HashMap<RegionVid, Lookup>, IndexVec<RegionIndex, Region>) {
        (self.lookup_map, self.regions)
    }

    fn renumber_regions<T>(&mut self, value: &T) -> T where T: TypeFoldable<'tcx> {
        self.infcx.tcx.fold_regions(value, &mut false, |_region, _depth| {
            self.regions.push(Region::default());
            self.infcx.next_region_var(rustc_infer::MiscVariable(DUMMY_SP))
        })
    }

    fn store_region(&mut self, region: &RegionKind, lookup: Lookup) {
        if let RegionKind::ReVar(rid) = *region {
            self.lookup_map.entry(rid).or_insert(lookup);
        }
    }

    fn store_ty_regions(&mut self, ty: &Ty<'tcx>, lookup: Lookup) {
        for region in ty.regions() {
            self.store_region(region, lookup);
        }
    }

    fn store_kind_regions(&mut self, kind: &'tcx Kind, lookup: Lookup) {
        if let Some(ty) = kind.as_type() {
            self.store_ty_regions(&ty, lookup);
        } else if let Some(region) = kind.as_region() {
            self.store_region(region, lookup);
        }
    }
}

impl<'a, 'gcx, 'tcx> MutVisitor<'tcx> for NLLVisitor<'a, 'gcx, 'tcx> {
    fn visit_ty(&mut self, ty: &mut Ty<'tcx>, lookup: Lookup) {
        let old_ty = *ty;
        *ty = self.renumber_regions(&old_ty);
        self.store_ty_regions(ty, lookup);
    }

    fn visit_substs(&mut self, substs: &mut &'tcx Substs<'tcx>, location: Location) {
        *substs = self.renumber_regions(&{*substs});
        let lookup = Lookup::Loc(location);
        for kind in *substs {
            self.store_kind_regions(kind, lookup);
        }
    }

    fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, location: Location) {
        match *rvalue {
            Rvalue::Ref(ref mut r, _, _) => {
                let old_r = *r;
                *r = self.renumber_regions(&old_r);
                let lookup = Lookup::Loc(location);
                self.store_region(r, lookup);
            }
            Rvalue::Use(..) |
            Rvalue::Repeat(..) |
            Rvalue::Len(..) |
            Rvalue::Cast(..) |
            Rvalue::BinaryOp(..) |
            Rvalue::CheckedBinaryOp(..) |
            Rvalue::UnaryOp(..) |
            Rvalue::Discriminant(..) |
            Rvalue::NullaryOp(..) |
            Rvalue::Aggregate(..) => {
                // These variants don't contain regions.
            }
        }
        self.super_rvalue(rvalue, location);
    }

    fn visit_closure_substs(&mut self,
                            substs: &mut ClosureSubsts<'tcx>,
                            location: Location) {
        *substs = self.renumber_regions(substs);
        let lookup = Lookup::Loc(location);
        for kind in substs.substs {
            self.store_kind_regions(kind, lookup);
        }
    }

    fn visit_statement(&mut self,
                       block: BasicBlock,
                       statement: &mut Statement<'tcx>,
                       location: Location) {
        if let StatementKind::EndRegion(_) = statement.kind {
            statement.kind = StatementKind::Nop;
        }
        self.super_statement(block, statement, location);
    }
}

// MIR Pass for non-lexical lifetimes
pub struct NLL;

impl MirPass for NLL {
    fn run_pass<'a, 'tcx>(&self,
                          tcx: TyCtxt<'a, 'tcx, 'tcx>,
                          source: MirSource,
                          mir: &mut Mir<'tcx>) {
        if !tcx.sess.opts.debugging_opts.nll {
            return;
        }

        tcx.infer_ctxt().enter(|infcx| {
            // Clone mir so we can mutate it without disturbing the rest of the compiler
            let mut renumbered_mir = mir.clone();
            let mut visitor = NLLVisitor::new(infcx);
            visitor.visit_mir(&mut renumbered_mir);
            mir_util::dump_mir(tcx, None, "nll", &0, source, mir, |pass_where, out| {
                if let PassWhere::BeforeCFG = pass_where {
                    for (index, value) in visitor.regions.iter_enumerated() {
                        writeln!(out, "// R{:03}: {:?}", index.0, value)?;
                    }
                }
                Ok(())
            });
            let (_lookup_map, regions) = visitor.into_results();
            let inference_context = InferenceContext::new(regions);
            inference_context.solve(infcx, &renumbered_mir);
        })
    }
}

#[derive(Clone, Default, PartialEq, Eq)]
struct Region {
    points: FxHashSet<Location>,
}

impl fmt::Debug for Region {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(formatter, "{:?}", self.points)
    }
}

impl Region {
    pub fn add_point(&mut self, point: Location) -> bool {
        self.points.insert(point)
    }

    pub fn may_contain(&self, point: Location) -> bool {
        self.points.contains(&point)
    }
}

newtype_index!(RegionIndex);
<commit_msg>Mark Region as public<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use self::infer::InferenceContext;
use rustc::ty::TypeFoldable;
use rustc::ty::subst::{Kind, Substs};
use rustc::ty::{Ty, TyCtxt, ClosureSubsts, RegionVid, RegionKind};
use rustc::mir::{Mir, Location, Rvalue, BasicBlock, Statement, StatementKind};
use rustc::mir::visit::{MutVisitor, Lookup};
use rustc::mir::transform::{MirPass, MirSource};
use rustc::infer::{self as rustc_infer, InferCtxt};
use rustc::util::nodemap::FxHashSet;
use rustc_data_structures::indexed_vec::{IndexVec, Idx};
use syntax_pos::DUMMY_SP;
use std::collections::HashMap;
use std::fmt;

use util as mir_util;
use self::mir_util::PassWhere;

mod infer;

#[allow(dead_code)]
struct NLLVisitor<'a, 'gcx: 'a + 'tcx, 'tcx: 'a> {
    lookup_map: HashMap<RegionVid, Lookup>,
    regions: IndexVec<RegionIndex, Region>,
    infcx: InferCtxt<'a, 'gcx, 'tcx>,
}

impl<'a, 'gcx, 'tcx> NLLVisitor<'a, 'gcx, 'tcx> {
    pub fn new(infcx: InferCtxt<'a, 'gcx, 'tcx>) -> Self {
        NLLVisitor {
            infcx,
            lookup_map: HashMap::new(),
            regions: IndexVec::new(),
        }
    }

    pub fn into_results(self) -> (HashMap<RegionVid, Lookup>, IndexVec<RegionIndex, Region>) {
        (self.lookup_map, self.regions)
    }

    fn renumber_regions<T>(&mut self, value: &T) -> T where T: TypeFoldable<'tcx> {
        self.infcx.tcx.fold_regions(value, &mut false, |_region, _depth| {
            self.regions.push(Region::default());
            self.infcx.next_region_var(rustc_infer::MiscVariable(DUMMY_SP))
        })
    }

    fn store_region(&mut self, region: &RegionKind, lookup: Lookup) {
        if let RegionKind::ReVar(rid) = *region {
            self.lookup_map.entry(rid).or_insert(lookup);
        }
    }

    fn store_ty_regions(&mut self, ty: &Ty<'tcx>, lookup: Lookup) {
        for region in ty.regions() {
            self.store_region(region, lookup);
        }
    }

    fn store_kind_regions(&mut self, kind: &'tcx Kind, lookup: Lookup) {
        if let Some(ty) = kind.as_type() {
            self.store_ty_regions(&ty, lookup);
        } else if let Some(region) = kind.as_region() {
            self.store_region(region, lookup);
        }
    }
}

impl<'a, 'gcx, 'tcx> MutVisitor<'tcx> for NLLVisitor<'a, 'gcx, 'tcx> {
    fn visit_ty(&mut self, ty: &mut Ty<'tcx>, lookup: Lookup) {
        let old_ty = *ty;
        *ty = self.renumber_regions(&old_ty);
        self.store_ty_regions(ty, lookup);
    }

    fn visit_substs(&mut self, substs: &mut &'tcx Substs<'tcx>, location: Location) {
        *substs = self.renumber_regions(&{*substs});
        let lookup = Lookup::Loc(location);
        for kind in *substs {
            self.store_kind_regions(kind, lookup);
        }
    }

    fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, location: Location) {
        match *rvalue {
            Rvalue::Ref(ref mut r, _, _) => {
                let old_r = *r;
                *r = self.renumber_regions(&old_r);
                let lookup = Lookup::Loc(location);
                self.store_region(r, lookup);
            }
            Rvalue::Use(..) |
            Rvalue::Repeat(..) |
            Rvalue::Len(..) |
            Rvalue::Cast(..) |
            Rvalue::BinaryOp(..) |
            Rvalue::CheckedBinaryOp(..) |
            Rvalue::UnaryOp(..) |
            Rvalue::Discriminant(..) |
            Rvalue::NullaryOp(..) |
            Rvalue::Aggregate(..) => {
                // These variants don't contain regions.
            }
        }
        self.super_rvalue(rvalue, location);
    }

    fn visit_closure_substs(&mut self,
                            substs: &mut ClosureSubsts<'tcx>,
                            location: Location) {
        *substs = self.renumber_regions(substs);
        let lookup = Lookup::Loc(location);
        for kind in substs.substs {
            self.store_kind_regions(kind, lookup);
        }
    }

    fn visit_statement(&mut self,
                       block: BasicBlock,
                       statement: &mut Statement<'tcx>,
                       location: Location) {
        if let StatementKind::EndRegion(_) = statement.kind {
            statement.kind = StatementKind::Nop;
        }
        self.super_statement(block, statement, location);
    }
}

// MIR Pass for non-lexical lifetimes
pub struct NLL;

impl MirPass for NLL {
    fn run_pass<'a, 'tcx>(&self,
                          tcx: TyCtxt<'a, 'tcx, 'tcx>,
                          source: MirSource,
                          mir: &mut Mir<'tcx>) {
        if !tcx.sess.opts.debugging_opts.nll {
            return;
        }

        tcx.infer_ctxt().enter(|infcx| {
            // Clone mir so we can mutate it without disturbing the rest of the compiler
            let mut renumbered_mir = mir.clone();
            let mut visitor = NLLVisitor::new(infcx);
            visitor.visit_mir(&mut renumbered_mir);
            mir_util::dump_mir(tcx, None, "nll", &0, source, mir, |pass_where, out| {
                if let PassWhere::BeforeCFG = pass_where {
                    for (index, value) in visitor.regions.iter_enumerated() {
                        writeln!(out, "// R{:03}: {:?}", index.0, value)?;
                    }
                }
                Ok(())
            });
            let (_lookup_map, regions) = visitor.into_results();
            let inference_context = InferenceContext::new(regions);
            inference_context.solve(infcx, &renumbered_mir);
        })
    }
}

#[derive(Clone, Default, PartialEq, Eq)]
pub struct Region {
    points: FxHashSet<Location>,
}

impl fmt::Debug for Region {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(formatter, "{:?}", self.points)
    }
}

impl Region {
    pub fn add_point(&mut self, point: Location) -> bool {
        self.points.insert(point)
    }

    pub fn may_contain(&self, point: Location) -> bool {
        self.points.contains(&point)
    }
}

newtype_index!(RegionIndex);
<|endoftext|>


<commit_before>// Claxon -- A FLAC decoding library in Rust
// Copyright 2014 Ruud van Asseldonk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// A copy of the License has been included in the root of the repository.

extern crate claxon;
extern crate hound;
extern crate walkdir;

use std::fs;
use std::io;
use std::path::Path;

fn run_metaflac_streaminfo<P: AsRef<Path>>(fname: P) -> String {
    use std::process::Command;

    // Run metaflac on the specified file and print all streaminfo data.
    let output = Command::new("metaflac")
        .arg("--show-min-blocksize")
        .arg("--show-max-blocksize")
        .arg("--show-min-framesize")
        .arg("--show-max-framesize")
        .arg("--show-sample-rate")
        .arg("--show-channels")
        .arg("--show-bps")
        .arg("--show-total-samples")
        .arg("--show-md5sum")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run metaflac");
    String::from_utf8(output.stdout).expect("metaflac wrote invalid UTF-8")
}

fn run_metaflac_vorbis_comment<P: AsRef<Path>>(fname: P) -> String {
    use std::process::Command;

    // Run metaflac on the specified file and print all Vorbis comment data.
    let output = Command::new("metaflac")
        .arg("--block-type=VORBIS_COMMENT")
        .arg("--list")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run metaflac");
    String::from_utf8(output.stdout).expect("metaflac wrote invalid UTF-8")
}

fn decode_file<P: AsRef<Path>>(fname: P) -> Vec<u8> {
    use std::process::Command;

    // Run the the reference flac decoder on the file.
    let output = Command::new("flac")
        .arg("--decode")
        .arg("--silent")
        .arg("--stdout")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run flac");

    assert!(output.status.success());
    output.stdout
}

fn print_hex(seq: &[u8]) -> String {
    let vec: Vec<String> = seq.iter().map(|x| format!("{:0>2x}", *x)).collect();
    vec.concat()
}

fn read_streaminfo<P: AsRef<Path>>(fname: P) -> String {
    // Use a buffered reader, this speeds up the test by 120%.
    let file = fs::File::open(fname).unwrap();
    let reader = io::BufReader::new(file);
    let stream = claxon::FlacReader::new(reader).unwrap();
    let streaminfo = stream.streaminfo();

    // Format the streaminfo in the same way that metaflac prints it.
    format!("{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n",
            streaminfo.min_block_size,
            streaminfo.max_block_size,
            streaminfo.min_frame_size.unwrap_or(0),
            streaminfo.max_frame_size.unwrap_or(0),
            streaminfo.sample_rate,
            streaminfo.channels,
            streaminfo.bits_per_sample,
            streaminfo.samples.unwrap_or(0),
            print_hex(&streaminfo.md5sum)) // TODO implement LowerHex for &[u8] and submit a PR.
}

fn compare_metaflac_streaminfo<P: AsRef<Path>>(fname: P) {
    let metaflac = run_metaflac_streaminfo(&fname);
    let streaminfo = read_streaminfo(&fname);
    let mut mf_lines = metaflac.lines();
    let mut si_lines = streaminfo.lines();
    while let (Some(mf), Some(si)) = (mf_lines.next(), si_lines.next()) {
        if mf != si {
            println!("metaflac\n--------\n{}", metaflac);
            println!("streaminfo\n----------\n{}", streaminfo);
            panic!("metaflac disagrees on parsed streaminfo");
        }
    }
}

fn compare_metaflac_vorbis_comment<P: AsRef<Path>>(fname: P) {
    let metaflac = run_metaflac_vorbis_comment(&fname);

    // Use a buffered reader, this speeds up the test by 120%.
    let file = fs::File::open(fname).unwrap();
    let buf_reader = io::BufReader::new(file);
    let reader = claxon::FlacReader::new(buf_reader).unwrap();

    let mut mf_lines = metaflac.lines();

    // Search for the vendor string in the metaflac output.
    while let Some(line) = mf_lines.next() {
        let prefix = "  vendor string: ";
        if line.starts_with(prefix) {
            let mf_vendor_string = &line[prefix.len()..];

            // If the vendor string starts with a null byte, metaflac will not
            // print it -- my guess is because metaflac is written in C and uses
            // C-style string manipulation. In that case we skip it.
            match reader.vendor() {
                Some(x) if x.starts_with('\0') => break,
                _ => {}
            }

            assert_eq!(reader.vendor(), Some(mf_vendor_string));
            break
        }
    }

    let mut tags = reader.tags().iter();

    // Loop through all of the comments.
    while let Some(line) = mf_lines.next() {
        let prefix = "    comment[";
        if line.starts_with(prefix) {
            let mf_line = &line[prefix.len()..];
            let prefix_sep_index = mf_line.find(' ').unwrap();
            let mf_pair = &mf_line[prefix_sep_index + 1..];

            let sep_index = mf_pair.find('=').unwrap();
            let mf_name = &mf_pair[..sep_index];
            let mf_value = &mf_pair[sep_index + 1..];

            let &(ref name, ref value_lines) = tags.next().unwrap();
            let mut value_lines_iter = value_lines.lines();
            let value = value_lines_iter.next().unwrap_or("");

            assert_eq!(name, mf_name);
            assert_eq!(value, mf_value);

            // If there are newlines, then we also need to read those as
            // separate lines from the metaflac untput. This does assume that
            // the newline count that Claxon read is correct, and because of the
            // behavior of the `.lines()` iterator this does not accurately
            // verify carriage returns, but we could not anyway, because
            // metaflac does not escape them.
            while let Some(actual_line) = value_lines_iter.next() {
                assert_eq!(actual_line, mf_lines.next().unwrap());
            }
        }
    }
}

fn compare_decoded_stream<P: AsRef<Path>>(fname: P) {
    let wav = decode_file(&fname);
    let cursor = io::Cursor::new(wav);

    // Compare the reference decoded by the 'flac' program (stored as wav, which
    // we read with Hound) to how Claxon decodes it, sample by sample.
    let mut ref_wav_reader = hound::WavReader::new(cursor).unwrap();

    let try_file = fs::File::open(fname).unwrap();
    let try_buf_reader = io::BufReader::new(try_file);
    let mut try_flac_reader = claxon::FlacReader::new(try_buf_reader).unwrap();

    // The streaminfo test will ensure that things like bit depth and
    // sample rate match, only the actual samples are compared here.
    let mut ref_samples = ref_wav_reader.samples::<i32>();

    let samples = try_flac_reader.streaminfo().samples.unwrap();
    let n_channels = try_flac_reader.streaminfo().channels;
    let mut blocks = try_flac_reader.blocks();
    let mut sample = 0u64;
    let mut b = 0u64;
    let mut buffer = Vec::new();

    while sample < samples {
        let block = blocks.read_next_or_eof(buffer).unwrap().unwrap();
        {
            let mut channels: Vec<_> = (0..n_channels)
                                           .map(|i| block.channel(i).iter().cloned())
                                           .collect();
            for i in 0..block.duration() {
                for ch in 0..n_channels as usize {
                    let ref_sample = ref_samples.next().map(|r| r.ok().unwrap());
                    let try_sample = channels[ch].next();
                    if ref_sample != try_sample {
                        println!("disagreement at sample {} of block {} in channel {}: \
                                  reference is {} but decoded is {}",
                                 i,
                                 b,
                                 ch,
                                 ref_sample.unwrap(),
                                 try_sample.unwrap());
                        panic!("decoding differs from reference decoder");
                    }
                }
            }
            sample = sample + block.duration() as u64;
        }
        b = b + 1;
        buffer = block.into_buffer();
    }
}

// Hard-coded tests for the test samples added py populate.sh.
// This allows us to run these tests in parallel and follow progress without
// enabling --nocapture.

#[test]
fn verify_streaminfo_p0() {
    compare_metaflac_streaminfo("testsamples/p0.flac");
}

#[test]
fn verify_streaminfo_p1() {
    compare_metaflac_streaminfo("testsamples/p1.flac");
}

#[test]
fn verify_streaminfo_p2() {
    compare_metaflac_streaminfo("testsamples/p2.flac");
}

#[test]
fn verify_streaminfo_p3() {
    compare_metaflac_streaminfo("testsamples/p3.flac");
}

#[test]
fn verify_streaminfo_p4() {
    compare_metaflac_streaminfo("testsamples/p4.flac");
}

#[test]
fn verify_streaminfo_pop() {
    compare_metaflac_streaminfo("testsamples/pop.flac");
}

#[test]
fn verify_streaminfo_short() {
    compare_metaflac_streaminfo("testsamples/short.flac");
}

#[test]
fn verify_streaminfo_wasted_bits() {
    compare_metaflac_streaminfo("testsamples/wasted_bits.flac");
}

#[test]
fn verify_vorbis_comment_p0() {
    compare_metaflac_vorbis_comment("testsamples/p0.flac");
}

#[test]
fn verify_vorbis_comment_p1() {
    compare_metaflac_vorbis_comment("testsamples/p1.flac");
}

#[test]
fn verify_vorbis_comment_p2() {
    compare_metaflac_vorbis_comment("testsamples/p2.flac");
}

#[test]
fn verify_vorbis_comment_p3() {
    compare_metaflac_vorbis_comment("testsamples/p3.flac");
}

#[test]
fn verify_vorbis_comment_p4() {
    compare_metaflac_vorbis_comment("testsamples/p4.flac");
}

#[test]
fn verify_decoded_stream_p0() {
    compare_decoded_stream("testsamples/p0.flac");
}

#[test]
fn verify_decoded_stream_p1() {
    compare_decoded_stream("testsamples/p1.flac");
}

#[test]
fn verify_decoded_stream_p2() {
    compare_decoded_stream("testsamples/p2.flac");
}

#[test]
fn verify_decoded_stream_p3() {
    compare_decoded_stream("testsamples/p3.flac");
}

#[test]
fn verify_decoded_stream_p4() {
    compare_decoded_stream("testsamples/p4.flac");
}

#[test]
fn verify_decoded_stream_pop() {
    compare_decoded_stream("testsamples/pop.flac");
}

#[test]
fn verify_decoded_stream_short() {
    // The short sample has only 4 samples, even less than pop.flac.
    compare_decoded_stream("testsamples/short.flac");
}

#[test]
fn verify_decoded_stream_wasted_bits() {
    // This sample has subframes where the number of wasted bits is not 0.
    compare_decoded_stream("testsamples/wasted_bits.flac");
}

#[test]
fn verify_limits_on_vendor_string() {
    // This file claims to have a vendor string which would not fit in the
    // block.
    let file = fs::File::open("testsamples/large_vendor_string.flac").unwrap();
    match claxon::FlacReader::new(file) {
        Ok(..) => panic!("This file should fail to load"),
        Err(err) => {
            assert_eq!(err, claxon::Error::FormatError("vendor string too long"))
        }
    }
}

#[test]
fn verify_limits_on_vorbis_comment_block() {
    // This file claims to have a very large Vorbis comment block, which could
    // make the decoder go OOM.
    let file = fs::File::open("testsamples/large_vorbis_comment_block.flac").unwrap();
    match claxon::FlacReader::new(file) {
        Ok(..) => panic!("This file should fail to load"),
        Err(claxon::Error::Unsupported(..)) => { }
        Err(..) => panic!("Expected 'Unsupported' error."),
    }
}

#[test]
fn verify_extra_samples() {
    use std::ffi::OsStr;

    if !Path::new("testsamples/extra").exists() {
        return
    }

    let wd = walkdir::WalkDir::new("testsamples/extra")
        .follow_links(true)
        .into_iter()
        .filter_map(|e| e.ok());

    // Recursively enumerate all the flac files in the testsamples/extra
    // directory, and compare the streaminfo and stream itself for those.
    for entry in wd {
        let path = entry.path();
        if path.is_file() && path.extension() == Some(OsStr::new("flac")) {
            print!("    comparing {} ...", path.to_str()
                                               .expect("unsupported filename"));
            compare_metaflac_streaminfo(&path);
            compare_metaflac_vorbis_comment(&path);
            compare_decoded_stream(&path);
            println!(" ok");
        }
    }
}

#[test]
fn regression_test_fuzz_samples() {
    use std::ffi::OsStr;

    // Enumerate all the flac files in the testsamples/fuzz directory,
    // and ensure that they can be decoded without panic.
    let dir = fs::read_dir("testsamples/fuzz")
                 .ok().expect("failed to enumerate flac files");
    for path in dir {
        let path = path.ok().expect("failed to obtain path info").path();
        if path.is_file() && path.extension() == Some(OsStr::new("flac")) {
            print!("    regression testing {} ...", path.to_str()
                   .expect("unsupported filename"));

            if let Ok(mut reader) = claxon::FlacReader::open(&path) {
                let mut buffer = Vec::new();
                while let Ok(Some(block)) = reader.blocks().read_next_or_eof(buffer) {
                    buffer = block.into_buffer();
                }
            }
            println!(" ok");
        }
    }
}
<commit_msg>Verify empty vendor string if it starts with \0<commit_after>// Claxon -- A FLAC decoding library in Rust
// Copyright 2014 Ruud van Asseldonk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// A copy of the License has been included in the root of the repository.

extern crate claxon;
extern crate hound;
extern crate walkdir;

use std::fs;
use std::io;
use std::path::Path;

fn run_metaflac_streaminfo<P: AsRef<Path>>(fname: P) -> String {
    use std::process::Command;

    // Run metaflac on the specified file and print all streaminfo data.
    let output = Command::new("metaflac")
        .arg("--show-min-blocksize")
        .arg("--show-max-blocksize")
        .arg("--show-min-framesize")
        .arg("--show-max-framesize")
        .arg("--show-sample-rate")
        .arg("--show-channels")
        .arg("--show-bps")
        .arg("--show-total-samples")
        .arg("--show-md5sum")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run metaflac");
    String::from_utf8(output.stdout).expect("metaflac wrote invalid UTF-8")
}

fn run_metaflac_vorbis_comment<P: AsRef<Path>>(fname: P) -> String {
    use std::process::Command;

    // Run metaflac on the specified file and print all Vorbis comment data.
    let output = Command::new("metaflac")
        .arg("--block-type=VORBIS_COMMENT")
        .arg("--list")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run metaflac");
    String::from_utf8(output.stdout).expect("metaflac wrote invalid UTF-8")
}

fn decode_file<P: AsRef<Path>>(fname: P) -> Vec<u8> {
    use std::process::Command;

    // Run the the reference flac decoder on the file.
    let output = Command::new("flac")
        .arg("--decode")
        .arg("--silent")
        .arg("--stdout")
        .arg(fname.as_ref().to_str().expect("unsupported filename"))
        .output()
        .expect("failed to run flac");

    assert!(output.status.success());
    output.stdout
}

fn print_hex(seq: &[u8]) -> String {
    let vec: Vec<String> = seq.iter().map(|x| format!("{:0>2x}", *x)).collect();
    vec.concat()
}

fn read_streaminfo<P: AsRef<Path>>(fname: P) -> String {
    // Use a buffered reader, this speeds up the test by 120%.
    let file = fs::File::open(fname).unwrap();
    let reader = io::BufReader::new(file);
    let stream = claxon::FlacReader::new(reader).unwrap();
    let streaminfo = stream.streaminfo();

    // Format the streaminfo in the same way that metaflac prints it.
    format!("{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n",
            streaminfo.min_block_size,
            streaminfo.max_block_size,
            streaminfo.min_frame_size.unwrap_or(0),
            streaminfo.max_frame_size.unwrap_or(0),
            streaminfo.sample_rate,
            streaminfo.channels,
            streaminfo.bits_per_sample,
            streaminfo.samples.unwrap_or(0),
            print_hex(&streaminfo.md5sum)) // TODO implement LowerHex for &[u8] and submit a PR.
}

fn compare_metaflac_streaminfo<P: AsRef<Path>>(fname: P) {
    let metaflac = run_metaflac_streaminfo(&fname);
    let streaminfo = read_streaminfo(&fname);
    let mut mf_lines = metaflac.lines();
    let mut si_lines = streaminfo.lines();
    while let (Some(mf), Some(si)) = (mf_lines.next(), si_lines.next()) {
        if mf != si {
            println!("metaflac\n--------\n{}", metaflac);
            println!("streaminfo\n----------\n{}", streaminfo);
            panic!("metaflac disagrees on parsed streaminfo");
        }
    }
}

fn compare_metaflac_vorbis_comment<P: AsRef<Path>>(fname: P) {
    let metaflac = run_metaflac_vorbis_comment(&fname);

    // Use a buffered reader, this speeds up the test by 120%.
    let file = fs::File::open(fname).unwrap();
    let buf_reader = io::BufReader::new(file);
    let reader = claxon::FlacReader::new(buf_reader).unwrap();

    let mut mf_lines = metaflac.lines();

    // Search for the vendor string in the metaflac output.
    while let Some(line) = mf_lines.next() {
        let prefix = "  vendor string: ";
        if line.starts_with(prefix) {
            let mf_vendor_string = &line[prefix.len()..];

            // If the vendor string starts with a null byte, metaflac will not
            // print it -- my guess is because metaflac is written in C and uses
            // C-style string manipulation. In that case we skip it.
            match reader.vendor() {
                Some(x) if x.starts_with('\0') => {
                    assert_eq!("", mf_vendor_string);
                    break
                }
                _ => {}
            }

            assert_eq!(reader.vendor(), Some(mf_vendor_string));
            break
        }
    }

    let mut tags = reader.tags().iter();

    // Loop through all of the comments.
    while let Some(line) = mf_lines.next() {
        let prefix = "    comment[";
        if line.starts_with(prefix) {
            let mf_line = &line[prefix.len()..];
            let prefix_sep_index = mf_line.find(' ').unwrap();
            let mf_pair = &mf_line[prefix_sep_index + 1..];

            let sep_index = mf_pair.find('=').unwrap();
            let mf_name = &mf_pair[..sep_index];
            let mf_value = &mf_pair[sep_index + 1..];

            let &(ref name, ref value_lines) = tags.next().unwrap();
            let mut value_lines_iter = value_lines.lines();
            let value = value_lines_iter.next().unwrap_or("");

            assert_eq!(name, mf_name);
            assert_eq!(value, mf_value);

            // If there are newlines, then we also need to read those as
            // separate lines from the metaflac untput. This does assume that
            // the newline count that Claxon read is correct, and because of the
            // behavior of the `.lines()` iterator this does not accurately
            // verify carriage returns, but we could not anyway, because
            // metaflac does not escape them.
            while let Some(actual_line) = value_lines_iter.next() {
                assert_eq!(actual_line, mf_lines.next().unwrap());
            }
        }
    }
}

fn compare_decoded_stream<P: AsRef<Path>>(fname: P) {
    let wav = decode_file(&fname);
    let cursor = io::Cursor::new(wav);

    // Compare the reference decoded by the 'flac' program (stored as wav, which
    // we read with Hound) to how Claxon decodes it, sample by sample.
    let mut ref_wav_reader = hound::WavReader::new(cursor).unwrap();

    let try_file = fs::File::open(fname).unwrap();
    let try_buf_reader = io::BufReader::new(try_file);
    let mut try_flac_reader = claxon::FlacReader::new(try_buf_reader).unwrap();

    // The streaminfo test will ensure that things like bit depth and
    // sample rate match, only the actual samples are compared here.
    let mut ref_samples = ref_wav_reader.samples::<i32>();

    let samples = try_flac_reader.streaminfo().samples.unwrap();
    let n_channels = try_flac_reader.streaminfo().channels;
    let mut blocks = try_flac_reader.blocks();
    let mut sample = 0u64;
    let mut b = 0u64;
    let mut buffer = Vec::new();

    while sample < samples {
        let block = blocks.read_next_or_eof(buffer).unwrap().unwrap();
        {
            let mut channels: Vec<_> = (0..n_channels)
                                           .map(|i| block.channel(i).iter().cloned())
                                           .collect();
            for i in 0..block.duration() {
                for ch in 0..n_channels as usize {
                    let ref_sample = ref_samples.next().map(|r| r.ok().unwrap());
                    let try_sample = channels[ch].next();
                    if ref_sample != try_sample {
                        println!("disagreement at sample {} of block {} in channel {}: \
                                  reference is {} but decoded is {}",
                                 i,
                                 b,
                                 ch,
                                 ref_sample.unwrap(),
                                 try_sample.unwrap());
                        panic!("decoding differs from reference decoder");
                    }
                }
            }
            sample = sample + block.duration() as u64;
        }
        b = b + 1;
        buffer = block.into_buffer();
    }
}

// Hard-coded tests for the test samples added py populate.sh.
// This allows us to run these tests in parallel and follow progress without
// enabling --nocapture.

#[test]
fn verify_streaminfo_p0() {
    compare_metaflac_streaminfo("testsamples/p0.flac");
}

#[test]
fn verify_streaminfo_p1() {
    compare_metaflac_streaminfo("testsamples/p1.flac");
}

#[test]
fn verify_streaminfo_p2() {
    compare_metaflac_streaminfo("testsamples/p2.flac");
}

#[test]
fn verify_streaminfo_p3() {
    compare_metaflac_streaminfo("testsamples/p3.flac");
}

#[test]
fn verify_streaminfo_p4() {
    compare_metaflac_streaminfo("testsamples/p4.flac");
}

#[test]
fn verify_streaminfo_pop() {
    compare_metaflac_streaminfo("testsamples/pop.flac");
}

#[test]
fn verify_streaminfo_short() {
    compare_metaflac_streaminfo("testsamples/short.flac");
}

#[test]
fn verify_streaminfo_wasted_bits() {
    compare_metaflac_streaminfo("testsamples/wasted_bits.flac");
}

#[test]
fn verify_vorbis_comment_p0() {
    compare_metaflac_vorbis_comment("testsamples/p0.flac");
}

#[test]
fn verify_vorbis_comment_p1() {
    compare_metaflac_vorbis_comment("testsamples/p1.flac");
}

#[test]
fn verify_vorbis_comment_p2() {
    compare_metaflac_vorbis_comment("testsamples/p2.flac");
}

#[test]
fn verify_vorbis_comment_p3() {
    compare_metaflac_vorbis_comment("testsamples/p3.flac");
}

#[test]
fn verify_vorbis_comment_p4() {
    compare_metaflac_vorbis_comment("testsamples/p4.flac");
}

#[test]
fn verify_decoded_stream_p0() {
    compare_decoded_stream("testsamples/p0.flac");
}

#[test]
fn verify_decoded_stream_p1() {
    compare_decoded_stream("testsamples/p1.flac");
}

#[test]
fn verify_decoded_stream_p2() {
    compare_decoded_stream("testsamples/p2.flac");
}

#[test]
fn verify_decoded_stream_p3() {
    compare_decoded_stream("testsamples/p3.flac");
}

#[test]
fn verify_decoded_stream_p4() {
    compare_decoded_stream("testsamples/p4.flac");
}

#[test]
fn verify_decoded_stream_pop() {
    compare_decoded_stream("testsamples/pop.flac");
}

#[test]
fn verify_decoded_stream_short() {
    // The short sample has only 4 samples, even less than pop.flac.
    compare_decoded_stream("testsamples/short.flac");
}

#[test]
fn verify_decoded_stream_wasted_bits() {
    // This sample has subframes where the number of wasted bits is not 0.
    compare_decoded_stream("testsamples/wasted_bits.flac");
}

#[test]
fn verify_limits_on_vendor_string() {
    // This file claims to have a vendor string which would not fit in the
    // block.
    let file = fs::File::open("testsamples/large_vendor_string.flac").unwrap();
    match claxon::FlacReader::new(file) {
        Ok(..) => panic!("This file should fail to load"),
        Err(err) => {
            assert_eq!(err, claxon::Error::FormatError("vendor string too long"))
        }
    }
}

#[test]
fn verify_limits_on_vorbis_comment_block() {
    // This file claims to have a very large Vorbis comment block, which could
    // make the decoder go OOM.
    let file = fs::File::open("testsamples/large_vorbis_comment_block.flac").unwrap();
    match claxon::FlacReader::new(file) {
        Ok(..) => panic!("This file should fail to load"),
        Err(claxon::Error::Unsupported(..)) => { }
        Err(..) => panic!("Expected 'Unsupported' error."),
    }
}

#[test]
fn verify_extra_samples() {
    use std::ffi::OsStr;

    if !Path::new("testsamples/extra").exists() {
        return
    }

    let wd = walkdir::WalkDir::new("testsamples/extra")
        .follow_links(true)
        .into_iter()
        .filter_map(|e| e.ok());

    // Recursively enumerate all the flac files in the testsamples/extra
    // directory, and compare the streaminfo and stream itself for those.
    for entry in wd {
        let path = entry.path();
        if path.is_file() && path.extension() == Some(OsStr::new("flac")) {
            print!("    comparing {} ...", path.to_str()
                                               .expect("unsupported filename"));
            compare_metaflac_streaminfo(&path);
            compare_metaflac_vorbis_comment(&path);
            compare_decoded_stream(&path);
            println!(" ok");
        }
    }
}

#[test]
fn regression_test_fuzz_samples() {
    use std::ffi::OsStr;

    // Enumerate all the flac files in the testsamples/fuzz directory,
    // and ensure that they can be decoded without panic.
    let dir = fs::read_dir("testsamples/fuzz")
                 .ok().expect("failed to enumerate flac files");
    for path in dir {
        let path = path.ok().expect("failed to obtain path info").path();
        if path.is_file() && path.extension() == Some(OsStr::new("flac")) {
            print!("    regression testing {} ...", path.to_str()
                   .expect("unsupported filename"));

            if let Ok(mut reader) = claxon::FlacReader::open(&path) {
                let mut buffer = Vec::new();
                while let Ok(Some(block)) = reader.blocks().read_next_or_eof(buffer) {
                    buffer = block.into_buffer();
                }
            }
            println!(" ok");
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! Suppose we have the following data structure in a smart contract:
//!
//! struct B {
//!   Map<String, String> mymap;
//! }
//!
//! struct A {
//!   B b;
//!   int my_int;
//! }
//!
//! struct C {
//!   List<int> mylist;
//! }
//!
//! A a;
//! C c;
//!
//! and the data belongs to Alice. Then an access to `a.b.mymap` would be translated to an access
//! to an entry in key-value store whose key is `<Alice>/a/b/mymap`. In the same way, the access to
//! `c.mylist` would need to query `<Alice>/c/mylist`.
//!
//! So an account stores its data in a directory structure, for example:
//!   <Alice>/balance:   10
//!   <Alice>/a/b/mymap: {"Bob" => "abcd", "Carol" => "efgh"}
//!   <Alice>/a/myint:   20
//!   <Alice>/c/mylist:  [3, 5, 7, 9]
//!
//! If someone needs to query the map above and find out what value associated with "Bob" is,
//! `address` will be set to Alice and `path` will be set to "/a/b/mymap/Bob".
//!
//! On the other hand, if you want to query only <Alice>/a/*, `address` will be set to Alice and
//! `path` will be set to "/a" and use the `get_prefix()` method from statedb

use crate::{
    account_address::AccountAddress,
    account_config::{
        account_resource_path, association_address, ACCOUNT_RECEIVED_EVENT_PATH,
        ACCOUNT_SENT_EVENT_PATH,
    },
    language_storage::{ModuleId, ResourceKey, StructTag},
    validator_set::validator_set_path,
};
use canonical_serialization::{
    CanonicalDeserialize, CanonicalDeserializer, CanonicalSerialize, CanonicalSerializer,
};
use crypto::hash::{CryptoHash, HashValue};
use failure::prelude::*;
use hex;
use lazy_static::lazy_static;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use proto_conv::{FromProto, IntoProto};
use radix_trie::TrieKey;
use serde::{Deserialize, Serialize};
use std::{
    fmt::{self, Formatter},
    slice::Iter,
    str::{self, FromStr},
};

#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Hash, Eq, Clone, Ord, PartialOrd)]
pub struct Field(String);

impl Field {
    pub fn new(s: &str) -> Field {
        Field(s.to_string())
    }

    pub fn name(&self) -> &str {
        &self.0
    }
}

impl From<String> for Field {
    fn from(s: String) -> Self {
        Field(s)
    }
}

impl fmt::Display for Field {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

#[derive(Eq, Hash, Serialize, Deserialize, Debug, Clone, PartialEq, Ord, PartialOrd)]
pub enum Access {
    Field(Field),
    Index(u64),
}

impl Access {
    pub fn new(s: &str) -> Self {
        Access::Field(Field::new(s))
    }
}

impl FromStr for Access {
    type Err = ::std::num::ParseIntError;

    fn from_str(s: &str) -> ::std::result::Result<Self, Self::Err> {
        if let Ok(idx) = s.parse::<u64>() {
            Ok(Access::Index(idx))
        } else {
            Ok(Access::Field(Field::new(s)))
        }
    }
}

impl fmt::Display for Access {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Access::Field(field) => write!(f, "\"{}\"", field),
            Access::Index(i) => write!(f, "{}", i),
        }
    }
}

/// Non-empty sequence of field accesses
#[derive(Eq, Hash, Serialize, Deserialize, Debug, Clone, PartialEq, Ord, PartialOrd)]
pub struct Accesses(Vec<Access>);

/// SEPARATOR is used as a delimiter between fields. It should not be a legal part of any identifier
/// in the language
const SEPARATOR: char = '/';

impl Accesses {
    pub fn empty() -> Self {
        Accesses(vec![])
    }

    pub fn new(field: Field) -> Self {
        Accesses(vec![Access::Field(field)])
    }

    /// Add a field to the end of the sequence
    pub fn add_field_to_back(&mut self, field: Field) {
        self.0.push(Access::Field(field))
    }

    /// Add an index to the end of the sequence
    pub fn add_index_to_back(&mut self, idx: u64) {
        self.0.push(Access::Index(idx))
    }

    pub fn append(&mut self, accesses: &mut Accesses) {
        self.0.append(&mut accesses.0)
    }

    /// Returns the first field in the sequence and reference to the remaining fields
    pub fn split_first(&self) -> (&Access, &[Access]) {
        self.0.split_first().unwrap()
    }

    /// Return the last access in the sequence
    pub fn last(&self) -> &Access {
        self.0.last().unwrap() // guaranteed not to fail because sequence is non-empty
    }

    pub fn iter(&self) -> Iter<'_, Access> {
        self.0.iter()
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }

    pub fn as_separated_string(&self) -> String {
        let mut path = String::new();
        for access in self.0.iter() {
            match access {
                Access::Field(s) => {
                    let access_str = s.name().as_ref();
                    assert!(access_str != "");
                    path.push_str(access_str)
                }
                Access::Index(i) => path.push_str(i.to_string().as_ref()),
            };
            path.push(SEPARATOR);
        }
        path
    }

    pub fn take_nth(&self, new_len: usize) -> Accesses {
        assert!(self.0.len() >= new_len);
        Accesses(self.0.clone().into_iter().take(new_len).collect())
    }
}

impl<'a> IntoIterator for &'a Accesses {
    type Item = &'a Access;
    type IntoIter = Iter<'a, Access>;
    fn into_iter(self) -> Self::IntoIter {
        self.0.iter()
    }
}

impl From<Vec<Access>> for Accesses {
    fn from(accesses: Vec<Access>) -> Accesses {
        Accesses(accesses)
    }
}

impl From<Vec<u8>> for Accesses {
    fn from(mut raw_bytes: Vec<u8>) -> Accesses {
        let access_str = String::from_utf8(raw_bytes.split_off(HashValue::LENGTH + 1)).unwrap();
        let fields_str = access_str.split(SEPARATOR).collect::<Vec<&str>>();
        let mut accesses = vec![];
        for access_str in fields_str.into_iter() {
            if access_str != "" {
                accesses.push(Access::from_str(access_str).unwrap());
            }
        }
        Accesses::from(accesses)
    }
}

impl TrieKey for Accesses {
    fn encode_bytes(&self) -> Vec<u8> {
        self.as_separated_string().into_bytes()
    }
}

lazy_static! {
    /// The access path where the Validator Set resource is stored.
    pub static ref VALIDATOR_SET_ACCESS_PATH: AccessPath =
        AccessPath::new(association_address(), validator_set_path());
}

#[derive(
    Clone,
    Eq,
    PartialEq,
    Default,
    Hash,
    Serialize,
    Deserialize,
    Ord,
    PartialOrd,
    FromProto,
    IntoProto,
)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
#[ProtoType(crate::proto::access_path::AccessPath)]
pub struct AccessPath {
    pub address: AccountAddress,
    pub path: Vec<u8>,
}

impl AccessPath {
    const CODE_TAG: u8 = 0;
    const RESOURCE_TAG: u8 = 1;

    pub fn new(address: AccountAddress, path: Vec<u8>) -> Self {
        AccessPath { address, path }
    }

    /// Given an address, returns the corresponding access path that stores the Account resource.
    pub fn new_for_account(address: AccountAddress) -> Self {
        Self::new(address, account_resource_path())
    }

    /// Create an AccessPath for a ContractEvent.
    /// That is an AccessPah that uniquely identifies a given event for a published resource.
    pub fn new_for_event(address: AccountAddress, root: &[u8], key: &[u8]) -> Self {
        let mut path: Vec<u8> = Vec::new();
        path.extend_from_slice(root);
        path.push(b'/');
        path.extend_from_slice(key);
        path.push(b'/');
        Self::new(address, path)
    }

    /// Create an AccessPath to the event for the sender account in a deposit operation.
    /// The sent counter in LibraAccount.T (LibraAccount.T.sent_events_count) is used to generate
    /// the AccessPath.
    /// That AccessPath can be used as a key into the event storage to retrieve all sent
    /// events for a given account.
    pub fn new_for_sent_event(address: AccountAddress) -> Self {
        Self::new(address, ACCOUNT_SENT_EVENT_PATH.to_vec())
    }

    /// Create an AccessPath to the event for the target account (the receiver)
    /// in a deposit operation.
    /// The received counter in LibraAccount.T (LibraAccount.T.received_events_count) is used to
    /// generate the AccessPath.
    /// That AccessPath can be used as a key into the event storage to retrieve all received
    /// events for a given account.
    pub fn new_for_received_event(address: AccountAddress) -> Self {
        Self::new(address, ACCOUNT_RECEIVED_EVENT_PATH.to_vec())
    }

    pub fn resource_access_vec(tag: &StructTag, accesses: &Accesses) -> Vec<u8> {
        let mut key = vec![];
        key.push(Self::RESOURCE_TAG);

        key.append(&mut tag.hash().to_vec());

        // We don't need accesses in production right now. Accesses are appended here just for
        // passing the old tests.
        key.append(&mut accesses.as_separated_string().into_bytes());
        key
    }

    /// Convert Accesses into a byte offset which would be used by the storage layer to resolve
    /// where fields are stored.
    pub fn resource_access_path(key: &ResourceKey, accesses: &Accesses) -> AccessPath {
        let path = AccessPath::resource_access_vec(&key.type_(), accesses);
        AccessPath {
            address: key.address().to_owned(),
            path,
        }
    }

    fn code_access_path_vec(key: &ModuleId) -> Vec<u8> {
        let mut root = vec![];
        root.push(Self::CODE_TAG);
        root.append(&mut key.hash().to_vec());
        root
    }

    pub fn code_access_path(key: &ModuleId) -> AccessPath {
        let path = AccessPath::code_access_path_vec(key);
        AccessPath {
            address: *key.address(),
            path,
        }
    }
}

impl fmt::Debug for AccessPath {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "AccessPath {{ address: {:x}, path: {} }}",
            self.address,
            hex::encode(&self.path)
        )
    }
}

impl fmt::Display for AccessPath {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        if self.path.len() < 1 + HashValue::LENGTH {
            write!(f, "{:?}", self)
        } else {
            write!(f, "AccessPath {{ address: {:x}, ", self.address)?;
            match self.path[0] {
                Self::RESOURCE_TAG => write!(f, "type: Resource, ")?,
                Self::CODE_TAG => write!(f, "type: Module, ")?,
                tag => write!(f, "type: {:?}, ", tag)?,
            };
            write!(
                f,
                "hash: {:?}, ",
                hex::encode(&self.path[1..=HashValue::LENGTH])
            )?;
            write!(
                f,
                "suffix: {:?} }} ",
                String::from_utf8_lossy(&self.path[1 + HashValue::LENGTH..])
            )
        }
    }
}

impl CanonicalSerialize for AccessPath {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_struct(&self.address)?
            .encode_bytes(&self.path)?;
        Ok(())
    }
}

impl CanonicalDeserialize for AccessPath {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let address = deserializer.decode_struct::<AccountAddress>()?;
        let path = deserializer.decode_bytes()?;

        Ok(Self { address, path })
    }
}
<commit_msg>[types] remove some unused, scary code around access paths<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! Suppose we have the following data structure in a smart contract:
//!
//! struct B {
//!   Map<String, String> mymap;
//! }
//!
//! struct A {
//!   B b;
//!   int my_int;
//! }
//!
//! struct C {
//!   List<int> mylist;
//! }
//!
//! A a;
//! C c;
//!
//! and the data belongs to Alice. Then an access to `a.b.mymap` would be translated to an access
//! to an entry in key-value store whose key is `<Alice>/a/b/mymap`. In the same way, the access to
//! `c.mylist` would need to query `<Alice>/c/mylist`.
//!
//! So an account stores its data in a directory structure, for example:
//!   <Alice>/balance:   10
//!   <Alice>/a/b/mymap: {"Bob" => "abcd", "Carol" => "efgh"}
//!   <Alice>/a/myint:   20
//!   <Alice>/c/mylist:  [3, 5, 7, 9]
//!
//! If someone needs to query the map above and find out what value associated with "Bob" is,
//! `address` will be set to Alice and `path` will be set to "/a/b/mymap/Bob".
//!
//! On the other hand, if you want to query only <Alice>/a/*, `address` will be set to Alice and
//! `path` will be set to "/a" and use the `get_prefix()` method from statedb

use crate::{
    account_address::AccountAddress,
    account_config::{
        account_resource_path, association_address, ACCOUNT_RECEIVED_EVENT_PATH,
        ACCOUNT_SENT_EVENT_PATH,
    },
    language_storage::{ModuleId, ResourceKey, StructTag},
    validator_set::validator_set_path,
};
use canonical_serialization::{
    CanonicalDeserialize, CanonicalDeserializer, CanonicalSerialize, CanonicalSerializer,
};
use crypto::hash::{CryptoHash, HashValue};
use failure::prelude::*;
use hex;
use lazy_static::lazy_static;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use proto_conv::{FromProto, IntoProto};
use radix_trie::TrieKey;
use serde::{Deserialize, Serialize};
use std::{fmt, slice::Iter};

#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Hash, Eq, Clone, Ord, PartialOrd)]
pub struct Field(String);

impl Field {
    pub fn new(s: &str) -> Field {
        Field(s.to_string())
    }

    pub fn name(&self) -> &str {
        &self.0
    }
}

impl From<String> for Field {
    fn from(s: String) -> Self {
        Field(s)
    }
}

impl fmt::Display for Field {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.0)
    }
}

#[derive(Eq, Hash, Serialize, Deserialize, Debug, Clone, PartialEq, Ord, PartialOrd)]
pub enum Access {
    Field(Field),
    Index(u64),
}

impl Access {
    pub fn new(s: &str) -> Self {
        Access::Field(Field::new(s))
    }
}

impl fmt::Display for Access {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Access::Field(field) => write!(f, "\"{}\"", field),
            Access::Index(i) => write!(f, "{}", i),
        }
    }
}

/// Non-empty sequence of field accesses
#[derive(Eq, Hash, Serialize, Deserialize, Debug, Clone, PartialEq, Ord, PartialOrd)]
pub struct Accesses(Vec<Access>);

/// SEPARATOR is used as a delimiter between fields. It should not be a legal part of any identifier
/// in the language
const SEPARATOR: char = '/';

impl Accesses {
    pub fn empty() -> Self {
        Accesses(vec![])
    }

    pub fn new(field: Field) -> Self {
        Accesses(vec![Access::Field(field)])
    }

    /// Add a field to the end of the sequence
    pub fn add_field_to_back(&mut self, field: Field) {
        self.0.push(Access::Field(field))
    }

    /// Add an index to the end of the sequence
    pub fn add_index_to_back(&mut self, idx: u64) {
        self.0.push(Access::Index(idx))
    }

    pub fn append(&mut self, accesses: &mut Accesses) {
        self.0.append(&mut accesses.0)
    }

    /// Returns the first field in the sequence and reference to the remaining fields
    pub fn split_first(&self) -> (&Access, &[Access]) {
        self.0.split_first().unwrap()
    }

    /// Return the last access in the sequence
    pub fn last(&self) -> &Access {
        self.0.last().unwrap() // guaranteed not to fail because sequence is non-empty
    }

    pub fn iter(&self) -> Iter<'_, Access> {
        self.0.iter()
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }

    pub fn as_separated_string(&self) -> String {
        let mut path = String::new();
        for access in self.0.iter() {
            match access {
                Access::Field(s) => {
                    let access_str = s.name().as_ref();
                    assert!(access_str != "");
                    path.push_str(access_str)
                }
                Access::Index(i) => path.push_str(i.to_string().as_ref()),
            };
            path.push(SEPARATOR);
        }
        path
    }

    pub fn take_nth(&self, new_len: usize) -> Accesses {
        assert!(self.0.len() >= new_len);
        Accesses(self.0.clone().into_iter().take(new_len).collect())
    }
}

impl<'a> IntoIterator for &'a Accesses {
    type Item = &'a Access;
    type IntoIter = Iter<'a, Access>;
    fn into_iter(self) -> Self::IntoIter {
        self.0.iter()
    }
}

impl From<Vec<Access>> for Accesses {
    fn from(accesses: Vec<Access>) -> Accesses {
        Accesses(accesses)
    }
}

impl TrieKey for Accesses {
    fn encode_bytes(&self) -> Vec<u8> {
        self.as_separated_string().into_bytes()
    }
}

lazy_static! {
    /// The access path where the Validator Set resource is stored.
    pub static ref VALIDATOR_SET_ACCESS_PATH: AccessPath =
        AccessPath::new(association_address(), validator_set_path());
}

#[derive(
    Clone,
    Eq,
    PartialEq,
    Default,
    Hash,
    Serialize,
    Deserialize,
    Ord,
    PartialOrd,
    FromProto,
    IntoProto,
)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
#[ProtoType(crate::proto::access_path::AccessPath)]
pub struct AccessPath {
    pub address: AccountAddress,
    pub path: Vec<u8>,
}

impl AccessPath {
    const CODE_TAG: u8 = 0;
    const RESOURCE_TAG: u8 = 1;

    pub fn new(address: AccountAddress, path: Vec<u8>) -> Self {
        AccessPath { address, path }
    }

    /// Given an address, returns the corresponding access path that stores the Account resource.
    pub fn new_for_account(address: AccountAddress) -> Self {
        Self::new(address, account_resource_path())
    }

    /// Create an AccessPath for a ContractEvent.
    /// That is an AccessPah that uniquely identifies a given event for a published resource.
    pub fn new_for_event(address: AccountAddress, root: &[u8], key: &[u8]) -> Self {
        let mut path: Vec<u8> = Vec::new();
        path.extend_from_slice(root);
        path.push(b'/');
        path.extend_from_slice(key);
        path.push(b'/');
        Self::new(address, path)
    }

    /// Create an AccessPath to the event for the sender account in a deposit operation.
    /// The sent counter in LibraAccount.T (LibraAccount.T.sent_events_count) is used to generate
    /// the AccessPath.
    /// That AccessPath can be used as a key into the event storage to retrieve all sent
    /// events for a given account.
    pub fn new_for_sent_event(address: AccountAddress) -> Self {
        Self::new(address, ACCOUNT_SENT_EVENT_PATH.to_vec())
    }

    /// Create an AccessPath to the event for the target account (the receiver)
    /// in a deposit operation.
    /// The received counter in LibraAccount.T (LibraAccount.T.received_events_count) is used to
    /// generate the AccessPath.
    /// That AccessPath can be used as a key into the event storage to retrieve all received
    /// events for a given account.
    pub fn new_for_received_event(address: AccountAddress) -> Self {
        Self::new(address, ACCOUNT_RECEIVED_EVENT_PATH.to_vec())
    }

    pub fn resource_access_vec(tag: &StructTag, accesses: &Accesses) -> Vec<u8> {
        let mut key = vec![];
        key.push(Self::RESOURCE_TAG);

        key.append(&mut tag.hash().to_vec());

        // We don't need accesses in production right now. Accesses are appended here just for
        // passing the old tests.
        key.append(&mut accesses.as_separated_string().into_bytes());
        key
    }

    /// Convert Accesses into a byte offset which would be used by the storage layer to resolve
    /// where fields are stored.
    pub fn resource_access_path(key: &ResourceKey, accesses: &Accesses) -> AccessPath {
        let path = AccessPath::resource_access_vec(&key.type_(), accesses);
        AccessPath {
            address: key.address().to_owned(),
            path,
        }
    }

    fn code_access_path_vec(key: &ModuleId) -> Vec<u8> {
        let mut root = vec![];
        root.push(Self::CODE_TAG);
        root.append(&mut key.hash().to_vec());
        root
    }

    pub fn code_access_path(key: &ModuleId) -> AccessPath {
        let path = AccessPath::code_access_path_vec(key);
        AccessPath {
            address: *key.address(),
            path,
        }
    }
}

impl fmt::Debug for AccessPath {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "AccessPath {{ address: {:x}, path: {} }}",
            self.address,
            hex::encode(&self.path)
        )
    }
}

impl fmt::Display for AccessPath {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.path.len() < 1 + HashValue::LENGTH {
            write!(f, "{:?}", self)
        } else {
            write!(f, "AccessPath {{ address: {:x}, ", self.address)?;
            match self.path[0] {
                Self::RESOURCE_TAG => write!(f, "type: Resource, ")?,
                Self::CODE_TAG => write!(f, "type: Module, ")?,
                tag => write!(f, "type: {:?}, ", tag)?,
            };
            write!(
                f,
                "hash: {:?}, ",
                hex::encode(&self.path[1..=HashValue::LENGTH])
            )?;
            write!(
                f,
                "suffix: {:?} }} ",
                String::from_utf8_lossy(&self.path[1 + HashValue::LENGTH..])
            )
        }
    }
}

impl CanonicalSerialize for AccessPath {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_struct(&self.address)?
            .encode_bytes(&self.path)?;
        Ok(())
    }
}

impl CanonicalDeserialize for AccessPath {
    fn deserialize(deserializer: &mut impl CanonicalDeserializer) -> Result<Self> {
        let address = deserializer.decode_struct::<AccountAddress>()?;
        let path = deserializer.decode_bytes()?;

        Ok(Self { address, path })
    }
}
<|endoftext|>


<commit_before>//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
//!
//! SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
//! MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
//!
//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
//! to be used for on-disk/permanent storage (e.g. checksums).
//!
//! # Benchmark
//!
//! On normal hardware, it is expected to run with a rate around 5.9-6.7 GB/S on a 2.5 GHz CPU.
//! Further improvement can be seen when hashing very big buffers in parallel.
//!
//! ## Ideal architecture
//!
//! SeaHash is designed and optimized for the most common architecture in use:
//!
//! - Little-endian
//! - 64-bit
//! - 64 or more bytes cache lines
//! - 4 or more instruction pipelines
//! - 4 or more 64-bit registers
//!
//! Anything that does not hold the above requirements will perform worse by up to 30-40%. Note that
//! this means it is still faster than CityHash (~1 GB/S), MurMurHash (~2.6 GB/S), FNV (~0.5 GB/S),
//! etc.
//!
//! # Achieving the performance
//!
//! Like any good general-purpose hash function, SeaHash reads 8 bytes at once effectively reducing
//! the running time by an order of ~5.
//!
//! Secondly, SeaHash achieves the performance by heavily exploiting Instruction-Level Parallelism.
//! In particular, it fetches 4 integers in every round and independently diffuses them. This
//! yields four different states, which are finally combined.
//!
//! # Advantages over other hash functions
//!
//! - Portability: SeaHash always gives the same hash across any platforms, and can thus be used
//!   for e.g. on-disk structures.
//! - Performance: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function that
//!   I know off.
//! - Hardware accelerateable: SeaHash is designed such that ASICs can implement it with really
//!   high performance.
//! - Provable quality guarantees: Contrary to most other non-cryptographic hash function, SeaHash
//!   can be proved to satisfy the avalanche criterion as well as BIC.
//!
//! # Warning!
//!
//! This is **not** a cryptographic function, and it certainly should not be used as one. If you
//! want a good cryptograhic hash function, you should use SHA-3 (Keccak) or BLAKE2.
//!
//! # Statistical guarantees
//!
//! SeaHash comes with certain proven guarantees about the statistical properties of the output:
//!
//! 1. Pick some _n_-byte sequence, _s_. The number of _n_-byte sequence colliding with _s_ is
//!    independent of the choice of _s_ (all equivalence class have equal size).
//! 2. If you flip any bit in the input, the probability for any bit in the output to be flipped is
//!    0.5.
//! 3. The hash value of a sequence of uniformly distributed bytes is itself uniformly distributed.
//!
//! The first guarantee can be derived through deduction, by proving that the diffusion function is
//! bijective (reverse the XORs and find the congruence inverses to the primes).
//!
//! The second guarantee requires more complex calculations: Construct a matrix of probabilities
//! and set one to certain (1), then apply transformations through the respective operations. The
//! proof is a bit long, but relatively simple.
//!
//! The third guarantee requires proving that the hash value is a tree, such that:
//! - Leafs represents the input values.
//! - Single-child nodes reduce to the diffusion of the child.
//! - Multiple-child nodes reduce to the sum of the children.
//!
//! Then simply show that each of these reductions transform uniformly distributed variables to
//! uniformly distributed variables.
//!
//! # Inner workings
//!
//! In technical terms, SeaHash follows a alternating 4-state length-padded Merkle–Damgård
//! construction with an XOR-diffuse compression function (click to enlargen):
//!
//! [![A diagram.](http://ticki.github.io/img/seahash_construction_diagram.svg)]
//! (http://ticki.github.io/img/seahash_construction_diagram.svg)
//!
//! It starts with 4 initial states, then it alternates between them (increment, wrap on 4) and
//! does XOR with the respective block. When a state has been visited the diffusion function (f) is
//! applied. The very last block is padded with zeros.
//!
//! After all the blocks have been gone over, all the states are XOR'd to the number of bytes
//! written. The sum is then passed through the diffusion function, which produces the final hash
//! value.
//!
//! The diffusion function is drawn below.
//!
//! ```notest
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! ```
//!
//! The advantage of having four completely segregated (note that there is no mix round, so they're
//! entirely independent) states is that fast parallelism is possible. For example, if I were to
//! hash 1 TB, I can spawn up four threads which can run independently without _any_
//! intercommunication or syncronization before the last round.
//!
//! If the diffusion function (f) was cryptographically secure, it would pass cryptoanalysis
//! trivially. This might seem irrelavant, as it clearly isn't cryptographically secure, but it
//! tells us something about the inner semantics. In particular, any diffusion function with
//! sufficient statistical quality will make up a good hash function in this construction.
//!
//! # ASIC version
//!
//! SeaHash is specifically designed such that it can be efficiently implemented in the form of
//! ASIC while only using very few transistors.
//!
//! # Specification
//!
//! See the [`reference`](./reference) module.

#![no_std]
#![warn(missing_docs)]

pub use buffer::hash;
pub use stream::SeaHasher;

pub mod reference;
mod buffer;
mod stream;

/// The diffusion function.
///
/// This is a bijective function emitting chaotic behavior. Such functions are used as building
/// blocks for hash functions.
fn diffuse(mut x: u64) -> u64 {
    // Move entropy down by XOR with shifting.
    x = x ^ (x >> 32);
    // Move entropy up by scattering through multiplication.
    x = x.wrapping_mul(0x7ed0e9fa0d94a33);
    // We still need more entropy downwards. Flipping higher bits won't flip lower ones, so far.
    // For example, if you flip the most significant bit, the 32'th bit will flip per the XOR-shift
    // subdiffusion, but this flip will only be scattered by the multiplication to flipping bits
    // higher than the 32'th, meaning that the ones lower will be unaffected. As such, we need to
    // get some entropy down.
    x = x ^ (x >> 32);
    // So far, the avalanche diagram looks pretty good, but it still emits stripe patterns. For
    // example, flipping the 5'th lowest bit won't flip the least significant bit because of the
    // choice of scalar (in particular, observe how it leaves the 32'th bit unflipped after the
    // multiplication, which means that the XOR-shift never affects the lowest bit). No choice of
    // scalar will make this go away, it will merely change the unaffected bits. Instead, we need
    // to make the behavior more undeterministic by scattering bits through multiplication.
    x = x.wrapping_mul(0x7ed0e9fa0d94a33);
    // This is the final stage of the diffusion function. There are still issues with the lowest
    // bits, which are still unaffected by the multiplication above. However, the multiplication
    // solved the higher bits' dependence, so lending entropy from the higher half will fix the
    // issues with the lower half.
    x = x ^ (x >> 32);

    x
}
<commit_msg>Subtle optimization in SeaHash, as noticed by @funny-falcon.<commit_after>//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
//!
//! SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
//! MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
//!
//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
//! to be used for on-disk/permanent storage (e.g. checksums).
//!
//! # Benchmark
//!
//! On normal hardware, it is expected to run with a rate around 5.9-6.7 GB/S on a 2.5 GHz CPU.
//! Further improvement can be seen when hashing very big buffers in parallel.
//!
//! ## Ideal architecture
//!
//! SeaHash is designed and optimized for the most common architecture in use:
//!
//! - Little-endian
//! - 64-bit
//! - 64 or more bytes cache lines
//! - 4 or more instruction pipelines
//! - 4 or more 64-bit registers
//!
//! Anything that does not hold the above requirements will perform worse by up to 30-40%. Note that
//! this means it is still faster than CityHash (~1 GB/S), MurMurHash (~2.6 GB/S), FNV (~0.5 GB/S),
//! etc.
//!
//! # Achieving the performance
//!
//! Like any good general-purpose hash function, SeaHash reads 8 bytes at once effectively reducing
//! the running time by an order of ~5.
//!
//! Secondly, SeaHash achieves the performance by heavily exploiting Instruction-Level Parallelism.
//! In particular, it fetches 4 integers in every round and independently diffuses them. This
//! yields four different states, which are finally combined.
//!
//! # Advantages over other hash functions
//!
//! - Portability: SeaHash always gives the same hash across any platforms, and can thus be used
//!   for e.g. on-disk structures.
//! - Performance: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function that
//!   I know off.
//! - Hardware accelerateable: SeaHash is designed such that ASICs can implement it with really
//!   high performance.
//! - Provable quality guarantees: Contrary to most other non-cryptographic hash function, SeaHash
//!   can be proved to satisfy the avalanche criterion as well as BIC.
//!
//! # Warning!
//!
//! This is **not** a cryptographic function, and it certainly should not be used as one. If you
//! want a good cryptograhic hash function, you should use SHA-3 (Keccak) or BLAKE2.
//!
//! # Statistical guarantees
//!
//! SeaHash comes with certain proven guarantees about the statistical properties of the output:
//!
//! 1. Pick some _n_-byte sequence, _s_. The number of _n_-byte sequence colliding with _s_ is
//!    independent of the choice of _s_ (all equivalence class have equal size).
//! 2. If you flip any bit in the input, the probability for any bit in the output to be flipped is
//!    0.5.
//! 3. The hash value of a sequence of uniformly distributed bytes is itself uniformly distributed.
//!
//! The first guarantee can be derived through deduction, by proving that the diffusion function is
//! bijective (reverse the XORs and find the congruence inverses to the primes).
//!
//! The second guarantee requires more complex calculations: Construct a matrix of probabilities
//! and set one to certain (1), then apply transformations through the respective operations. The
//! proof is a bit long, but relatively simple.
//!
//! The third guarantee requires proving that the hash value is a tree, such that:
//! - Leafs represents the input values.
//! - Single-child nodes reduce to the diffusion of the child.
//! - Multiple-child nodes reduce to the sum of the children.
//!
//! Then simply show that each of these reductions transform uniformly distributed variables to
//! uniformly distributed variables.
//!
//! # Inner workings
//!
//! In technical terms, SeaHash follows a alternating 4-state length-padded Merkle–Damgård
//! construction with an XOR-diffuse compression function (click to enlargen):
//!
//! [![A diagram.](http://ticki.github.io/img/seahash_construction_diagram.svg)]
//! (http://ticki.github.io/img/seahash_construction_diagram.svg)
//!
//! It starts with 4 initial states, then it alternates between them (increment, wrap on 4) and
//! does XOR with the respective block. When a state has been visited the diffusion function (f) is
//! applied. The very last block is padded with zeros.
//!
//! After all the blocks have been gone over, all the states are XOR'd to the number of bytes
//! written. The sum is then passed through the diffusion function, which produces the final hash
//! value.
//!
//! The diffusion function is drawn below.
//!
//! ```notest
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! ```
//!
//! The advantage of having four completely segregated (note that there is no mix round, so they're
//! entirely independent) states is that fast parallelism is possible. For example, if I were to
//! hash 1 TB, I can spawn up four threads which can run independently without _any_
//! intercommunication or syncronization before the last round.
//!
//! If the diffusion function (f) was cryptographically secure, it would pass cryptoanalysis
//! trivially. This might seem irrelavant, as it clearly isn't cryptographically secure, but it
//! tells us something about the inner semantics. In particular, any diffusion function with
//! sufficient statistical quality will make up a good hash function in this construction.
//!
//! # ASIC version
//!
//! SeaHash is specifically designed such that it can be efficiently implemented in the form of
//! ASIC while only using very few transistors.
//!
//! # Specification
//!
//! See the [`reference`](./reference) module.

#![no_std]
#![warn(missing_docs)]

pub use buffer::hash;
pub use stream::SeaHasher;

pub mod reference;
mod buffer;
mod stream;

/// The diffusion function.
///
/// This is a bijective function emitting chaotic behavior. Such functions are used as building
/// blocks for hash functions.
fn diffuse(mut x: u64) -> u64 {
    // Move entropy up by scattering through multiplication.
    x = x.wrapping_mul(0x7ed0e9fa0d94a33);
    // We still need more entropy downwards. Flipping higher bits won't flip lower ones, so far.
    // For example, if you flip the most significant bit, the 32'th bit will flip per the XOR-shift
    // subdiffusion, but this flip will only be scattered by the multiplication to flipping bits
    // higher than the 32'th, meaning that the ones lower will be unaffected. As such, we need to
    // get some entropy down.
    x = x ^ (x >> 32);
    // So far, the avalanche diagram looks pretty good, but it still emits stripe patterns. For
    // example, flipping the 5'th lowest bit won't flip the least significant bit because of the
    // choice of scalar (in particular, observe how it leaves the 32'th bit unflipped after the
    // multiplication, which means that the XOR-shift never affects the lowest bit). No choice of
    // scalar will make this go away, it will merely change the unaffected bits. Instead, we need
    // to make the behavior more undeterministic by scattering bits through multiplication.
    x = x.wrapping_mul(0x7ed0e9fa0d94a33);
    // This is the final stage of the diffusion function. There are still issues with the lowest
    // bits, which are still unaffected by the multiplication above. However, the multiplication
    // solved the higher bits' dependence, so lending entropy from the higher half will fix the
    // issues with the lower half.
    x = x ^ (x >> 32);

    // There is still a bias, but this is solved in the very last round of the hash function,
    // because applying this function twice, reduces this bias.

    x
}
<|endoftext|>


<commit_before>use std::env;
use std::process::Command;

#[cfg(not(feature = "rustc-dep-of-std"))]
fn build() {
	let out_dir = env::var("OUT_DIR").unwrap();
	let profile = env::var("PROFILE").expect("PROFILE was not set");

	let _output = Command::new("cargo")
		.current_dir(out_dir.clone())
		.arg("download")
		.arg("--output")
		.arg(out_dir.clone()+"/rusty-hermit")
		.arg("--extract")
		.arg("rusty-hermit")
		.output()
		.expect("Unable to download rusty-hermit. Please install `cargo-download`.");

	if profile == "release" {
		let _output = Command::new("cargo")
			.current_dir(out_dir.clone()+"/rusty-hermit")
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-hermit-kernel")
			.arg("--release")
			.output()
			.expect("Unable to build kernel");
	} else {
		let _output = Command::new("cargo")
			.current_dir(out_dir.clone()+"/rusty-hermit")
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-hermit-kernel")
			.output()
			.expect("Unable to build kernel");
	}

	println!(
		"cargo:rustc-link-search=native={}/rusty-hermit/target/x86_64-unknown-hermit-kernel/{}",
		out_dir.clone(),
		profile
	);
	println!("cargo:rustc-link-lib=static=hermit");
}

fn main() {
	#[cfg(not(feature = "rustc-dep-of-std"))]
	build();
}
<commit_msg>minor changes to pass format checks<commit_after>use std::env;
use std::process::Command;

#[cfg(not(feature = "rustc-dep-of-std"))]
fn build() {
	let out_dir = env::var("OUT_DIR").unwrap();
	let profile = env::var("PROFILE").expect("PROFILE was not set");

	let _output = Command::new("cargo")
		.current_dir(out_dir.clone())
		.arg("download")
		.arg("--output")
		.arg(out_dir.clone()+"/rusty-hermit")
		.arg("--extract")
		.arg("rusty-hermit")
		.output()
		.expect("Unable to download rusty-hermit. Please install `cargo-download`.");

	if profile == "release" {
		let _output = Command::new("cargo")
			.current_dir(out_dir.clone() + "/rusty-hermit")
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-hermit-kernel")
			.arg("--release")
			.output()
			.expect("Unable to build kernel");
	} else {
		let _output = Command::new("cargo")
			.current_dir(out_dir.clone() + "/rusty-hermit")
			.arg("build")
			.arg("-Z")
			.arg("build-std=core,alloc")
			.arg("--target")
			.arg("x86_64-unknown-hermit-kernel")
			.output()
			.expect("Unable to build kernel");
	}

	println!(
		"cargo:rustc-link-search=native={}/rusty-hermit/target/x86_64-unknown-hermit-kernel/{}",
		out_dir.clone(),
		profile
	);
	println!("cargo:rustc-link-lib=static=hermit");
}

fn main() {
	#[cfg(not(feature = "rustc-dep-of-std"))]
	build();
}
<|endoftext|>