<commit_before>use crate::error::{Error,
                   Result};
use blake2b_simd::{Params,
                   State};
use hex::FromHex;
use serde::Serialize;
use std::{convert::TryInto,
          fmt,
          fs::File,
          io::{BufReader,
               Read},
          path::Path,
          str::FromStr};

/// When hashing byte streams, we'll read 1KB at a time, adding this to the
/// internal hashing state as we compute the final digest.
const BUF_SIZE: usize = 1024;

/// The size of our Blake2b hash digests (32 bytes)
const HASH_DIGEST_SIZE: usize = 32;

/// Convenience wrapper type for a 32-byte Blake2b hash digest.
///
/// Implements secure equality comparison, as well as hex-encoding via
/// `std::fmt::Display`.
///
/// There are intentionally no explicit constructors; to get a
/// instance, you'll need to either convert it directly from a `&[u8]`
/// or parse it from a hex string.
#[derive(Clone, Debug)]
pub struct Blake2bHash {
    digest:     [u8; HASH_DIGEST_SIZE],
    /// Temporary field to support Deref<str> for backwards
    /// compatibility with Builder until it can use the new types.
    hex_string: String,
}

impl Blake2bHash {
    /// Temporary constructor while we store the hex encoding in the
    /// type directly.
    fn new(digest: [u8; HASH_DIGEST_SIZE]) -> Self {
        let hex_string = hex::encode(&digest).to_lowercase();
        Blake2bHash { digest, hex_string }
    }
}

impl From<blake2b_simd::Hash> for Blake2bHash {
    fn from(src: blake2b_simd::Hash) -> Self {
        let digest = src.as_bytes()
                        .try_into()
                        .expect("We know we can safely convert to a byte array");
        Blake2bHash::new(digest)
    }
}

impl AsRef<[u8]> for Blake2bHash {
    fn as_ref(&self) -> &[u8] { &self.digest }
}

impl PartialEq for Blake2bHash {
    /// Implement secure equality comparison for our hashes here so we
    /// don't have to think about it elsewhere.
    // Strictly speaking, secure equality comparison shouldn't be
    // necessary; however, having a type encapsulate this gives us a
    // convenient way to modify this in the future.
    fn eq(&self, other: &Blake2bHash) -> bool { crate::crypto::secure_eq(self, other) }
}

impl Eq for Blake2bHash {}

impl fmt::Display for Blake2bHash {
    /// Displays a Blake2bHash as a lowercase hex-encoded string.
    ///
    /// Due to historical precedent, the lowercasing *is* significant,
    /// as we sign the lowercase hex-encoded version of a Blake2b
    /// hash, and not simply the Blake2b hash itself, when we sign a
    /// HART file.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // hex::encode currently outputs lowercase strings, but we
        // want to strictly enforce this and guard against any future
        // changes to that crate.
        hex::encode(self).to_lowercase().fmt(f)
    }
}

impl FromStr for Blake2bHash {
    type Err = Error;

    /// Convert a hex-encoded byte string into a Blake2bHash. Ensures
    /// the string represents a byte array of proper length.
    ///
    /// Case of the incoming string is not significant (e.g.,
    /// "DEADBEEF" and "deadbeef" are equivalent).
    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        // FromHex has an implementation for [u8; 32], so this ensures
        // the proper length of bytes... well, that and the compiler,
        // of course :)
        FromHex::from_hex(s).map(Self::new).map_err(|e| {
                                               Error::CryptoError(format!("Could not parse \
                                                                           Blake2bHash from \
                                                                           string: {}",
                                                                          e))
                                           })
    }
}

impl Serialize for Blake2bHash {
    /// Serializes a `Blake2bHash` according to its `Display`
    /// implementation (i.e., a lowercase hex-encoded string).
    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
        where S: serde::Serializer
    {
        serializer.serialize_str(&self.to_string())
    }
}

/// Temporary implementation to ease adoption in Builder. Once that's
/// been updated, remove this (and the `hex_string` field).
impl std::ops::Deref for Blake2bHash {
    type Target = str;

    fn deref(&self) -> &Self::Target { self.hex_string.as_str() }
}

////////////////////////////////////////////////////////////////////////

/// Initialize the hasher state. In particular, set the digest length
/// to 32 bytes. All hashing functions must use this to ensure
/// consistency!
fn hash_state() -> State {
    let mut params = Params::new();
    params.hash_length(HASH_DIGEST_SIZE);
    params.to_state()
}

/// Calculate the BLAKE2b hash of a file.
/// NOTE: the hashing is keyless
pub fn hash_file<P>(filename: P) -> Result<Blake2bHash>
    where P: AsRef<Path>
{
    let file = File::open(filename.as_ref())?;
    let mut reader = BufReader::new(file);
    hash_reader(&mut reader)
}

pub fn hash_bytes<B>(data: B) -> Blake2bHash
    where B: AsRef<[u8]>
{
    let mut state = hash_state();
    state.update(data.as_ref());
    state.finalize().into()
}

pub fn hash_reader(reader: &mut dyn Read) -> Result<Blake2bHash> {
    let mut state = hash_state();

    let mut buf = [0u8; BUF_SIZE];
    loop {
        let bytes_read = reader.read(&mut buf)?;
        if bytes_read == 0 {
            break;
        }
        let chunk = &buf[0..bytes_read];
        state.update(chunk);
    }

    Ok(state.finalize().into())
}

#[cfg(test)]
mod test {
    #[allow(unused_imports)]
    use std::fs::{self,
                  File};
    #[allow(unused_imports)]
    use std::io;

    use super::{super::test_support::*,
                *};

    /// Helper function to create a Blake2bHash directly from a
    /// hex-encoded string.
    fn hash_from_hex(hex: &str) -> Blake2bHash { hex.parse().unwrap() }

    #[test]
    fn hash_file_working() {
        // The expected values were computed using the `b2sum` program from
        // https://github.com/dchest/b2sum using the the `-s=32` option. For example:
        //      b2sum -s=32 signme.dat

        let computed = hash_file(&fixture("signme.dat")).unwrap();
        let expected =
            hash_from_hex("20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233");
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.sig.key")).unwrap();
        let expected =
            hash_from_hex("e966844bbc50b7a3a6d81e94dd38e27b92814b944095a8e55f1780921bfcfbe1");
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.pub")).unwrap();
        let expected =
            hash_from_hex("b80c4f412f9a0a7727b6e6f115e1b5fa3bae79ad2fcf47f769ed4e42cfb12265");
        assert_eq!(computed, expected);
    }

    #[test]
    fn strings_can_be_hashed() {
        let message = "supercalifragilisticexpialadocious";
        let expected =
            hash_from_hex("2ca8ebafca7e189de2a36125b92a1db20f393d1e2708f5daa55e51cf05114437");
        let actual = hash_bytes(message);

        assert_eq!(expected, actual);
    }

    #[test]
    fn bytes_can_be_hashed() {
        let message = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8];
        let expected =
            hash_from_hex("8b57a796a5d07cb04cc1614dfc2acb3f73edc712d7f433619ca3bbe66bb15f49");
        let actual = hash_bytes(&message);

        assert_eq!(expected, actual);
    }

    #[test]
    #[cfg(feature = "functional")]
    fn hash_file_large_binary() {
        let dir = tempfile::Builder::new().prefix("large_file")
                                          .tempdir()
                                          .unwrap();

        let url = "http://www.kernel.org/pub/linux/kernel/v4.x/linux-4.3.tar.gz";
        let file = dir.path().join(url.rsplit('/').next().unwrap());

        // Download the doc to the temp directory
        let mut f = File::create(&file).unwrap();
        let mut res = reqwest::blocking::get(url).unwrap();
        res.copy_to(&mut f).unwrap();

        let computed = Blake2bHash::from_file(&file).unwrap();
        let expected =
            hash_from_hex("ba640dc063f0ed27e60b38dbb7cf19778cf7805d9fc91eb129fb68b409d46414");
        assert_eq!(computed, expected);
    }

    mod blake2bhash {
        use super::*;

        #[test]
        fn eq() {
            let zeroes = Blake2bHash::new([0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
                                           0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
                                           0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]);

            let ones = Blake2bHash::new([1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8]);

            assert_ne!(zeroes, ones);
            assert_eq!(zeroes, zeroes);
            assert_eq!(ones, ones);
        }

        #[test]
        fn display() {
            let ones = Blake2bHash::new([1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8]);
            assert_eq!(ones.to_string(),
                       "0101010101010101010101010101010101010101010101010101010101010101");
        }

        #[test]
        fn from_str_good() {
            // Exactly 32 bytes long
            let h = "0101010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_ok());
        }

        #[test]
        fn from_str_short() {
            // This is one byte too short
            let h = "01010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_err());
        }
        #[test]
        fn from_str_long() {
            // This is one byte too long
            let h = "010101010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_err());
        }

        #[test]
        fn from_str_case_is_insignificant() {
            let lower_case = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let upper_case = "20590A52C4F00588C500328B16D466C982A26FABAA5FA4DCC83052DD0A84F233";

            let l = lower_case.parse::<Blake2bHash>().unwrap();
            let u = upper_case.parse::<Blake2bHash>().unwrap();

            assert_eq!(l, u);
        }

        #[test]
        fn from_str_display_round_trip() {
            let input = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let output = input.parse::<Blake2bHash>().unwrap().to_string();
            assert_eq!(output, input);
        }

        #[test]
        fn serialize_as_hex_encoding() {
            let input = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let hash: Blake2bHash = input.parse().unwrap();
            serde_test::assert_ser_tokens(&hash, &[serde_test::Token::Str(input)]);
        }
    }
}
<commit_msg>[REFACTOR] Delete Deref impl for Blake2bHash<commit_after>use crate::error::{Error,
                   Result};
use blake2b_simd::{Params,
                   State};
use hex::FromHex;
use serde::Serialize;
use std::{convert::TryInto,
          fmt,
          fs::File,
          io::{BufReader,
               Read},
          path::Path,
          str::FromStr};

/// When hashing byte streams, we'll read 1KB at a time, adding this to the
/// internal hashing state as we compute the final digest.
const BUF_SIZE: usize = 1024;

/// The size of our Blake2b hash digests (32 bytes)
const HASH_DIGEST_SIZE: usize = 32;

/// Convenience wrapper type for a 32-byte Blake2b hash digest.
///
/// Implements secure equality comparison, as well as hex-encoding via
/// `std::fmt::Display`.
///
/// There are intentionally no explicit constructors; to get a
/// instance, you'll need to either convert it directly from a `&[u8]`
/// or parse it from a hex string.
#[derive(Clone, Debug)]
pub struct Blake2bHash {
    digest:     [u8; HASH_DIGEST_SIZE],
    /// Temporary field to support Deref<str> for backwards
    /// compatibility with Builder until it can use the new types.
    hex_string: String,
}

impl Blake2bHash {
    /// Temporary constructor while we store the hex encoding in the
    /// type directly.
    fn new(digest: [u8; HASH_DIGEST_SIZE]) -> Self {
        let hex_string = hex::encode(&digest).to_lowercase();
        Blake2bHash { digest, hex_string }
    }
}

impl From<blake2b_simd::Hash> for Blake2bHash {
    fn from(src: blake2b_simd::Hash) -> Self {
        let digest = src.as_bytes()
                        .try_into()
                        .expect("We know we can safely convert to a byte array");
        Blake2bHash::new(digest)
    }
}

impl AsRef<[u8]> for Blake2bHash {
    fn as_ref(&self) -> &[u8] { &self.digest }
}

impl PartialEq for Blake2bHash {
    /// Implement secure equality comparison for our hashes here so we
    /// don't have to think about it elsewhere.
    // Strictly speaking, secure equality comparison shouldn't be
    // necessary; however, having a type encapsulate this gives us a
    // convenient way to modify this in the future.
    fn eq(&self, other: &Blake2bHash) -> bool { crate::crypto::secure_eq(self, other) }
}

impl Eq for Blake2bHash {}

impl fmt::Display for Blake2bHash {
    /// Displays a Blake2bHash as a lowercase hex-encoded string.
    ///
    /// Due to historical precedent, the lowercasing *is* significant,
    /// as we sign the lowercase hex-encoded version of a Blake2b
    /// hash, and not simply the Blake2b hash itself, when we sign a
    /// HART file.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // hex::encode currently outputs lowercase strings, but we
        // want to strictly enforce this and guard against any future
        // changes to that crate.
        hex::encode(self).to_lowercase().fmt(f)
    }
}

impl FromStr for Blake2bHash {
    type Err = Error;

    /// Convert a hex-encoded byte string into a Blake2bHash. Ensures
    /// the string represents a byte array of proper length.
    ///
    /// Case of the incoming string is not significant (e.g.,
    /// "DEADBEEF" and "deadbeef" are equivalent).
    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        // FromHex has an implementation for [u8; 32], so this ensures
        // the proper length of bytes... well, that and the compiler,
        // of course :)
        FromHex::from_hex(s).map(Self::new).map_err(|e| {
                                               Error::CryptoError(format!("Could not parse \
                                                                           Blake2bHash from \
                                                                           string: {}",
                                                                          e))
                                           })
    }
}

impl Serialize for Blake2bHash {
    /// Serializes a `Blake2bHash` according to its `Display`
    /// implementation (i.e., a lowercase hex-encoded string).
    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
        where S: serde::Serializer
    {
        serializer.serialize_str(&self.to_string())
    }
}

////////////////////////////////////////////////////////////////////////

/// Initialize the hasher state. In particular, set the digest length
/// to 32 bytes. All hashing functions must use this to ensure
/// consistency!
fn hash_state() -> State {
    let mut params = Params::new();
    params.hash_length(HASH_DIGEST_SIZE);
    params.to_state()
}

/// Calculate the BLAKE2b hash of a file.
/// NOTE: the hashing is keyless
pub fn hash_file<P>(filename: P) -> Result<Blake2bHash>
    where P: AsRef<Path>
{
    let file = File::open(filename.as_ref())?;
    let mut reader = BufReader::new(file);
    hash_reader(&mut reader)
}

pub fn hash_bytes<B>(data: B) -> Blake2bHash
    where B: AsRef<[u8]>
{
    let mut state = hash_state();
    state.update(data.as_ref());
    state.finalize().into()
}

pub fn hash_reader(reader: &mut dyn Read) -> Result<Blake2bHash> {
    let mut state = hash_state();

    let mut buf = [0u8; BUF_SIZE];
    loop {
        let bytes_read = reader.read(&mut buf)?;
        if bytes_read == 0 {
            break;
        }
        let chunk = &buf[0..bytes_read];
        state.update(chunk);
    }

    Ok(state.finalize().into())
}

#[cfg(test)]
mod test {
    #[allow(unused_imports)]
    use std::fs::{self,
                  File};
    #[allow(unused_imports)]
    use std::io;

    use super::{super::test_support::*,
                *};

    /// Helper function to create a Blake2bHash directly from a
    /// hex-encoded string.
    fn hash_from_hex(hex: &str) -> Blake2bHash { hex.parse().unwrap() }

    #[test]
    fn hash_file_working() {
        // The expected values were computed using the `b2sum` program from
        // https://github.com/dchest/b2sum using the the `-s=32` option. For example:
        //      b2sum -s=32 signme.dat

        let computed = hash_file(&fixture("signme.dat")).unwrap();
        let expected =
            hash_from_hex("20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233");
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.sig.key")).unwrap();
        let expected =
            hash_from_hex("e966844bbc50b7a3a6d81e94dd38e27b92814b944095a8e55f1780921bfcfbe1");
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.pub")).unwrap();
        let expected =
            hash_from_hex("b80c4f412f9a0a7727b6e6f115e1b5fa3bae79ad2fcf47f769ed4e42cfb12265");
        assert_eq!(computed, expected);
    }

    #[test]
    fn strings_can_be_hashed() {
        let message = "supercalifragilisticexpialadocious";
        let expected =
            hash_from_hex("2ca8ebafca7e189de2a36125b92a1db20f393d1e2708f5daa55e51cf05114437");
        let actual = hash_bytes(message);

        assert_eq!(expected, actual);
    }

    #[test]
    fn bytes_can_be_hashed() {
        let message = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, 8u8, 9u8];
        let expected =
            hash_from_hex("8b57a796a5d07cb04cc1614dfc2acb3f73edc712d7f433619ca3bbe66bb15f49");
        let actual = hash_bytes(&message);

        assert_eq!(expected, actual);
    }

    #[test]
    #[cfg(feature = "functional")]
    fn hash_file_large_binary() {
        let dir = tempfile::Builder::new().prefix("large_file")
                                          .tempdir()
                                          .unwrap();

        let url = "http://www.kernel.org/pub/linux/kernel/v4.x/linux-4.3.tar.gz";
        let file = dir.path().join(url.rsplit('/').next().unwrap());

        // Download the doc to the temp directory
        let mut f = File::create(&file).unwrap();
        let mut res = reqwest::blocking::get(url).unwrap();
        res.copy_to(&mut f).unwrap();

        let computed = Blake2bHash::from_file(&file).unwrap();
        let expected =
            hash_from_hex("ba640dc063f0ed27e60b38dbb7cf19778cf7805d9fc91eb129fb68b409d46414");
        assert_eq!(computed, expected);
    }

    mod blake2bhash {
        use super::*;

        #[test]
        fn eq() {
            let zeroes = Blake2bHash::new([0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
                                           0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
                                           0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]);

            let ones = Blake2bHash::new([1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8]);

            assert_ne!(zeroes, ones);
            assert_eq!(zeroes, zeroes);
            assert_eq!(ones, ones);
        }

        #[test]
        fn display() {
            let ones = Blake2bHash::new([1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8,
                                         1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8, 1u8]);
            assert_eq!(ones.to_string(),
                       "0101010101010101010101010101010101010101010101010101010101010101");
        }

        #[test]
        fn from_str_good() {
            // Exactly 32 bytes long
            let h = "0101010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_ok());
        }

        #[test]
        fn from_str_short() {
            // This is one byte too short
            let h = "01010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_err());
        }
        #[test]
        fn from_str_long() {
            // This is one byte too long
            let h = "010101010101010101010101010101010101010101010101010101010101010101".parse::<Blake2bHash>();
            assert!(h.is_err());
        }

        #[test]
        fn from_str_case_is_insignificant() {
            let lower_case = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let upper_case = "20590A52C4F00588C500328B16D466C982A26FABAA5FA4DCC83052DD0A84F233";

            let l = lower_case.parse::<Blake2bHash>().unwrap();
            let u = upper_case.parse::<Blake2bHash>().unwrap();

            assert_eq!(l, u);
        }

        #[test]
        fn from_str_display_round_trip() {
            let input = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let output = input.parse::<Blake2bHash>().unwrap().to_string();
            assert_eq!(output, input);
        }

        #[test]
        fn serialize_as_hex_encoding() {
            let input = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
            let hash: Blake2bHash = input.parse().unwrap();
            serde_test::assert_ser_tokens(&hash, &[serde_test::Token::Str(input)]);
        }
    }
}
<|endoftext|>


<commit_before>#![allow(missing_docs)]

use std::fmt;
use elements::{BlockType, External, GlobalEntry, GlobalType, Internal, MemoryType,
               Module, Opcode, ResizableLimits, TableType, ValueType, InitExpr};
use common::stack;
use self::context::ModuleContext;
use self::func::Validator;

pub use self::module::ValidatedModule;

mod context;
mod module;
mod func;

#[cfg(test)]
mod tests;

pub struct Error(String);

impl fmt::Display for Error {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		write!(f, "{}", self.0)
	}
}

impl From<stack::Error> for Error {
	fn from(e: stack::Error) -> Error {
		Error(format!("Stack: {}", e))
	}
}

pub fn validate_module(module: &Module) -> Result<ValidatedModule, Error> {
	let context = prepare_context(module)?;

	let function_section_len = module
		.function_section()
		.map(|s| s.entries().len())
		.unwrap_or(0);
	let code_section_len = module.code_section().map(|s| s.bodies().len()).unwrap_or(0);
	if function_section_len != code_section_len {
		return Err(Error(format!(
			"length of function section is {}, while len of code section is {}",
			function_section_len,
			code_section_len
		)));
	}

	// validate every function body in user modules
	if function_section_len != 0 {
		// tests use invalid code
		let function_section = module
			.function_section()
			.expect("function_section_len != 0; qed");
		let code_section = module
			.code_section()
			.expect("function_section_len != 0; function_section_len == code_section_len; qed");
		// check every function body
		for (index, function) in function_section.entries().iter().enumerate() {
			let function_body = code_section
				.bodies()
				.get(index as usize)
				.ok_or(Error(format!("Missing body for function {}", index)))?;
			Validator::validate_function(&context, function, function_body).map_err(|e| {
				let Error(ref msg) = e;
				Error(format!("Function #{} validation error: {}", index, msg))
			})?;
		}
	}

	// validate start section
	if let Some(start_function) = module.start_section() {
		let (params, return_ty) = context.require_function(start_function)?;
		if return_ty != BlockType::NoResult || params.len() != 0 {
			return Err(Error(
				"start function expected to have type [] -> []".into(),
			));
		}
	}

	// validate export section
	if let Some(export_section) = module.export_section() {
		for export in export_section.entries() {
			match export.internal() {
				&Internal::Function(function_index) => {
					context.require_function(function_index)?;
				}
				&Internal::Global(global_index) => {
					context.require_global(global_index, Some(false))?;
				}
				&Internal::Memory(memory_index) => {
					context.require_memory(memory_index)?;
				}
				&Internal::Table(table_index) => {
					context.require_table(table_index)?;
				}
			}
		}
	}

	// validate import section
	if let Some(import_section) = module.import_section() {
		for import in import_section.entries() {
			match import.external() {
				&External::Function(function_type_index) => {
					context.require_function(function_type_index)?;
				},
				&External::Global(ref global_type) => {
					if global_type.is_mutable() {
						return Err(Error(format!("trying to import mutable global {}", import.field())));
					}
				},
				&External::Memory(ref memory_type) => {
					memory_type.validate()?;
				},
				&External::Table(ref table_type) => {
					table_type.validate()?;
				},
			}
		}
	}

	// there must be no greater than 1 table in tables index space
	if context.tables().len() > 1 {
		return Err(Error(format!("too many tables in index space: {}", context.tables().len())));
	}

	// there must be no greater than 1 linear memory in memory index space
	if context.memories().len() > 1 {
		return Err(Error(format!("too many memory regions in index space: {}", context.memories().len())));
	}

	// use data section to initialize linear memory regions
	if let Some(data_section) = module.data_section() {
		for data_segment in data_section.entries() {
			context.require_memory(data_segment.index())?;
			let init_ty = data_segment.offset().expr_const_type(context.globals())?;
			if init_ty != ValueType::I32 {
				return Err(Error(format!("segment offset should return I32")));
			}
		}
	}

	// use element section to fill tables
	if let Some(element_section) = module.elements_section() {
		for element_segment in element_section.entries() {
			context.require_table(element_segment.index())?;

			let init_ty = element_segment.offset().expr_const_type(context.globals())?;
			if init_ty != ValueType::I32 {
				return Err(Error(format!("segment offset should return I32")));
			}

			for function_index in element_segment.members() {
				context.require_function(*function_index)?;
			}
		}
	}

	let ModuleContext {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	} = context;

	Ok(ValidatedModule {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	})
}

fn prepare_context(module: &Module) -> Result<ModuleContext, Error> {
	// Copy types from module as is.
	let types = module
		.type_section()
		.map(|ts| ts.types().into_iter().cloned().collect())
		.unwrap_or_default();

	// Fill elements with imported values.

	// TODO: Use Func::type_ref?
	let mut func_type_indexes = Vec::new();
	let mut tables = Vec::new();
	let mut memories = Vec::new();
	let mut globals = Vec::new();

	for import_entry in module
		.import_section()
		.map(|i| i.entries())
		.unwrap_or_default()
	{
		match import_entry.external() {
			&External::Function(idx) => func_type_indexes.push(idx),
			&External::Table(ref table) => tables.push(table.clone()),
			&External::Memory(ref memory) => memories.push(memory.clone()),
			&External::Global(ref global) => globals.push(global.clone()),
		}
	}

	// Concatenate elements with defined in the module.
	if let Some(function_section) = module.function_section() {
		for func_entry in function_section.entries() {
			func_type_indexes.push(func_entry.type_ref());
		}
	}
	if let Some(table_section) = module.table_section() {
		for table_entry in table_section.entries() {
			table_entry.validate()?;
			tables.push(table_entry.clone());
		}
	}
	if let Some(mem_section) = module.memory_section() {
		for mem_entry in mem_section.entries() {
			mem_entry.validate()?;
			memories.push(mem_entry.clone());
		}
	}
	if let Some(global_section) = module.global_section() {
		// Validation of globals is defined over modified context C', which
		// contains only imported globals. So we do globals validation
		// in two passes, in first we validate globals and after all globals are validated
		// add them in globals list.
		for global_entry in global_section.entries() {
			global_entry.validate(&globals)?;
		}

		for global_entry in global_section.entries() {
			globals.push(global_entry.global_type().clone());
		}
	}

	Ok(ModuleContext {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	})
}

impl ResizableLimits {
	fn validate(&self) -> Result<(), Error> {
		if let Some(maximum) = self.maximum() {
			if self.initial() > maximum {
				return Err(Error(format!(
					"maximum limit {} is lesser than minimum {}",
					maximum,
					self.initial()
				)));
			}
		}
		Ok(())
	}
}

impl MemoryType {
	fn validate(&self) -> Result<(), Error> {
		self.limits().validate()
	}
}

impl TableType {
	fn validate(&self) -> Result<(), Error> {
		self.limits().validate()
	}
}

impl GlobalEntry {
	fn validate(&self, globals: &[GlobalType]) -> Result<(), Error> {
		let init = self.init_expr();
		let init_expr_ty = init.expr_const_type(globals)?;
		if init_expr_ty != self.global_type().content_type() {
			return Err(Error(format!(
				"Trying to initialize variable of type {:?} with value of type {:?}",
				self.global_type().content_type(),
				init_expr_ty
			)));
		}
		Ok(())
	}
}

impl InitExpr {
	/// Returns type of this constant expression.
	fn expr_const_type(&self, globals: &[GlobalType]) -> Result<ValueType, Error> {
		let code = self.code();
		if code.len() != 2 {
			return Err(Error(
				format!("Init expression should always be with length 2")
			));
		}
		let expr_ty: ValueType = match code[0] {
			Opcode::I32Const(_) => ValueType::I32,
			Opcode::I64Const(_) => ValueType::I64,
			Opcode::F32Const(_) => ValueType::F32,
			Opcode::F64Const(_) => ValueType::F64,
			Opcode::GetGlobal(idx) => match globals.get(idx as usize) {
				Some(target_global) => {
					if target_global.is_mutable() {
						return Err(Error(format!("Global {} is mutable", idx)));
					}
					target_global.content_type()
				}
				None => {
					return Err(Error(
						format!("Global {} doesn't exists or not yet defined", idx),
					))
				}
			},
			_ => return Err(Error(format!("Non constant opcode in init expr"))),
		};
		if code[1] != Opcode::End {
			return Err(Error(format!("Expression doesn't ends with `end` opcode")));
		}
		Ok(expr_ty)
	}
}
<commit_msg>Clean<commit_after>#![allow(missing_docs)]

use std::fmt;
use elements::{BlockType, External, GlobalEntry, GlobalType, Internal, MemoryType,
               Module, Opcode, ResizableLimits, TableType, ValueType, InitExpr};
use common::stack;
use self::context::ModuleContext;
use self::func::Validator;

pub use self::module::ValidatedModule;

mod context;
mod module;
mod func;

#[cfg(test)]
mod tests;

pub struct Error(String);

impl fmt::Display for Error {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		write!(f, "{}", self.0)
	}
}

impl From<stack::Error> for Error {
	fn from(e: stack::Error) -> Error {
		Error(format!("Stack: {}", e))
	}
}

pub fn validate_module(module: &Module) -> Result<ValidatedModule, Error> {
	let context = prepare_context(module)?;

	let function_section_len = module
		.function_section()
		.map(|s| s.entries().len())
		.unwrap_or(0);
	let code_section_len = module.code_section().map(|s| s.bodies().len()).unwrap_or(0);
	if function_section_len != code_section_len {
		return Err(Error(format!(
			"length of function section is {}, while len of code section is {}",
			function_section_len,
			code_section_len
		)));
	}

	// validate every function body in user modules
	if function_section_len != 0 {
		// tests use invalid code
		let function_section = module
			.function_section()
			.expect("function_section_len != 0; qed");
		let code_section = module
			.code_section()
			.expect("function_section_len != 0; function_section_len == code_section_len; qed");
		// check every function body
		for (index, function) in function_section.entries().iter().enumerate() {
			let function_body = code_section
				.bodies()
				.get(index as usize)
				.ok_or(Error(format!("Missing body for function {}", index)))?;
			Validator::validate_function(&context, function, function_body).map_err(|e| {
				let Error(ref msg) = e;
				Error(format!("Function #{} validation error: {}", index, msg))
			})?;
		}
	}

	// validate start section
	if let Some(start_function) = module.start_section() {
		let (params, return_ty) = context.require_function(start_function)?;
		if return_ty != BlockType::NoResult || params.len() != 0 {
			return Err(Error(
				"start function expected to have type [] -> []".into(),
			));
		}
	}

	// validate export section
	if let Some(export_section) = module.export_section() {
		for export in export_section.entries() {
			match *export.internal() {
				Internal::Function(function_index) => {
					context.require_function(function_index)?;
				}
				Internal::Global(global_index) => {
					context.require_global(global_index, Some(false))?;
				}
				Internal::Memory(memory_index) => {
					context.require_memory(memory_index)?;
				}
				Internal::Table(table_index) => {
					context.require_table(table_index)?;
				}
			}
		}
	}

	// validate import section
	if let Some(import_section) = module.import_section() {
		for import in import_section.entries() {
			match *import.external() {
				External::Function(function_type_index) => {
					context.require_function(function_type_index)?;
				},
				External::Global(ref global_type) => {
					if global_type.is_mutable() {
						return Err(Error(format!("trying to import mutable global {}", import.field())));
					}
				},
				External::Memory(ref memory_type) => {
					memory_type.validate()?;
				},
				External::Table(ref table_type) => {
					table_type.validate()?;
				},
			}
		}
	}

	// there must be no greater than 1 table in tables index space
	if context.tables().len() > 1 {
		return Err(Error(format!("too many tables in index space: {}", context.tables().len())));
	}

	// there must be no greater than 1 linear memory in memory index space
	if context.memories().len() > 1 {
		return Err(Error(format!("too many memory regions in index space: {}", context.memories().len())));
	}

	// use data section to initialize linear memory regions
	if let Some(data_section) = module.data_section() {
		for data_segment in data_section.entries() {
			context.require_memory(data_segment.index())?;
			let init_ty = data_segment.offset().expr_const_type(context.globals())?;
			if init_ty != ValueType::I32 {
				return Err(Error("segment offset should return I32".into()));
			}
		}
	}

	// use element section to fill tables
	if let Some(element_section) = module.elements_section() {
		for element_segment in element_section.entries() {
			context.require_table(element_segment.index())?;

			let init_ty = element_segment.offset().expr_const_type(context.globals())?;
			if init_ty != ValueType::I32 {
				return Err(Error("segment offset should return I32".into()));
			}

			for function_index in element_segment.members() {
				context.require_function(*function_index)?;
			}
		}
	}

	let ModuleContext {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	} = context;

	Ok(ValidatedModule {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	})
}

fn prepare_context(module: &Module) -> Result<ModuleContext, Error> {
	// Copy types from module as is.
	let types = module
		.type_section()
		.map(|ts| ts.types().into_iter().cloned().collect())
		.unwrap_or_default();

	// Fill elements with imported values.

	// TODO: Use Func::type_ref?
	let mut func_type_indexes = Vec::new();
	let mut tables = Vec::new();
	let mut memories = Vec::new();
	let mut globals = Vec::new();

	for import_entry in module
		.import_section()
		.map(|i| i.entries())
		.unwrap_or_default()
	{
		match import_entry.external() {
			&External::Function(idx) => func_type_indexes.push(idx),
			&External::Table(ref table) => tables.push(table.clone()),
			&External::Memory(ref memory) => memories.push(memory.clone()),
			&External::Global(ref global) => globals.push(global.clone()),
		}
	}

	// Concatenate elements with defined in the module.
	if let Some(function_section) = module.function_section() {
		for func_entry in function_section.entries() {
			func_type_indexes.push(func_entry.type_ref());
		}
	}
	if let Some(table_section) = module.table_section() {
		for table_entry in table_section.entries() {
			table_entry.validate()?;
			tables.push(table_entry.clone());
		}
	}
	if let Some(mem_section) = module.memory_section() {
		for mem_entry in mem_section.entries() {
			mem_entry.validate()?;
			memories.push(mem_entry.clone());
		}
	}
	if let Some(global_section) = module.global_section() {
		// Validation of globals is defined over modified context C', which
		// contains only imported globals. So we do globals validation
		// in two passes, in first we validate globals and after all globals are validated
		// add them in globals list.
		for global_entry in global_section.entries() {
			global_entry.validate(&globals)?;
		}

		for global_entry in global_section.entries() {
			globals.push(global_entry.global_type().clone());
		}
	}

	Ok(ModuleContext {
		types,
		tables,
		memories,
		globals,
		func_type_indexes,
	})
}

impl ResizableLimits {
	fn validate(&self) -> Result<(), Error> {
		if let Some(maximum) = self.maximum() {
			if self.initial() > maximum {
				return Err(Error(format!(
					"maximum limit {} is lesser than minimum {}",
					maximum,
					self.initial()
				)));
			}
		}
		Ok(())
	}
}

impl MemoryType {
	fn validate(&self) -> Result<(), Error> {
		self.limits().validate()
	}
}

impl TableType {
	fn validate(&self) -> Result<(), Error> {
		self.limits().validate()
	}
}

impl GlobalEntry {
	fn validate(&self, globals: &[GlobalType]) -> Result<(), Error> {
		let init = self.init_expr();
		let init_expr_ty = init.expr_const_type(globals)?;
		if init_expr_ty != self.global_type().content_type() {
			return Err(Error(format!(
				"Trying to initialize variable of type {:?} with value of type {:?}",
				self.global_type().content_type(),
				init_expr_ty
			)));
		}
		Ok(())
	}
}

impl InitExpr {
	/// Returns type of this constant expression.
	fn expr_const_type(&self, globals: &[GlobalType]) -> Result<ValueType, Error> {
		let code = self.code();
		if code.len() != 2 {
			return Err(Error("Init expression should always be with length 2".into()));
		}
		let expr_ty: ValueType = match code[0] {
			Opcode::I32Const(_) => ValueType::I32,
			Opcode::I64Const(_) => ValueType::I64,
			Opcode::F32Const(_) => ValueType::F32,
			Opcode::F64Const(_) => ValueType::F64,
			Opcode::GetGlobal(idx) => match globals.get(idx as usize) {
				Some(target_global) => {
					if target_global.is_mutable() {
						return Err(Error(format!("Global {} is mutable", idx)));
					}
					target_global.content_type()
				}
				None => {
					return Err(Error(
						format!("Global {} doesn't exists or not yet defined", idx),
					))
				}
			},
			_ => return Err(Error("Non constant opcode in init expr".into())),
		};
		if code[1] != Opcode::End {
			return Err(Error("Expression doesn't ends with `end` opcode".into()));
		}
		Ok(expr_ty)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{borrow::Cow, cell::Cell, mem};

use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use enum_primitive_derive::Primitive;
use failure::{self, ensure, format_err};
use num_traits::FromPrimitive;
use serde_derive::{Deserialize, Serialize};

use crate::{BinaryKey, BinaryValue};

use super::{IndexAccess, IndexAddress, View};

/// TODO Add documentation. [ECR-2820]
const INDEXES_POOL_NAME: &str = "__INDEXES_POOL__";

/// TODO Add documentation. [ECR-2820]
#[derive(Debug, Copy, Clone, PartialEq, Primitive, Serialize, Deserialize)]
#[repr(u32)]
pub enum IndexType {
    Map = 1,
    List = 2,
    Entry = 3,
    ValueSet = 4,
    KeySet = 5,
    SparseList = 6,
    ProofList = 7,
    ProofMap = 8,
    Unknown = 255,
}

/// Index state attribute tag.
const INDEX_STATE_TAG: u32 = 0;
/// Separator between the name and the additional bytes in family indexes.
const INDEX_NAME_SEPARATOR: &[u8] = &[0];

/// TODO Add documentation. [ECR-2820]
pub trait BinaryAttribute {
    /// TODO Add documentation. [ECR-2820]
    fn size(&self) -> usize;
    /// TODO Add documentation. [ECR-2820]
    fn write<W: std::io::Write>(&self, buffer: &mut W);
    /// TODO Add documentation. [ECR-2820]
    fn read<R: std::io::Read>(buffer: &mut R) -> Self;
}

/// No-op implementation.
impl BinaryAttribute for () {
    fn size(&self) -> usize {
        0
    }

    fn write<W: std::io::Write>(&self, _buffer: &mut W) {}

    fn read<R: std::io::Read>(_buffer: &mut R) -> Self {}
}

impl BinaryAttribute for u64 {
    fn size(&self) -> usize {
        mem::size_of_val(self)
    }

    fn write<W: std::io::Write>(&self, buffer: &mut W) {
        buffer.write_u64::<LittleEndian>(*self).unwrap()
    }

    fn read<R: std::io::Read>(buffer: &mut R) -> Self {
        buffer.read_u64::<LittleEndian>().unwrap()
    }
}

impl Default for IndexType {
    fn default() -> Self {
        IndexType::Unknown
    }
}

/// TODO Add documentation. [ECR-2820]
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct IndexMetadata<V> {
    identifier: u64,
    index_type: IndexType,
    state: V,
}

impl<V> BinaryValue for IndexMetadata<V>
where
    V: BinaryAttribute,
{
    fn to_bytes(&self) -> Vec<u8> {
        let state_len = self.state.size();
        let mut buf = Vec::with_capacity(
            mem::size_of_val(&self.identifier)
                + mem::size_of_val(&self.index_type)
                + mem::size_of::<u32>()
                + mem::size_of_val(&INDEX_STATE_TAG)
                + state_len,
        );

        buf.write_u64::<LittleEndian>(self.identifier).unwrap();
        buf.write_u32::<LittleEndian>(self.index_type as u32)
            .unwrap();
        // Writes indes state in TLV (tag, lenght, value) form.
        buf.write_u32::<LittleEndian>(INDEX_STATE_TAG).unwrap();
        buf.write_u32::<LittleEndian>(state_len as u32).unwrap();
        self.state.write(&mut buf);
        buf
    }

    fn from_bytes(bytes: Cow<[u8]>) -> Result<Self, failure::Error> {
        let mut bytes = bytes.as_ref();

        let identifier = bytes.read_u64::<LittleEndian>()?;
        let index_type = bytes.read_u32::<LittleEndian>()?;
        // Reads index state in TLV (tag, lenght, value) form.
        let state_tag = bytes.read_u32::<LittleEndian>()?;
        let state_len = bytes.read_u32::<LittleEndian>()? as usize;

        ensure!(
            state_tag == INDEX_STATE_TAG,
            "Attribute with unknown tag: {}",
            state_tag
        );
        ensure!(bytes.len() >= state_len, "Index state is too short");

        let mut state_bytes = &bytes[0..state_len];
        let state = V::read(&mut state_bytes);

        Ok(Self {
            identifier,
            index_type: IndexType::from_u32(index_type)
                .ok_or_else(|| format_err!("Unknown index type: {}", index_type))?,
            state,
        })
    }
}

impl<V> IndexMetadata<V> {
    fn index_address(&self) -> IndexAddress {
        IndexAddress::new().append_bytes(&self.identifier)
    }
}

impl IndexAddress {
    fn fully_qualified_name(&self) -> Vec<u8> {
        if let Some(bytes) = self.bytes() {
            concat_keys!(self.name(), INDEX_NAME_SEPARATOR, bytes)
        } else {
            concat_keys!(self.name())
        }
    }
}

/// TODO Add documentation. [ECR-2820]
pub fn index_metadata<T, V>(
    index_access: T,
    index_address: &IndexAddress,
    index_type: IndexType,
) -> (IndexAddress, IndexState<T, V>)
where
    T: IndexAccess,
    V: BinaryAttribute + Copy + Default,
{
    let index_name = index_address.fully_qualified_name();

    let mut pool = IndexesPool::new(index_access);
    let metadata = if let Some(metadata) = pool.index_metadata(&index_name) {
        assert_eq!(
            metadata.index_type, index_type,
            "Index type doesn't match specified"
        );
        metadata
    } else {
        pool.create_index_metadata(&index_name, index_type)
    };

    let index_address = metadata.index_address();
    let index_state = IndexState::new(index_access, index_name, metadata);
    (index_address, index_state)
}

/// TODO Add documentation. [ECR-2820]
struct IndexesPool<T: IndexAccess>(View<T>);

impl<T: IndexAccess> IndexesPool<T> {
    fn new(index_access: T) -> Self {
        let pool_address = IndexAddress::from(INDEXES_POOL_NAME);
        Self(View::new(index_access, pool_address))
    }

    fn len(&self) -> u64 {
        self.0.get(&()).unwrap_or_default()
    }

    fn set_len(&mut self, len: u64) {
        self.0.put(&(), len)
    }

    fn index_metadata<V>(&self, index_name: &[u8]) -> Option<IndexMetadata<V>>
    where
        V: BinaryAttribute + Default + Copy,
    {
        self.0.get(index_name)
    }

    fn create_index_metadata<V>(
        &mut self,
        index_name: &[u8],
        index_type: IndexType,
    ) -> IndexMetadata<V>
    where
        V: BinaryAttribute + Default + Copy,
    {
        let len = self.len();

        let metadata = IndexMetadata {
            index_type,
            identifier: len,
            state: V::default(),
        };

        self.0.put(index_name, metadata.to_bytes());
        self.set_len(len + 1);
        metadata
    }
}

/// TODO Add documentation. [ECR-2820]
pub struct IndexState<T, V>
where
    V: BinaryAttribute + Default + Copy,
    T: IndexAccess,
{
    index_access: T,
    index_name: Vec<u8>,
    cache: Cell<IndexMetadata<V>>,
}

impl<T, V> IndexState<T, V>
where
    V: BinaryAttribute + Default + Copy,
    T: IndexAccess,
{
    fn new(index_access: T, index_name: Vec<u8>, metadata: IndexMetadata<V>) -> Self {
        Self {
            index_access,
            index_name,
            cache: Cell::new(metadata),
        }
    }

    /// TODO Add documentation. [ECR-2820]
    pub fn get(&self) -> V {
        self.cache.get().state
    }

    /// TODO Add documentation. [ECR-2820]
    pub fn set(&mut self, state: V) {
        let mut cache = self.cache.get_mut();
        cache.state = state;
        View::new(self.index_access, IndexAddress::from(INDEXES_POOL_NAME))
            .put(&self.index_name, cache.to_bytes());
    }
}

impl<T, V> std::fmt::Debug for IndexState<T, V>
where
    T: IndexAccess,
    V: BinaryAttribute + Default + Copy,
{
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        f.debug_struct("IndexState").finish()
    }
}

#[cfg(test)]
mod tests {
    use std::io::Cursor;

    use crate::BinaryValue;

    use super::{BinaryAttribute, IndexMetadata, IndexType};

    #[test]
    fn test_binary_attribute_read_write() {
        let mut buf = Vec::new();
        11_u64.write(&mut buf);
        12_u64.write(&mut buf);
        assert_eq!(buf.len(), 16);

        let mut reader = Cursor::new(buf);
        let a = u64::read(&mut reader);
        let b = u64::read(&mut reader);
        assert_eq!(a, 11);
        assert_eq!(b, 12);
    }

    #[test]
    fn test_index_metadata_binary_value() {
        let metadata = IndexMetadata {
            identifier: 12,
            index_type: IndexType::ProofList,
            state: 16_u64,
        };

        let bytes = metadata.to_bytes();
        assert_eq!(IndexMetadata::from_bytes(bytes.into()).unwrap(), metadata);
    }

    #[test]
    #[should_panic(expected = "Attribute with unknown tag")]
    fn test_index_metadata_unknown_tag() {
        let metadata = IndexMetadata {
            identifier: 12,
            index_type: IndexType::ProofList,
            state: 16_u64,
        };

        let mut bytes = metadata.to_bytes();
        bytes[13] = 1; // Modifies index state tag.
        assert_eq!(IndexMetadata::from_bytes(bytes.into()).unwrap(), metadata);
    }
}
<commit_msg>fix lints<commit_after>// Copyright 2018 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{borrow::Cow, cell::Cell, mem};

use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use enum_primitive_derive::Primitive;
use failure::{self, ensure, format_err};
use num_traits::FromPrimitive;
use serde_derive::{Deserialize, Serialize};

use crate::{BinaryKey, BinaryValue};

use super::{IndexAccess, IndexAddress, View};

/// TODO Add documentation. [ECR-2820]
const INDEXES_POOL_NAME: &str = "__INDEXES_POOL__";

/// TODO Add documentation. [ECR-2820]
#[derive(Debug, Copy, Clone, PartialEq, Primitive, Serialize, Deserialize)]
#[repr(u32)]
pub enum IndexType {
    Map = 1,
    List = 2,
    Entry = 3,
    ValueSet = 4,
    KeySet = 5,
    SparseList = 6,
    ProofList = 7,
    ProofMap = 8,
    Unknown = 255,
}

/// Index state attribute tag.
const INDEX_STATE_TAG: u32 = 0;
/// Separator between the name and the additional bytes in family indexes.
const INDEX_NAME_SEPARATOR: &[u8] = &[0];

/// TODO Add documentation. [ECR-2820]
pub trait BinaryAttribute {
    /// TODO Add documentation. [ECR-2820]
    fn size(&self) -> usize;
    /// TODO Add documentation. [ECR-2820]
    fn write<W: std::io::Write>(&self, buffer: &mut W);
    /// TODO Add documentation. [ECR-2820]
    fn read<R: std::io::Read>(buffer: &mut R) -> Self;
}

/// No-op implementation.
impl BinaryAttribute for () {
    fn size(&self) -> usize {
        0
    }

    fn write<W: std::io::Write>(&self, _buffer: &mut W) {}

    fn read<R: std::io::Read>(_buffer: &mut R) -> Self {}
}

impl BinaryAttribute for u64 {
    fn size(&self) -> usize {
        mem::size_of_val(self)
    }

    fn write<W: std::io::Write>(&self, buffer: &mut W) {
        buffer.write_u64::<LittleEndian>(*self).unwrap()
    }

    fn read<R: std::io::Read>(buffer: &mut R) -> Self {
        buffer.read_u64::<LittleEndian>().unwrap()
    }
}

impl Default for IndexType {
    fn default() -> Self {
        IndexType::Unknown
    }
}

/// TODO Add documentation. [ECR-2820]
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct IndexMetadata<V> {
    identifier: u64,
    index_type: IndexType,
    state: V,
}

impl<V> BinaryValue for IndexMetadata<V>
where
    V: BinaryAttribute,
{
    fn to_bytes(&self) -> Vec<u8> {
        let state_len = self.state.size();
        let mut buf = Vec::with_capacity(
            mem::size_of_val(&self.identifier)
                + mem::size_of_val(&self.index_type)
                + mem::size_of::<u32>()
                + mem::size_of_val(&INDEX_STATE_TAG)
                + state_len,
        );

        buf.write_u64::<LittleEndian>(self.identifier).unwrap();
        buf.write_u32::<LittleEndian>(self.index_type as u32)
            .unwrap();
        // Writes index state in TLV (tag, length, value) form.
        buf.write_u32::<LittleEndian>(INDEX_STATE_TAG).unwrap();
        buf.write_u32::<LittleEndian>(state_len as u32).unwrap();
        self.state.write(&mut buf);
        buf
    }

    fn from_bytes(bytes: Cow<[u8]>) -> Result<Self, failure::Error> {
        let mut bytes = bytes.as_ref();

        let identifier = bytes.read_u64::<LittleEndian>()?;
        let index_type = bytes.read_u32::<LittleEndian>()?;
        // Reads index state in TLV (tag, length, value) form.
        let state_tag = bytes.read_u32::<LittleEndian>()?;
        let state_len = bytes.read_u32::<LittleEndian>()? as usize;

        ensure!(
            state_tag == INDEX_STATE_TAG,
            "Attribute with unknown tag: {}",
            state_tag
        );
        ensure!(bytes.len() >= state_len, "Index state is too short");

        let mut state_bytes = &bytes[0..state_len];
        let state = V::read(&mut state_bytes);

        Ok(Self {
            identifier,
            index_type: IndexType::from_u32(index_type)
                .ok_or_else(|| format_err!("Unknown index type: {}", index_type))?,
            state,
        })
    }
}

impl<V> IndexMetadata<V> {
    fn index_address(&self) -> IndexAddress {
        IndexAddress::new().append_bytes(&self.identifier)
    }
}

impl IndexAddress {
    fn fully_qualified_name(&self) -> Vec<u8> {
        if let Some(bytes) = self.bytes() {
            concat_keys!(self.name(), INDEX_NAME_SEPARATOR, bytes)
        } else {
            concat_keys!(self.name())
        }
    }
}

/// TODO Add documentation. [ECR-2820]
pub fn index_metadata<T, V>(
    index_access: T,
    index_address: &IndexAddress,
    index_type: IndexType,
) -> (IndexAddress, IndexState<T, V>)
where
    T: IndexAccess,
    V: BinaryAttribute + Copy + Default,
{
    let index_name = index_address.fully_qualified_name();

    let mut pool = IndexesPool::new(index_access);
    let metadata = if let Some(metadata) = pool.index_metadata(&index_name) {
        assert_eq!(
            metadata.index_type, index_type,
            "Index type doesn't match specified"
        );
        metadata
    } else {
        pool.create_index_metadata(&index_name, index_type)
    };

    let index_address = metadata.index_address();
    let index_state = IndexState::new(index_access, index_name, metadata);
    (index_address, index_state)
}

/// TODO Add documentation. [ECR-2820]
struct IndexesPool<T: IndexAccess>(View<T>);

impl<T: IndexAccess> IndexesPool<T> {
    fn new(index_access: T) -> Self {
        let pool_address = IndexAddress::from(INDEXES_POOL_NAME);
        Self(View::new(index_access, pool_address))
    }

    fn len(&self) -> u64 {
        self.0.get(&()).unwrap_or_default()
    }

    fn set_len(&mut self, len: u64) {
        self.0.put(&(), len)
    }

    fn index_metadata<V>(&self, index_name: &[u8]) -> Option<IndexMetadata<V>>
    where
        V: BinaryAttribute + Default + Copy,
    {
        self.0.get(index_name)
    }

    fn create_index_metadata<V>(
        &mut self,
        index_name: &[u8],
        index_type: IndexType,
    ) -> IndexMetadata<V>
    where
        V: BinaryAttribute + Default + Copy,
    {
        let len = self.len();

        let metadata = IndexMetadata {
            index_type,
            identifier: len,
            state: V::default(),
        };

        self.0.put(index_name, metadata.to_bytes());
        self.set_len(len + 1);
        metadata
    }
}

/// TODO Add documentation. [ECR-2820]
pub struct IndexState<T, V>
where
    V: BinaryAttribute + Default + Copy,
    T: IndexAccess,
{
    index_access: T,
    index_name: Vec<u8>,
    cache: Cell<IndexMetadata<V>>,
}

impl<T, V> IndexState<T, V>
where
    V: BinaryAttribute + Default + Copy,
    T: IndexAccess,
{
    fn new(index_access: T, index_name: Vec<u8>, metadata: IndexMetadata<V>) -> Self {
        Self {
            index_access,
            index_name,
            cache: Cell::new(metadata),
        }
    }

    /// TODO Add documentation. [ECR-2820]
    pub fn get(&self) -> V {
        self.cache.get().state
    }

    /// TODO Add documentation. [ECR-2820]
    pub fn set(&mut self, state: V) {
        let mut cache = self.cache.get_mut();
        cache.state = state;
        View::new(self.index_access, IndexAddress::from(INDEXES_POOL_NAME))
            .put(&self.index_name, cache.to_bytes());
    }
}

impl<T, V> std::fmt::Debug for IndexState<T, V>
where
    T: IndexAccess,
    V: BinaryAttribute + Default + Copy,
{
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        f.debug_struct("IndexState").finish()
    }
}

#[cfg(test)]
mod tests {
    use std::io::Cursor;

    use crate::BinaryValue;

    use super::{BinaryAttribute, IndexMetadata, IndexType};

    #[test]
    fn test_binary_attribute_read_write() {
        let mut buf = Vec::new();
        11_u64.write(&mut buf);
        12_u64.write(&mut buf);
        assert_eq!(buf.len(), 16);

        let mut reader = Cursor::new(buf);
        let a = u64::read(&mut reader);
        let b = u64::read(&mut reader);
        assert_eq!(a, 11);
        assert_eq!(b, 12);
    }

    #[test]
    fn test_index_metadata_binary_value() {
        let metadata = IndexMetadata {
            identifier: 12,
            index_type: IndexType::ProofList,
            state: 16_u64,
        };

        let bytes = metadata.to_bytes();
        assert_eq!(IndexMetadata::from_bytes(bytes.into()).unwrap(), metadata);
    }

    #[test]
    #[should_panic(expected = "Attribute with unknown tag")]
    fn test_index_metadata_unknown_tag() {
        let metadata = IndexMetadata {
            identifier: 12,
            index_type: IndexType::ProofList,
            state: 16_u64,
        };

        let mut bytes = metadata.to_bytes();
        bytes[13] = 1; // Modifies index state tag.
        assert_eq!(IndexMetadata::from_bytes(bytes.into()).unwrap(), metadata);
    }
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2016-2017, Robert Ou <rqou@robertou.com> and contributors
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

//! # xc2bit: A library for working with Xilinx Coolrunner-II bitstreams
//!
//! xc2bit is a library for reading and writing bitstreams for the Xilinx Coolrunner-II family of CPLD devices.
//!
//! This project is the result of a reverse-engineering effort involving a combination of [imaging physical
//! CPLD devices](http://siliconexposed.blogspot.com/2014/03/getting-my-feet-wet-with-invasive.html) and black-box
//! reverse-engineering of generated .jed files. It is not an official project of Xilinx, Inc. and is not
//! affiliated or endorsed by Xilinx, Inc.
//!
//! Logically, a Coolrunner-II CPLD contains the following major blocks: function blocks (occasionally abbreviated
//! to FBs), a global interconnect (occasionally referred to as the ZIA or the AIM), and input/output blocks
//! (occasionally abbreviated to IOBs). Function blocks are further divided into the PLA (programmable logic array,
//! a matrix of AND and OR gates) and macrocells. In the Coolrunner-II architecture, macrocells also contain
//! an XOR gate and a register. The global interconnect accepts inputs from IOBs and function blocks and connects these
//! inputs into the PLA of each function block. IOBs also have direct connections to a corresponding macrocell in
//! a function block. (The reverse is not always true - on larger devices, there are macrocells that are not connected
//! to IOBs.) As a special exception, the smallest 32-macrocell devices also have one single input-only pin that is
//! connected directly into the global interconnect and does not have a corresponding macrocell.

/// The number of inputs from the ZIA interconnect into the AND gate section of each PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const INPUTS_PER_ANDTERM: usize = 40;
/// The number of AND gates in each PLA. This is also the number of inputs into each OR gate in the PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const ANDTERMS_PER_FB: usize = 56;
/// The number of macrocells in each function block. This is also the number of OR gates in each PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const MCS_PER_FB: usize = 16;

mod bitstream;
pub use bitstream::{XC2Bitstream, XC2BitstreamBits, XC2GlobalNets, XC2ClockDivRatio, XC2ClockDiv, process_jed,
                    process_crbit};

mod crbit;
pub use crbit::{FuseArray};

mod fb;
pub use fb::{XC2BitstreamFB, CTC, CTR, CTS, CTE, get_pta, get_ptb, get_ptc};

mod fusemap_logical;
mod fusemap_physical;

mod iob;
pub use iob::{XC2MCSmallIOB, XC2IOBZIAMode, XC2IOBOBufMode, XC2ExtraIBuf, XC2IOBIbufMode, XC2MCLargeIOB,
              iob_num_to_fb_ff_num, fb_ff_num_to_iob_num};

mod mc;
pub use mc::{XC2Macrocell, XC2MCRegClkSrc, XC2MCRegResetSrc, XC2MCRegSetSrc, XC2MCRegMode, XC2MCFeedbackMode,
             XC2MCXorMode};

mod partdb;
pub use partdb::{XC2Device, XC2Speed, XC2Package, is_valid_part_combination, parse_part_name_string};

mod pla;
pub use pla::{XC2PLAAndTerm, XC2PLAOrTerm};

mod zia;
pub use zia::{XC2ZIARowPiece, XC2ZIAInput, ZIA_MAP_32, ZIA_MAP_64, ZIA_MAP_128, ZIA_MAP_256, ZIA_MAP_384, ZIA_MAP_512};

mod jed;
pub use jed::{read_jed};

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::File;
    use std::io::Read;

    #[test]
    fn xc2bit_reftests() {
        let files = std::fs::read_dir("./reftests").unwrap();

        for file in files {
            let path = file.expect("failed to get path").path();
            if path.extension().expect("bogus reftest filename (doesn't have extension)") == "jed" {
                let jed_path = path;
                let mut txt_path = jed_path.clone();
                txt_path.set_extension("txt");

                println!("{:?}", jed_path);

                let mut jed_data = Vec::new();
                let mut txt_data = Vec::new();

                File::open(&jed_path).expect("failed to open jed file")
                    .read_to_end(&mut jed_data).expect("failed to read jed file");
                File::open(&txt_path).expect("failed to open txt file")
                    .read_to_end(&mut txt_data).expect("failed to read txt file");

                let (parsed_jed_data, device_name) = read_jed(&jed_data).expect("failed to read jed");
                let device_name = device_name.expect("missing device name in jed");

                let parsed_bitstream_data = process_jed(&parsed_jed_data, &device_name)
                    .expect("failed to process jed");

                // FIXME: This is quite hacky
                let mut new_jed = Vec::new();
                parsed_bitstream_data.write_jed(&mut new_jed).expect("failed to write jed");
                assert_eq!(jed_data, new_jed);

                let mut human_readable_data = Vec::new();
                parsed_bitstream_data.dump_human_readable(&mut human_readable_data)
                    .expect("failed to get human readable");
                assert_eq!(txt_data, human_readable_data);
            }
        }
    }
}
<commit_msg>xc2bit: Make reftests take a round-trip through crbit<commit_after>/*
Copyright (c) 2016-2017, Robert Ou <rqou@robertou.com> and contributors
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

//! # xc2bit: A library for working with Xilinx Coolrunner-II bitstreams
//!
//! xc2bit is a library for reading and writing bitstreams for the Xilinx Coolrunner-II family of CPLD devices.
//!
//! This project is the result of a reverse-engineering effort involving a combination of [imaging physical
//! CPLD devices](http://siliconexposed.blogspot.com/2014/03/getting-my-feet-wet-with-invasive.html) and black-box
//! reverse-engineering of generated .jed files. It is not an official project of Xilinx, Inc. and is not
//! affiliated or endorsed by Xilinx, Inc.
//!
//! Logically, a Coolrunner-II CPLD contains the following major blocks: function blocks (occasionally abbreviated
//! to FBs), a global interconnect (occasionally referred to as the ZIA or the AIM), and input/output blocks
//! (occasionally abbreviated to IOBs). Function blocks are further divided into the PLA (programmable logic array,
//! a matrix of AND and OR gates) and macrocells. In the Coolrunner-II architecture, macrocells also contain
//! an XOR gate and a register. The global interconnect accepts inputs from IOBs and function blocks and connects these
//! inputs into the PLA of each function block. IOBs also have direct connections to a corresponding macrocell in
//! a function block. (The reverse is not always true - on larger devices, there are macrocells that are not connected
//! to IOBs.) As a special exception, the smallest 32-macrocell devices also have one single input-only pin that is
//! connected directly into the global interconnect and does not have a corresponding macrocell.

/// The number of inputs from the ZIA interconnect into the AND gate section of each PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const INPUTS_PER_ANDTERM: usize = 40;
/// The number of AND gates in each PLA. This is also the number of inputs into each OR gate in the PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const ANDTERMS_PER_FB: usize = 56;
/// The number of macrocells in each function block. This is also the number of OR gates in each PLA.
/// This is an unchangeable property of the architecture of the CPLD.
pub const MCS_PER_FB: usize = 16;

mod bitstream;
pub use bitstream::{XC2Bitstream, XC2BitstreamBits, XC2GlobalNets, XC2ClockDivRatio, XC2ClockDiv, process_jed,
                    process_crbit};

mod crbit;
pub use crbit::{FuseArray};

mod fb;
pub use fb::{XC2BitstreamFB, CTC, CTR, CTS, CTE, get_pta, get_ptb, get_ptc};

mod fusemap_logical;
mod fusemap_physical;

mod iob;
pub use iob::{XC2MCSmallIOB, XC2IOBZIAMode, XC2IOBOBufMode, XC2ExtraIBuf, XC2IOBIbufMode, XC2MCLargeIOB,
              iob_num_to_fb_ff_num, fb_ff_num_to_iob_num};

mod mc;
pub use mc::{XC2Macrocell, XC2MCRegClkSrc, XC2MCRegResetSrc, XC2MCRegSetSrc, XC2MCRegMode, XC2MCFeedbackMode,
             XC2MCXorMode};

mod partdb;
pub use partdb::{XC2Device, XC2Speed, XC2Package, is_valid_part_combination, parse_part_name_string};

mod pla;
pub use pla::{XC2PLAAndTerm, XC2PLAOrTerm};

mod zia;
pub use zia::{XC2ZIARowPiece, XC2ZIAInput, ZIA_MAP_32, ZIA_MAP_64, ZIA_MAP_128, ZIA_MAP_256, ZIA_MAP_384, ZIA_MAP_512};

mod jed;
pub use jed::{read_jed};

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::File;
    use std::io::Read;

    #[test]
    fn xc2bit_reftests() {
        let files = std::fs::read_dir("./reftests").unwrap();

        for file in files {
            let path = file.expect("failed to get path").path();
            if path.extension().expect("bogus reftest filename (doesn't have extension)") == "jed" {
                let jed_path = path;
                let mut txt_path = jed_path.clone();
                txt_path.set_extension("txt");

                println!("{:?}", jed_path);

                let mut jed_data = Vec::new();
                let mut txt_data = Vec::new();

                File::open(&jed_path).expect("failed to open jed file")
                    .read_to_end(&mut jed_data).expect("failed to read jed file");
                File::open(&txt_path).expect("failed to open txt file")
                    .read_to_end(&mut txt_data).expect("failed to read txt file");

                // Read original JED
                let (parsed_jed_data, device_name) = read_jed(&jed_data).expect("failed to read jed");
                let device_name = device_name.expect("missing device name in jed");

                let parsed_bitstream_data = process_jed(&parsed_jed_data, &device_name)
                    .expect("failed to process jed");

                // Write to crbit
                let mut crbit = Vec::new();
                let write_fuse_array = parsed_bitstream_data.to_crbit();
                write_fuse_array.write_to_writer(&mut crbit).expect("failed to write crbit");

                // Read back from crbit
                let read_fuse_array = FuseArray::from_file_contents(&crbit).expect("failed to read crbit");
                let parsed_bitstream_data = process_crbit(&read_fuse_array).expect("failed to process crbit");

                // FIXME: This is quite hacky
                let mut new_jed = Vec::new();
                parsed_bitstream_data.write_jed(&mut new_jed).expect("failed to write jed");
                assert_eq!(jed_data, new_jed);

                let mut human_readable_data = Vec::new();
                parsed_bitstream_data.dump_human_readable(&mut human_readable_data)
                    .expect("failed to get human readable");
                assert_eq!(txt_data, human_readable_data);
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::slice;

use libc::{c_void, c_uint, c_double, c_int, size_t};

use symbols::{ZopfliGetDistExtraBits, ZopfliGetLengthExtraBits, ZopfliGetLengthSymbol, ZopfliGetDistSymbol, ZOPFLI_NUM_LL, ZOPFLI_NUM_D};

/// Cost model which should exactly match fixed tree.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostFixed(litlen: c_uint, dist: c_uint, _unused: c_void) -> c_double {
    let result = if dist == 0 {
        if litlen <= 143 {
            8
        } else {
            9
        }
    } else {
        let dbits = ZopfliGetDistExtraBits(dist as c_int);
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int);
        let lsym = ZopfliGetLengthSymbol(litlen as c_int);
        let mut cost = 0;
        if lsym <= 279 {
            cost += 7;
        } else {
            cost += 8;
        }
        cost += 5;  // Every dist symbol has length 5.
        cost + dbits + lbits
    };
    result as c_double
}

/// Cost model based on symbol statistics.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostStat(litlen: c_uint, dist: c_uint, context: *const c_void) -> c_double {
    let stats = unsafe {
        assert!(!context.is_null());
        &*(context as *const SymbolStats)
    };
    if dist == 0 {
        stats.ll_symbols[litlen as usize]
    } else {
        let lsym = ZopfliGetLengthSymbol(litlen as c_int) as usize;
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int) as c_double;
        let dsym = ZopfliGetDistSymbol(dist as c_int) as usize;
        let dbits = ZopfliGetDistExtraBits(dist as c_int) as c_double;
        stats.ll_symbols[lsym] + lbits + stats.d_symbols[dsym] + dbits
    }
}

pub struct RanState {
    m_w: u32,
    m_z: u32,
}

impl RanState {
    pub fn new() -> RanState {
        RanState {
            m_w: 1,
            m_z: 2,
        }
    }

    /// Get random number: "Multiply-With-Carry" generator of G. Marsaglia
    pub fn random_marsaglia(&mut self) -> u32 {
        self.m_z = 36969 * (self.m_z & 65535) + (self.m_z >> 16);
        self.m_w = 18000 * (self.m_w & 65535) + (self.m_w >> 16);
        (self.m_z << 16) + self.m_w // 32-bit result.
    }
}

#[no_mangle]
pub extern fn ran_state_new() -> *mut RanState {
    Box::into_raw(Box::new(RanState::new()))
}

#[repr(C)]
pub struct SymbolStats {
  /* The literal and length symbols. */
  litlens: [size_t; ZOPFLI_NUM_LL],
  /* The 32 unique dist symbols, not the 32768 possible dists. */
  dists: [size_t; ZOPFLI_NUM_D],

  /* Length of each lit/len symbol in bits. */
  ll_symbols: [c_double; ZOPFLI_NUM_LL],
  /* Length of each dist symbol in bits. */
  d_symbols: [c_double; ZOPFLI_NUM_D],
}

pub fn randomize_freqs(state_ptr: *mut RanState, freqs_ptr: *mut size_t, n: c_int) {
    let freqs = unsafe {
        assert!(!freqs_ptr.is_null());
        slice::from_raw_parts_mut(freqs_ptr, n as usize)
    };
    let state = unsafe {
        assert!(!state_ptr.is_null());
        &mut *state_ptr
    };

    let mut i: usize = 0;
    let end = n as usize;

    while i < end {
        if (state.random_marsaglia() >> 4) % 3 == 0 {
            let index = state.random_marsaglia() % n as c_uint;
            freqs[i] = freqs[index as usize];
        }
        i += 1;
    }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn RandomizeStatFreqs(state_ptr: *mut RanState, stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };

    randomize_freqs(state_ptr, stats.litlens.as_mut_ptr(), ZOPFLI_NUM_LL as c_int);
    randomize_freqs(state_ptr, stats.dists.as_mut_ptr(), ZOPFLI_NUM_D as c_int);
    stats.litlens[256] = 1; // End symbol.
}
<commit_msg>Move random freqs into the object it should be<commit_after>use libc::{c_void, c_uint, c_double, c_int, size_t};

use symbols::{ZopfliGetDistExtraBits, ZopfliGetLengthExtraBits, ZopfliGetLengthSymbol, ZopfliGetDistSymbol, ZOPFLI_NUM_LL, ZOPFLI_NUM_D};

/// Cost model which should exactly match fixed tree.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostFixed(litlen: c_uint, dist: c_uint, _unused: c_void) -> c_double {
    let result = if dist == 0 {
        if litlen <= 143 {
            8
        } else {
            9
        }
    } else {
        let dbits = ZopfliGetDistExtraBits(dist as c_int);
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int);
        let lsym = ZopfliGetLengthSymbol(litlen as c_int);
        let mut cost = 0;
        if lsym <= 279 {
            cost += 7;
        } else {
            cost += 8;
        }
        cost += 5;  // Every dist symbol has length 5.
        cost + dbits + lbits
    };
    result as c_double
}

/// Cost model based on symbol statistics.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostStat(litlen: c_uint, dist: c_uint, context: *const c_void) -> c_double {
    let stats = unsafe {
        assert!(!context.is_null());
        &*(context as *const SymbolStats)
    };
    if dist == 0 {
        stats.ll_symbols[litlen as usize]
    } else {
        let lsym = ZopfliGetLengthSymbol(litlen as c_int) as usize;
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int) as c_double;
        let dsym = ZopfliGetDistSymbol(dist as c_int) as usize;
        let dbits = ZopfliGetDistExtraBits(dist as c_int) as c_double;
        stats.ll_symbols[lsym] + lbits + stats.d_symbols[dsym] + dbits
    }
}

pub struct RanState {
    m_w: u32,
    m_z: u32,
}

impl RanState {
    pub fn new() -> RanState {
        RanState {
            m_w: 1,
            m_z: 2,
        }
    }

    /// Get random number: "Multiply-With-Carry" generator of G. Marsaglia
    pub fn random_marsaglia(&mut self) -> u32 {
        self.m_z = 36969 * (self.m_z & 65535) + (self.m_z >> 16);
        self.m_w = 18000 * (self.m_w & 65535) + (self.m_w >> 16);
        (self.m_z << 16) + self.m_w // 32-bit result.
    }
}

#[no_mangle]
pub extern fn ran_state_new() -> *mut RanState {
    Box::into_raw(Box::new(RanState::new()))
}

#[repr(C)]
pub struct SymbolStats {
  /* The literal and length symbols. */
  litlens: [size_t; ZOPFLI_NUM_LL],
  /* The 32 unique dist symbols, not the 32768 possible dists. */
  dists: [size_t; ZOPFLI_NUM_D],

  /* Length of each lit/len symbol in bits. */
  ll_symbols: [c_double; ZOPFLI_NUM_LL],
  /* Length of each dist symbol in bits. */
  d_symbols: [c_double; ZOPFLI_NUM_D],
}

impl SymbolStats {
    pub fn randomize_stat_freqs(&mut self, state: &mut RanState) {
        fn randomize_freqs(freqs: &mut [size_t], state: &mut RanState) {
            let n = freqs.len();
            let mut i: usize = 0;
            let end = n as usize;

            while i < end {
                if (state.random_marsaglia() >> 4) % 3 == 0 {
                    let index = state.random_marsaglia() % n as c_uint;
                    freqs[i] = freqs[index as usize];
                }
                i += 1;
            }
        }
        randomize_freqs(&mut self.litlens, state);
        randomize_freqs(&mut self.dists, state);
        self.litlens[256] = 1; // End symbol.
    }
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn RandomizeStatFreqs(state_ptr: *mut RanState, stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    let state = unsafe {
        assert!(!state_ptr.is_null());
        &mut *state_ptr
    };

    stats.randomize_stat_freqs(state);
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Buffer that is written once then read for as long as it is alive.
//!
//! Use this buffer when you have data that you never modify.
//!
//! Only the first ever command buffer that uses this buffer can write to it (for example by
//! copying from another buffer). Any subsequent command buffer **must** only read from the buffer,
//! or a panic will happen.
//!
//! The buffer will be stored in device-local memory if possible
//!

use std::marker::PhantomData;
use std::mem;
use std::ops::Range;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use smallvec::SmallVec;

use buffer::sys::UnsafeBuffer;
use buffer::sys::Usage;
use buffer::traits::AccessRange;
use buffer::traits::Buffer;
use buffer::traits::GpuAccessResult;
use buffer::traits::TypedBuffer;
use command_buffer::Submission;
use device::Device;
use instance::QueueFamily;
use memory::pool::MemoryPool;
use memory::pool::MemoryPoolAlloc;
use memory::pool::StdMemoryPool;
use sync::Sharing;

use OomError;

/// Buffer that is written once then read for as long as it is alive.
pub struct ImmutableBuffer<T: ?Sized, A = StdMemoryPool> where A: MemoryPool {
    // Inner content.
    inner: UnsafeBuffer,

    memory: A::Alloc,

    // Queue families allowed to access this buffer.
    queue_families: SmallVec<[u32; 4]>,

    latest_write_submission: Mutex<Option<Weak<Submission>>>,        // TODO: can use `Weak::new()` once it's stabilized

    started_reading: AtomicBool,

    marker: PhantomData<Box<T>>,
}

impl<T> ImmutableBuffer<T> {
    /// Builds a new buffer. Only allowed for sized data.
    #[inline]
    pub fn new<'a, I>(device: &Arc<Device>, usage: &Usage, queue_families: I)
                      -> Result<Arc<ImmutableBuffer<T>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        unsafe {
            ImmutableBuffer::raw(device, mem::size_of::<T>(), usage, queue_families)
        }
    }
}

impl<T> ImmutableBuffer<[T]> {
    /// Builds a new buffer. Can be used for arrays.
    #[inline]
    pub fn array<'a, I>(device: &Arc<Device>, len: usize, usage: &Usage, queue_families: I)
                      -> Result<Arc<ImmutableBuffer<[T]>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        unsafe {
            ImmutableBuffer::raw(device, len * mem::size_of::<T>(), usage, queue_families)
        }
    }
}

impl<T: ?Sized> ImmutableBuffer<T> {
    /// Builds a new buffer without checking the size.
    ///
    /// # Safety
    ///
    /// You must ensure that the size that you pass is correct for `T`.
    ///
    pub unsafe fn raw<'a, I>(device: &Arc<Device>, size: usize, usage: &Usage, queue_families: I)
                             -> Result<Arc<ImmutableBuffer<T>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        let queue_families = queue_families.into_iter().map(|f| f.id())
                                           .collect::<SmallVec<[u32; 4]>>();

        let (buffer, mem_reqs) = {
            let sharing = if queue_families.len() >= 2 {
                Sharing::Concurrent(queue_families.iter().cloned())
            } else {
                Sharing::Exclusive
            };

            try!(UnsafeBuffer::new(device, size, &usage, sharing))
        };

        let mem_ty = {
            let device_local = device.physical_device().memory_types()
                                     .filter(|t| (mem_reqs.memory_type_bits & (1 << t.id())) != 0)
                                     .filter(|t| t.is_device_local());
            let any = device.physical_device().memory_types()
                            .filter(|t| (mem_reqs.memory_type_bits & (1 << t.id())) != 0);
            device_local.chain(any).next().unwrap()
        };

        let mem = try!(MemoryPool::alloc(&device.standard_pool(), mem_ty,
                                         mem_reqs.size, mem_reqs.alignment));
        debug_assert!((mem.offset() % mem_reqs.alignment) == 0);
        try!(buffer.bind_memory(mem.memory(), mem.offset()));

        Ok(Arc::new(ImmutableBuffer {
            inner: buffer,
            memory: mem,
            queue_families: queue_families,
            latest_write_submission: Mutex::new(None),
            started_reading: AtomicBool::new(false),
            marker: PhantomData,
        }))
    }
}

impl<T: ?Sized, A> ImmutableBuffer<T, A> where A: MemoryPool {
    /// Returns the device used to create this buffer.
    #[inline]
    pub fn device(&self) -> &Arc<Device> {
        self.inner.device()
    }

    /// Returns the queue families this buffer can be used on.
    // TODO: use a custom iterator
    #[inline]
    pub fn queue_families(&self) -> Vec<QueueFamily> {
        self.queue_families.iter().map(|&num| {
            self.device().physical_device().queue_family_by_id(num).unwrap()
        }).collect()
    }
}

unsafe impl<T: ?Sized, A> Buffer for ImmutableBuffer<T, A>
    where T: 'static + Send + Sync, A: MemoryPool
{
    #[inline]
    fn inner_buffer(&self) -> &UnsafeBuffer {
        &self.inner
    }

    #[inline]
    fn blocks(&self, _: Range<usize>) -> Vec<usize> {
        vec![0]
    }

    #[inline]
    fn block_memory_range(&self, _: usize) -> Range<usize> {
        let offset = self.memory.offset();
        offset .. offset + self.size()
    }

    fn needs_fence(&self, _: bool, _: Range<usize>) -> Option<bool> {
        Some(true)
    }

    #[inline]
    fn host_accesses(&self, _: usize) -> bool {
        false
    }

    unsafe fn gpu_access(&self, ranges: &mut Iterator<Item = AccessRange>,
                         submission: &Arc<Submission>) -> GpuAccessResult
    {
        let queue_id = submission.queue().family().id();
        if self.queue_families.iter().find(|&&id| id == queue_id).is_none() {
            panic!()
        }

        let write = {
            let mut write = false;
            while let Some(range) = ranges.next() {
                if range.write { write = true; break; }
            }
            write
        };

        if write {
            assert!(self.started_reading.load(Ordering::AcqRel) == false);
        }

        let dependency = {
            let mut latest_submission = self.latest_write_submission.lock().unwrap();

            if write {
                mem::replace(&mut *latest_submission, Some(Arc::downgrade(submission)))
            } else {
                latest_submission.clone()
            }
        };
        let dependency = dependency.and_then(|d| d.upgrade());

        if write {
            assert!(self.started_reading.load(Ordering::AcqRel) == false);
        } else {
            self.started_reading.store(true, Ordering::AcqRel);
        }

        GpuAccessResult {
            dependencies: if let Some(dependency) = dependency {
                vec![dependency]
            } else {
                vec![]
            },
            additional_wait_semaphore: None,
            additional_signal_semaphore: None,
        }
    }
}

unsafe impl<T: ?Sized, A> TypedBuffer for ImmutableBuffer<T, A>
    where T: 'static + Send + Sync, A: MemoryPool
{
    type Content = T;
}
<commit_msg>Fix bug in ImmutableBuffer implementation<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Buffer that is written once then read for as long as it is alive.
//!
//! Use this buffer when you have data that you never modify.
//!
//! Only the first ever command buffer that uses this buffer can write to it (for example by
//! copying from another buffer). Any subsequent command buffer **must** only read from the buffer,
//! or a panic will happen.
//!
//! The buffer will be stored in device-local memory if possible
//!

use std::marker::PhantomData;
use std::mem;
use std::ops::Range;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use smallvec::SmallVec;

use buffer::sys::UnsafeBuffer;
use buffer::sys::Usage;
use buffer::traits::AccessRange;
use buffer::traits::Buffer;
use buffer::traits::GpuAccessResult;
use buffer::traits::TypedBuffer;
use command_buffer::Submission;
use device::Device;
use instance::QueueFamily;
use memory::pool::MemoryPool;
use memory::pool::MemoryPoolAlloc;
use memory::pool::StdMemoryPool;
use sync::Sharing;

use OomError;

/// Buffer that is written once then read for as long as it is alive.
pub struct ImmutableBuffer<T: ?Sized, A = StdMemoryPool> where A: MemoryPool {
    // Inner content.
    inner: UnsafeBuffer,

    memory: A::Alloc,

    // Queue families allowed to access this buffer.
    queue_families: SmallVec<[u32; 4]>,

    latest_write_submission: Mutex<Option<Weak<Submission>>>,        // TODO: can use `Weak::new()` once it's stabilized

    started_reading: AtomicBool,

    marker: PhantomData<Box<T>>,
}

impl<T> ImmutableBuffer<T> {
    /// Builds a new buffer. Only allowed for sized data.
    #[inline]
    pub fn new<'a, I>(device: &Arc<Device>, usage: &Usage, queue_families: I)
                      -> Result<Arc<ImmutableBuffer<T>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        unsafe {
            ImmutableBuffer::raw(device, mem::size_of::<T>(), usage, queue_families)
        }
    }
}

impl<T> ImmutableBuffer<[T]> {
    /// Builds a new buffer. Can be used for arrays.
    #[inline]
    pub fn array<'a, I>(device: &Arc<Device>, len: usize, usage: &Usage, queue_families: I)
                      -> Result<Arc<ImmutableBuffer<[T]>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        unsafe {
            ImmutableBuffer::raw(device, len * mem::size_of::<T>(), usage, queue_families)
        }
    }
}

impl<T: ?Sized> ImmutableBuffer<T> {
    /// Builds a new buffer without checking the size.
    ///
    /// # Safety
    ///
    /// You must ensure that the size that you pass is correct for `T`.
    ///
    pub unsafe fn raw<'a, I>(device: &Arc<Device>, size: usize, usage: &Usage, queue_families: I)
                             -> Result<Arc<ImmutableBuffer<T>>, OomError>
        where I: IntoIterator<Item = QueueFamily<'a>>
    {
        let queue_families = queue_families.into_iter().map(|f| f.id())
                                           .collect::<SmallVec<[u32; 4]>>();

        let (buffer, mem_reqs) = {
            let sharing = if queue_families.len() >= 2 {
                Sharing::Concurrent(queue_families.iter().cloned())
            } else {
                Sharing::Exclusive
            };

            try!(UnsafeBuffer::new(device, size, &usage, sharing))
        };

        let mem_ty = {
            let device_local = device.physical_device().memory_types()
                                     .filter(|t| (mem_reqs.memory_type_bits & (1 << t.id())) != 0)
                                     .filter(|t| t.is_device_local());
            let any = device.physical_device().memory_types()
                            .filter(|t| (mem_reqs.memory_type_bits & (1 << t.id())) != 0);
            device_local.chain(any).next().unwrap()
        };

        let mem = try!(MemoryPool::alloc(&device.standard_pool(), mem_ty,
                                         mem_reqs.size, mem_reqs.alignment));
        debug_assert!((mem.offset() % mem_reqs.alignment) == 0);
        try!(buffer.bind_memory(mem.memory(), mem.offset()));

        Ok(Arc::new(ImmutableBuffer {
            inner: buffer,
            memory: mem,
            queue_families: queue_families,
            latest_write_submission: Mutex::new(None),
            started_reading: AtomicBool::new(false),
            marker: PhantomData,
        }))
    }
}

impl<T: ?Sized, A> ImmutableBuffer<T, A> where A: MemoryPool {
    /// Returns the device used to create this buffer.
    #[inline]
    pub fn device(&self) -> &Arc<Device> {
        self.inner.device()
    }

    /// Returns the queue families this buffer can be used on.
    // TODO: use a custom iterator
    #[inline]
    pub fn queue_families(&self) -> Vec<QueueFamily> {
        self.queue_families.iter().map(|&num| {
            self.device().physical_device().queue_family_by_id(num).unwrap()
        }).collect()
    }
}

unsafe impl<T: ?Sized, A> Buffer for ImmutableBuffer<T, A>
    where T: 'static + Send + Sync, A: MemoryPool
{
    #[inline]
    fn inner_buffer(&self) -> &UnsafeBuffer {
        &self.inner
    }

    #[inline]
    fn blocks(&self, _: Range<usize>) -> Vec<usize> {
        vec![0]
    }

    #[inline]
    fn block_memory_range(&self, _: usize) -> Range<usize> {
        let offset = self.memory.offset();
        offset .. offset + self.size()
    }

    fn needs_fence(&self, _: bool, _: Range<usize>) -> Option<bool> {
        Some(true)
    }

    #[inline]
    fn host_accesses(&self, _: usize) -> bool {
        false
    }

    unsafe fn gpu_access(&self, ranges: &mut Iterator<Item = AccessRange>,
                         submission: &Arc<Submission>) -> GpuAccessResult
    {
        let queue_id = submission.queue().family().id();
        if self.queue_families.iter().find(|&&id| id == queue_id).is_none() {
            panic!()
        }

        let write = {
            let mut write = false;
            while let Some(range) = ranges.next() {
                if range.write { write = true; break; }
            }
            write
        };

        if write {
            assert!(self.started_reading.load(Ordering::Acquire) == false);
        }

        let dependency = {
            let mut latest_submission = self.latest_write_submission.lock().unwrap();

            if write {
                mem::replace(&mut *latest_submission, Some(Arc::downgrade(submission)))
            } else {
                latest_submission.clone()
            }
        };
        let dependency = dependency.and_then(|d| d.upgrade());

        if write {
            assert!(self.started_reading.load(Ordering::Acquire) == false);
        } else {
            self.started_reading.store(true, Ordering::Release);
        }

        GpuAccessResult {
            dependencies: if let Some(dependency) = dependency {
                vec![dependency]
            } else {
                vec![]
            },
            additional_wait_semaphore: None,
            additional_signal_semaphore: None,
        }
    }
}

unsafe impl<T: ?Sized, A> TypedBuffer for ImmutableBuffer<T, A>
    where T: 'static + Send + Sync, A: MemoryPool
{
    type Content = T;
}
<|endoftext|>