<commit_before>/// Implements the block_add procedure

/*

Block storing is a tricky part; blocks are stored in the spent-tree and referenced in the
 hash-index

 This can go out of order:  For instance consider 5 blocks added in the order A, B, D, E, C
 (for this pseudocode, each block has the previous letter as prev_block_hash)

 We show the some pseudocode for actions on hashindex (hi) and spent-tree (st),
 for the insertion of this sequence

 insert A:
    st.store_block(A)
    fn do_connect(null,A) =
      (prev = null, no get_or_set)
      hi.set(A)

 insert B:
    st.store_block(B)
    hi.get_or_set(A, guard[B]) returns A
    fn do_connect(A,B) =
      st.connect_block(A,B)
      hi.set(B)

 insert D:
    st.store_block(D)
    hi.get_or_set(C, guard[D]) return none & adds guard[D] at hash C

 insert E:
   st.store_block(E)
   hi.get_or_set(D, guard[E]) return none & adds guard[E] at hash D

 insert C:
   st.store_block(C)
   hi.get_or_set(B, guard[C]) returns B
   fn do_connect(B,C) =
      st.connect_block(B,C)
      hi.set(C) -> fail due to guard([D])

        loop {
            hi.get(C) -> return guard[D]
            do_connect(C,D)
            hi.set(C, guard([D]) -> fail or break
        }

  Now let us consider what happens when C' is inserted *during* processing of B. Copyied from B above:
  B:
    st.store_block(B)
    hi.get_or_set(A, guard[B]) returns A
    fn do_connect(A,B) =
      st.connect_block(A,B)
        < here C' is inserted as B -> guard[C']
      hi.set(B) -> fail due to guard[C']
      loop {
            hi.get(B) -> return guard[C']
            do_connect(B,C')
            hi.set(B, guard([C']) -> success!
        }

*/

use hash::*;
use util::*;
use buffer::*;
use rayon::prelude::*;
use slog ;

use store::Store;
use transaction;
use merkle_tree;
use block::*;
use store::Record;
use store::BlockPtr;
use store::HashIndexGuard;

type BlockResult<T> = Result<T, BlockError>;

/// Returns true if the given hash is the hash of a genesis block
fn is_genesis_block(hash: Hash32) -> bool {
    const HASH_GENESIS: &'static str =
        "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f";

    let genesis = Hash32Buf::from_slice(&from_hex_rev(HASH_GENESIS));

    genesis.as_ref() == hash
}

// Connects two blocks (A,B) in the spent-tree and then stores the hash of B in the hash-index
// Connecting the blocks will verify double-spents
//
// It can be that another block C is also waiting for B; this will trigger their connection (B,C) too
// and maybe (C,D)... etcetera
//
// This would be neat to do recursively, but this could exhaust the stack , we use a loop with
// a to_do for connections.
fn connect_block(
    store:           &mut Store,
    this_block_hash: Hash32,
    previous_block:  Option<BlockPtr>,
    this_block:      BlockPtr)

    -> BlockResult<()>
{
    trace!(store.logger, "Connect block";
        "this_hash"  =>  format!("{:?}", this_block_hash),
        "prev_end"   =>  format!("{:?}", previous_block),
        "this_start" =>  format!("{:?}", this_block)
    );

    // we lay connections between the end of one block and the start of this_block
    // prev_end is None only for genesis
    #[derive(Debug)]
    struct Connection {
        block:         BlockPtr,
        block_hash:    Hash32Buf,
        solved_guards: Vec<BlockPtr>
    }

    // connect first block ...
    if let Some(previous_block) = previous_block {
        store.spent_tree.connect_block( &mut store.spent_index, & store.logger, previous_block, this_block) ?;
    }

    // The to_do list contains blocks that are connected to their previous but not yet added to the
    // block-index.
    let mut todo = vec![Connection {
        block:         this_block,
        block_hash:    this_block_hash.as_buf(),
        solved_guards: vec![]
    }];

    while let Some(conn) = todo.pop() {

        trace!(store.logger, "Connect block - set-hash-loop";
            "conn"  => format!("{:?}",   conn));

        // if we can store this hash we can move to the next one
        if store.block_index.set(conn.block_hash.as_ref(), conn.block.to_non_guard(), &conn.solved_guards) {
            trace!(store.logger, "Connect block - set-hash-loop - ok");
            continue;
        }

        let guards = store.block_index.get(conn.block_hash.as_ref());

        if guards.iter().any(|ptr| !ptr.is_guard()) {
            // this is not a guard, this is _this_ block. It seems the block
            // was added by a concurrent user; will do fine.
            trace!(store.logger, "Connect block - set-hash-loop - concurrent add");
            continue;
        }

        //let guards = store.hash_index.get(conn.this_hash);
        todo.push(Connection {
            block: conn.block,
            block_hash: conn.block_hash,
            solved_guards: guards.clone()
        });

        //let guards = store.hash_index.get(conn.this_hash.as_buf());

        for ptr in guards {
            if conn.solved_guards.contains(&ptr) {
                continue;
            }

            let hash = store.get_block_hash(ptr);

            trace!(store.logger, "Connect block - set-hash-loop";
                "guard" => format!("{:?}", ptr),
                "hash" => format!("{:?}",   hash),
                "conn"  => format!("{:?}",   conn));

            store.spent_tree.revolve_orphan_pointers(
                &mut store.transactions,
                &mut store.tx_index,
                ptr
            );

            store.spent_tree.connect_block(&mut store.spent_index, &store.logger, conn.block, ptr)?;

            todo.push(Connection {
                block: ptr,
                block_hash: hash,
                solved_guards: vec![]

            });

        }
    }

    Ok(())
}

/// Returns true if the block is already stored
fn block_exists(store: & mut Store, block_hash: Hash32) -> bool {
    let ptr = store.block_index.get(block_hash);

    ptr.iter().any( | ptr | !ptr.is_guard())

}

/// Verifies and stores the transactions in the block.
/// Also verifies the merkle_root & the amounts
///
/// Returns a list fileptrs to the transactions
///
fn verify_and_store_transactions(store: &mut Store, block: &Block) -> BlockResult<Vec<Record>> {

    // check block-size
    if block.to_raw().len() > ::block::MAX_BLOCK_SIZE {

        return Err(BlockError::BlockTooLarge);
    }

    // hash all transactions (in parallel)
    let hashes: Vec<Hash32Buf> = block.txs.par_iter().map(|tx| {

        Hash32Buf::double_sha256(tx.to_raw())
    }).collect();

    // here we verify and store
    let mut records: Vec<Record> = Vec::new();
    for (n,tx) in block.txs.iter().enumerate() {

        let hash = hashes[n].as_ref();

        let res = tx.verify_and_store(&mut store.tx_index, &mut store.transactions, hash).unwrap();

        // AlreadyExists and VerifiedAndStored are both ok here
        let ptr = match res {
            transaction::TransactionOk::AlreadyExists(ptr) => ptr,
            transaction::TransactionOk::VerifiedAndStored(ptr) => ptr
        };

        records.push(Record::new_transaction(ptr));
        records.append(&mut tx.get_output_records(store));

    }

    // check merkle roots
    let calculated_merkle_root = merkle_tree::get_merkle_root(hashes);
    block.verify_merkle_root(calculated_merkle_root.as_ref()).unwrap();

    Ok(records)
}

/// Validates and stores a block;
///
pub fn add_block(store: &mut Store, buffer: &[u8]) {

    // parse
    let block      = Block::new(buffer) .unwrap();
    let block_hash = Hash32Buf::double_sha256( block.header.to_raw());

    let block_logger = slog::Logger::new(&store.logger, o!());
    info!(block_logger, "add_block - start"; "hash" => format!("{:?}", block_hash));

    // already done?
    if block_exists(store, block_hash.as_ref()) {
        info!(store.logger, "add_block - Block already exists");
        return;
    }

    // check and store the transactions in block_content and check the merkle_root
    let spent_tree_ptrs = verify_and_store_transactions(store, &block).unwrap();

    // store the blockheader in block_content
    let block_header_ptr = store.block_headers.write( &block.header.to_raw());

    // we also store the txcount, although we only use it for a reindex benchmark
    let _ = store.block_headers.write_fixed( &block.txs.len());

    // store the block in the spent_tree

    let block_ptr       = store.spent_tree.store_block(block_header_ptr, spent_tree_ptrs);

/*
    if is_genesis_block(block_hash.as_ref()) {

        info ! (block_logger, "add_block - storing genesis block");

        // there is None previous block, but we call connect_block anyway as this will also
        // connect to next blocks if they are already in
        connect_block(store, block_hash.as_ref(), None, block_ptr).unwrap();
    }
    else {

        // we retrieve the pointer to the end of the previous block from the hash-index
        // if it is not yet in, this hash will be inserted as a guard-block
        let previous_block = store.block_index.get_or_set( block.header.prev_hash, block_ptr.to_guard());

        info! (block_logger, "add_block - tranactions done";
            "previous" => format!("{:?}", block.header.prev_hash),
            "ptr" => format!("{:?}", previous_block));

        // if it is in, we will connect
        if let Some(previous_block) = previous_block {

            connect_block(store, block_hash.as_ref(), Some(previous_block), block_ptr).unwrap();
        }

    }
*/
    // TODO verify amounts
    // TODO verify PoW
    // TODO verify header-syntax

    info!(block_logger, "add_block - done");
}

#[cfg(test)]
mod tests {

    use store;
    use super::*;

    #[test]
    fn test_clone_into_thread() {

        struct X { data: Vec<u32> };
        impl Clone for X {
            fn clone(&self) -> X { X { data: self.data.clone() } }
        }
        let x = X { data: vec![1,2,3,4,5,6,7,8,9,10]};

        let y = vec![10,11,12,13,14,15,16,17,18,19,20];

        y.into_par_iter().map(|z| {
            let u = &mut x.clone(); // is this possible?
            println!("{:?} {:?} ", z, u.data); u.data.push(2);
        });
    }

    #[test]
    fn test_block_simple() {

        let mut store = store::Store::new(& test_cfg!());

        tx_builder!(bld);

        let block0 = genesis!();

        let block1 = blk!(prev = block0;
            tx!(bld; coinbase => b;11 ),
            tx!(bld; b => c,e )
        );

        let block2 = blk!(prev = block1;
            tx!(bld; coinbase => f;12 ),
            tx!(bld; c => g )
        );

        add_block(&mut store, &block0);
        add_block(&mut store, &block1);
        add_block(&mut store, &block2);

    }

    #[test]
    fn test_blocks_reorder() {

        let mut store = store::Store::new(& test_cfg!());

        tx_builder!(bld);

        let block0 = genesis!();
        let block1 = blk!(prev = block0;
            tx!(bld; coinbase => b;11 ),
            tx!(bld; b => c,e )
        );

        let block2 = blk!(prev = block1;
            tx!(bld; coinbase => f;12 ),
            tx!(bld; c => g )
        );

        //        add_block(&mut store, &block0);

        println!("block1 = {:?}", block1);
        //println!("tx1 = {:?}", ::hash::Hash32Buf::double_sha256(&tx1));
        add_block(&mut store, &block0);
        add_block(&mut store, &block2);
        add_block(&mut store, &block1);

    }

}

<commit_msg>Parellelize initial transaction hashing<commit_after>/// Implements the block_add procedure

/*

Block storing is a tricky part; blocks are stored in the spent-tree and referenced in the
 hash-index

 This can go out of order:  For instance consider 5 blocks added in the order A, B, D, E, C
 (for this pseudocode, each block has the previous letter as prev_block_hash)

 We show the some pseudocode for actions on hashindex (hi) and spent-tree (st),
 for the insertion of this sequence

 insert A:
    st.store_block(A)
    fn do_connect(null,A) =
      (prev = null, no get_or_set)
      hi.set(A)

 insert B:
    st.store_block(B)
    hi.get_or_set(A, guard[B]) returns A
    fn do_connect(A,B) =
      st.connect_block(A,B)
      hi.set(B)

 insert D:
    st.store_block(D)
    hi.get_or_set(C, guard[D]) return none & adds guard[D] at hash C

 insert E:
   st.store_block(E)
   hi.get_or_set(D, guard[E]) return none & adds guard[E] at hash D

 insert C:
   st.store_block(C)
   hi.get_or_set(B, guard[C]) returns B
   fn do_connect(B,C) =
      st.connect_block(B,C)
      hi.set(C) -> fail due to guard([D])

        loop {
            hi.get(C) -> return guard[D]
            do_connect(C,D)
            hi.set(C, guard([D]) -> fail or break
        }

  Now let us consider what happens when C' is inserted *during* processing of B. Copyied from B above:
  B:
    st.store_block(B)
    hi.get_or_set(A, guard[B]) returns A
    fn do_connect(A,B) =
      st.connect_block(A,B)
        < here C' is inserted as B -> guard[C']
      hi.set(B) -> fail due to guard[C']
      loop {
            hi.get(B) -> return guard[C']
            do_connect(B,C')
            hi.set(B, guard([C']) -> success!
        }

*/

use hash::*;
use util::*;
use buffer::*;
use rayon::prelude::*;
use slog ;

use store::Store;
use transaction;
use merkle_tree;
use block::*;
use store::Record;
use store::BlockPtr;
use store::HashIndexGuard;

type BlockResult<T> = Result<T, BlockError>;

// minimum number of hashes to use parallel hashing
const PARALLEL_HASHING_THRESHOLD: usize = 60;

/// Returns true if the given hash is the hash of a genesis block
fn is_genesis_block(hash: Hash32) -> bool {
    const HASH_GENESIS: &'static str =
        "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f";

    let genesis = Hash32Buf::from_slice(&from_hex_rev(HASH_GENESIS));

    genesis.as_ref() == hash
}

// Connects two blocks (A,B) in the spent-tree and then stores the hash of B in the hash-index
// Connecting the blocks will verify double-spents
//
// It can be that another block C is also waiting for B; this will trigger their connection (B,C) too
// and maybe (C,D)... etcetera
//
// This would be neat to do recursively, but this could exhaust the stack , we use a loop with
// a to_do for connections.
fn connect_block(
    store:           &mut Store,
    this_block_hash: Hash32,
    previous_block:  Option<BlockPtr>,
    this_block:      BlockPtr)

    -> BlockResult<()>
{
    trace!(store.logger, "Connect block";
        "this_hash"  =>  format!("{:?}", this_block_hash),
        "prev_end"   =>  format!("{:?}", previous_block),
        "this_start" =>  format!("{:?}", this_block)
    );

    // we lay connections between the end of one block and the start of this_block
    // prev_end is None only for genesis
    #[derive(Debug)]
    struct Connection {
        block:         BlockPtr,
        block_hash:    Hash32Buf,
        solved_guards: Vec<BlockPtr>
    }

    // connect first block ...
    if let Some(previous_block) = previous_block {
        store.spent_tree.connect_block( &mut store.spent_index, & store.logger, previous_block, this_block) ?;
    }

    // The to_do list contains blocks that are connected to their previous but not yet added to the
    // block-index.
    let mut todo = vec![Connection {
        block:         this_block,
        block_hash:    this_block_hash.as_buf(),
        solved_guards: vec![]
    }];

    while let Some(conn) = todo.pop() {

        trace!(store.logger, "Connect block - set-hash-loop";
            "conn"  => format!("{:?}",   conn));

        // if we can store this hash we can move to the next one
        if store.block_index.set(conn.block_hash.as_ref(), conn.block.to_non_guard(), &conn.solved_guards) {
            trace!(store.logger, "Connect block - set-hash-loop - ok");
            continue;
        }

        let guards = store.block_index.get(conn.block_hash.as_ref());

        if guards.iter().any(|ptr| !ptr.is_guard()) {
            // this is not a guard, this is _this_ block. It seems the block
            // was added by a concurrent user; will do fine.
            trace!(store.logger, "Connect block - set-hash-loop - concurrent add");
            continue;
        }

        //let guards = store.hash_index.get(conn.this_hash);
        todo.push(Connection {
            block: conn.block,
            block_hash: conn.block_hash,
            solved_guards: guards.clone()
        });

        //let guards = store.hash_index.get(conn.this_hash.as_buf());

        for ptr in guards {
            if conn.solved_guards.contains(&ptr) {
                continue;
            }

            let hash = store.get_block_hash(ptr);

            trace!(store.logger, "Connect block - set-hash-loop";
                "guard" => format!("{:?}", ptr),
                "hash" => format!("{:?}",   hash),
                "conn"  => format!("{:?}",   conn));

            store.spent_tree.revolve_orphan_pointers(
                &mut store.transactions,
                &mut store.tx_index,
                ptr
            );

            store.spent_tree.connect_block(&mut store.spent_index, &store.logger, conn.block, ptr)?;

            todo.push(Connection {
                block: ptr,
                block_hash: hash,
                solved_guards: vec![]

            });

        }
    }

    Ok(())
}

/// Returns true if the block is already stored
fn block_exists(store: & mut Store, block_hash: Hash32) -> bool {
    let ptr = store.block_index.get(block_hash);

    ptr.iter().any( | ptr | !ptr.is_guard())

}

/// Verifies and stores the transactions in the block.
/// Also verifies the merkle_root & the amounts
///
/// Returns a list fileptrs to the transactions
///
fn verify_and_store_transactions(store: &mut Store, block: &Block) -> BlockResult<Vec<Record>> {

    // check block-size
    if block.to_raw().len() > ::block::MAX_BLOCK_SIZE {

        return Err(BlockError::BlockTooLarge);
    }

    // hash all transactions (in parallel)
    let hashes: Vec<Hash32Buf> = if block.txs.len() > PARALLEL_HASHING_THRESHOLD {
        block.txs.par_iter().map(|tx| {
            Hash32Buf::double_sha256(tx.to_raw())
        }).collect()
    }
    else // or sequential
    {
        block.txs.iter().map(|tx| {
            Hash32Buf::double_sha256(tx.to_raw())
        }).collect()
    };

    // here we verify and store
    let mut records: Vec<Record> = Vec::new();
    for (n,tx) in block.txs.iter().enumerate() {

        let hash = hashes[n].as_ref();

        let res = tx.verify_and_store(&mut store.tx_index, &mut store.transactions, hash).unwrap();

        // AlreadyExists and VerifiedAndStored are both ok here
        let ptr = match res {
            transaction::TransactionOk::AlreadyExists(ptr) => ptr,
            transaction::TransactionOk::VerifiedAndStored(ptr) => ptr
        };

        records.push(Record::new_transaction(ptr));
        records.append(&mut tx.get_output_records(store));

    }

    // check merkle roots
    let calculated_merkle_root = merkle_tree::get_merkle_root(hashes);
    block.verify_merkle_root(calculated_merkle_root.as_ref()).unwrap();

    Ok(records)
}

/// Validates and stores a block;
///
pub fn add_block(store: &mut Store, buffer: &[u8]) {

    // parse
    let block      = Block::new(buffer) .unwrap();
    let block_hash = Hash32Buf::double_sha256( block.header.to_raw());

    let block_logger = slog::Logger::new(&store.logger, o!());
    info!(block_logger, "add_block - start"; "hash" => format!("{:?}", block_hash));

    // already done?
    if block_exists(store, block_hash.as_ref()) {
        info!(store.logger, "add_block - Block already exists");
        return;
    }

    // check and store the transactions in block_content and check the merkle_root
    let spent_tree_ptrs = verify_and_store_transactions(store, &block).unwrap();

    // store the blockheader in block_content
    let block_header_ptr = store.block_headers.write( &block.header.to_raw());

    // we also store the txcount, although we only use it for a reindex benchmark
    let _ = store.block_headers.write_fixed( &block.txs.len());

    // store the block in the spent_tree

    let block_ptr       = store.spent_tree.store_block(block_header_ptr, spent_tree_ptrs);

    if is_genesis_block(block_hash.as_ref()) {

        info ! (block_logger, "add_block - storing genesis block");

        // there is None previous block, but we call connect_block anyway as this will also
        // connect to next blocks if they are already in
        connect_block(store, block_hash.as_ref(), None, block_ptr).unwrap();
    }
    else {

        // we retrieve the pointer to the end of the previous block from the hash-index
        // if it is not yet in, this hash will be inserted as a guard-block
        let previous_block = store.block_index.get_or_set( block.header.prev_hash, block_ptr.to_guard());

        info! (block_logger, "add_block - tranactions done";
            "previous" => format!("{:?}", block.header.prev_hash),
            "ptr" => format!("{:?}", previous_block));

        // if it is in, we will connect
        if let Some(previous_block) = previous_block {

            connect_block(store, block_hash.as_ref(), Some(previous_block), block_ptr).unwrap();
        }

    }

    // TODO verify amounts
    // TODO verify PoW
    // TODO verify header-syntax

    info!(block_logger, "add_block - done");
}

#[cfg(test)]
mod tests {

    use store;
    use super::*;

    #[test]
    fn test_clone_into_thread() {

        struct X { data: Vec<u32> };
        impl Clone for X {
            fn clone(&self) -> X { X { data: self.data.clone() } }
        }
        let x = X { data: vec![1,2,3,4,5,6,7,8,9,10]};

        let y = vec![10,11,12,13,14,15,16,17,18,19,20];

        y.into_par_iter().map(|z| {
            let u = &mut x.clone(); // is this possible?
            println!("{:?} {:?} ", z, u.data); u.data.push(2);
        });
    }

    #[test]
    fn test_block_simple() {

        let mut store = store::Store::new(& test_cfg!());

        tx_builder!(bld);

        let block0 = genesis!();

        let block1 = blk!(prev = block0;
            tx!(bld; coinbase => b;11 ),
            tx!(bld; b => c,e )
        );

        let block2 = blk!(prev = block1;
            tx!(bld; coinbase => f;12 ),
            tx!(bld; c => g )
        );

        add_block(&mut store, &block0);
        add_block(&mut store, &block1);
        add_block(&mut store, &block2);

    }

    #[test]
    fn test_blocks_reorder() {

        let mut store = store::Store::new(& test_cfg!());

        tx_builder!(bld);

        let block0 = genesis!();
        let block1 = blk!(prev = block0;
            tx!(bld; coinbase => b;11 ),
            tx!(bld; b => c,e )
        );

        let block2 = blk!(prev = block1;
            tx!(bld; coinbase => f;12 ),
            tx!(bld; c => g )
        );

        //        add_block(&mut store, &block0);

        println!("block1 = {:?}", block1);
        //println!("tx1 = {:?}", ::hash::Hash32Buf::double_sha256(&tx1));
        add_block(&mut store, &block0);
        add_block(&mut store, &block2);
        add_block(&mut store, &block1);

    }

}

<|endoftext|>


<commit_before>#![crate_name="cargo"]
#![crate_type="rlib"]

#![feature(macro_rules, phase)]
#![feature(default_type_params)]

extern crate debug;
extern crate term;
extern crate collections;
extern crate url;
extern crate serialize;
extern crate semver;
extern crate toml;

#[phase(plugin, link)]
extern crate hammer;

#[phase(plugin, link)]
extern crate log;

#[cfg(test)]
extern crate hamcrest;

use serialize::{Decoder, Encoder, Decodable, Encodable, json};
use std::io;
use std::io::{stdout, stderr};
use std::io::stdio::{stdout_raw, stderr_raw};
use hammer::{Flags, decode_args, usage};

use core::{Shell, MultiShell, ShellConfig};
use term::color::{BLACK};

pub use util::{CargoError, CliError, CliResult, human};

macro_rules! some(
    ($e:expr) => (
        match $e {
            Some(e) => e,
            None => return None
        }
    )
)

// Added so that the try! macro below can refer to cargo::util, while
// other external importers of this macro can use it as well.
//
// "Hygiene strikes again" - @acrichton
mod cargo {
    pub use super::util;
}

#[macro_export]
macro_rules! try (
    ($expr:expr) => ({
        use cargo::util::FromError;
        match $expr.map_err(FromError::from_error) {
            Ok(val) => val,
            Err(err) => return Err(err)
        }
    })
)

macro_rules! raw_try (
    ($expr:expr) => ({
        match $expr {
            Ok(val) => val,
            Err(err) => return Err(err)
        }
    })
)

pub mod core;
pub mod ops;
pub mod sources;
pub mod util;

trait RepresentsJSON : Decodable<json::Decoder, json::DecoderError> {}
impl<T: Decodable<json::Decoder, json::DecoderError>> RepresentsJSON for T {}

#[deriving(Decodable)]
pub struct NoFlags;

hammer_config!(NoFlags)

#[deriving(Show, Decodable)]
pub struct GlobalFlags {
    verbose: bool,
    help: bool,
    rest: Vec<String>
}

hammer_config!(GlobalFlags |c| {
    c.short("verbose", 'v').short("help", 'h')
})

pub fn execute_main<'a,
                    T: Flags,
                    U: RepresentsJSON,
                    V: Encodable<json::Encoder<'a>, io::IoError>>(
                        exec: fn(T, U, &mut MultiShell) -> CliResult<Option<V>>) {
    fn call<'a,
            T: Flags,
            U: RepresentsJSON,
            V: Encodable<json::Encoder<'a>, io::IoError>>(
                exec: fn(T, U, &mut MultiShell) -> CliResult<Option<V>>,
                shell: &mut MultiShell,
                args: &[String]) -> CliResult<Option<V>> {
        let flags = try!(flags_from_args::<T>(args));
        let json = try!(json_from_stdin::<U>());

        exec(flags, json, shell)
    }

    process::<T, V>(|rest, shell| call(exec, shell, rest));
}

pub fn execute_main_without_stdin<'a,
                                  T: Flags,
                                  V: Encodable<json::Encoder<'a>, io::IoError>>(
                                      exec: fn(T, &mut MultiShell) -> CliResult<Option<V>>) {
    fn call<'a,
            T: Flags,
            V: Encodable<json::Encoder<'a>, io::IoError>>(
                exec: fn(T, &mut MultiShell) -> CliResult<Option<V>>,
                shell: &mut MultiShell,
                args: &[String]) -> CliResult<Option<V>> {
        let flags = try!(flags_from_args::<T>(args));
        exec(flags, shell)
    }

    process::<T, V>(|rest, shell| call(exec, shell, rest));
}

fn process<'a,
           T: Flags,
           V: Encodable<json::Encoder<'a>, io::IoError>>(
               callback: |&[String], &mut MultiShell| -> CliResult<Option<V>>) {

    match global_flags() {
        Err(e) => handle_error(e, &mut shell(false)),
        Ok(val) => {
            let mut shell = shell(val.verbose);

            if val.help {
                let (desc, options) = usage::<T>(true);

                desc.map(|d| println!("{}\n", d));

                println!("Options:\n");

                print!("{}", options);

                let (_, options) = usage::<GlobalFlags>(false);
                print!("{}", options);
            } else {
                process_executed(callback(val.rest.as_slice(), &mut shell), &mut shell)
            }
        }
    }
}

pub fn process_executed<'a,
                        T: Encodable<json::Encoder<'a>, io::IoError>>(
                            result: CliResult<Option<T>>,
                            shell: &mut MultiShell) {
    match result {
        Err(e) => handle_error(e, shell),
        Ok(encodable) => {
            encodable.map(|encodable| {
                let encoded = json::encode(&encodable);
                println!("{}", encoded);
            });
        }
    }
}

pub fn shell(verbose: bool) -> MultiShell {
    let tty = stderr_raw().isatty();
    let stderr = box stderr() as Box<Writer>;

    let config = ShellConfig { color: true, verbose: verbose, tty: tty };
    let err = Shell::create(stderr, config);

    let tty = stdout_raw().isatty();
    let stdout = box stdout() as Box<Writer>;

    let config = ShellConfig { color: true, verbose: verbose, tty: tty };
    let out = Shell::create(stdout, config);

    MultiShell::new(out, err, verbose)
}

pub fn handle_error(err: CliError, shell: &mut MultiShell) {
    log!(4, "handle_error; err={}", err);

    let CliError { error, exit_code, unknown, .. } = err;

    if unknown {
        let _ = shell.error("An unknown error occurred");
    } else {
        let _ = shell.error(error.to_string());
    }

    if error.cause().is_some() {
        let _ = shell.concise(|shell| {
            shell.err().say("\nTo learn more, run the command again with --verbose.", BLACK)
        });
    }

    let _ = shell.verbose(|shell| {
        error.cause().map(|err| {
            let _ = handle_cause(err, shell);
        });
        Ok(())
      });

    std::os::set_exit_status(exit_code as int);
}

fn handle_cause(err: &CargoError, shell: &mut MultiShell) {
    let _ = shell.err().say("\nCaused by:", BLACK);
    let _ = shell.err().say(format!("  {}", err.description()), BLACK);

    err.cause().map(|e| handle_cause(e, shell));
}

fn args() -> Vec<String> {
    std::os::args()
}

fn flags_from_args<T: Flags>(args: &[String]) -> CliResult<T> {
    decode_args(args).map_err(|e| {
        CliError::new(e.message, 1)
    })
}

fn global_flags() -> CliResult<GlobalFlags> {
    decode_args(args().tail()).map_err(|e| {
        CliError::new(e.message, 1)
    })
}

fn json_from_stdin<T: RepresentsJSON>() -> CliResult<T> {
    let mut reader = io::stdin();
    let input = try!(reader.read_to_string().map_err(|_| {
        CliError::new("Standard in did not exist or was not UTF-8", 1)
    }));

    let json = try!(json::from_str(input.as_slice()).map_err(|_| {
        CliError::new("Could not parse standard in as JSON", 1)
    }));
    let mut decoder = json::Decoder::new(json);

    Decodable::decode(&mut decoder).map_err(|_| {
        CliError::new("Could not process standard in as input", 1)
    })
}
<commit_msg>Print unknown errors with --verbose<commit_after>#![crate_name="cargo"]
#![crate_type="rlib"]

#![feature(macro_rules, phase)]
#![feature(default_type_params)]

extern crate debug;
extern crate term;
extern crate collections;
extern crate url;
extern crate serialize;
extern crate semver;
extern crate toml;

#[phase(plugin, link)]
extern crate hammer;

#[phase(plugin, link)]
extern crate log;

#[cfg(test)]
extern crate hamcrest;

use serialize::{Decoder, Encoder, Decodable, Encodable, json};
use std::io;
use std::io::{stdout, stderr};
use std::io::stdio::{stdout_raw, stderr_raw};
use hammer::{Flags, decode_args, usage};

use core::{Shell, MultiShell, ShellConfig};
use term::color::{BLACK};

pub use util::{CargoError, CliError, CliResult, human};

macro_rules! some(
    ($e:expr) => (
        match $e {
            Some(e) => e,
            None => return None
        }
    )
)

// Added so that the try! macro below can refer to cargo::util, while
// other external importers of this macro can use it as well.
//
// "Hygiene strikes again" - @acrichton
mod cargo {
    pub use super::util;
}

#[macro_export]
macro_rules! try (
    ($expr:expr) => ({
        use cargo::util::FromError;
        match $expr.map_err(FromError::from_error) {
            Ok(val) => val,
            Err(err) => return Err(err)
        }
    })
)

macro_rules! raw_try (
    ($expr:expr) => ({
        match $expr {
            Ok(val) => val,
            Err(err) => return Err(err)
        }
    })
)

pub mod core;
pub mod ops;
pub mod sources;
pub mod util;

trait RepresentsJSON : Decodable<json::Decoder, json::DecoderError> {}
impl<T: Decodable<json::Decoder, json::DecoderError>> RepresentsJSON for T {}

#[deriving(Decodable)]
pub struct NoFlags;

hammer_config!(NoFlags)

#[deriving(Show, Decodable)]
pub struct GlobalFlags {
    verbose: bool,
    help: bool,
    rest: Vec<String>
}

hammer_config!(GlobalFlags |c| {
    c.short("verbose", 'v').short("help", 'h')
})

pub fn execute_main<'a,
                    T: Flags,
                    U: RepresentsJSON,
                    V: Encodable<json::Encoder<'a>, io::IoError>>(
                        exec: fn(T, U, &mut MultiShell) -> CliResult<Option<V>>) {
    fn call<'a,
            T: Flags,
            U: RepresentsJSON,
            V: Encodable<json::Encoder<'a>, io::IoError>>(
                exec: fn(T, U, &mut MultiShell) -> CliResult<Option<V>>,
                shell: &mut MultiShell,
                args: &[String]) -> CliResult<Option<V>> {
        let flags = try!(flags_from_args::<T>(args));
        let json = try!(json_from_stdin::<U>());

        exec(flags, json, shell)
    }

    process::<T, V>(|rest, shell| call(exec, shell, rest));
}

pub fn execute_main_without_stdin<'a,
                                  T: Flags,
                                  V: Encodable<json::Encoder<'a>, io::IoError>>(
                                      exec: fn(T, &mut MultiShell) -> CliResult<Option<V>>) {
    fn call<'a,
            T: Flags,
            V: Encodable<json::Encoder<'a>, io::IoError>>(
                exec: fn(T, &mut MultiShell) -> CliResult<Option<V>>,
                shell: &mut MultiShell,
                args: &[String]) -> CliResult<Option<V>> {
        let flags = try!(flags_from_args::<T>(args));
        exec(flags, shell)
    }

    process::<T, V>(|rest, shell| call(exec, shell, rest));
}

fn process<'a,
           T: Flags,
           V: Encodable<json::Encoder<'a>, io::IoError>>(
               callback: |&[String], &mut MultiShell| -> CliResult<Option<V>>) {

    match global_flags() {
        Err(e) => handle_error(e, &mut shell(false)),
        Ok(val) => {
            let mut shell = shell(val.verbose);

            if val.help {
                let (desc, options) = usage::<T>(true);

                desc.map(|d| println!("{}\n", d));

                println!("Options:\n");

                print!("{}", options);

                let (_, options) = usage::<GlobalFlags>(false);
                print!("{}", options);
            } else {
                process_executed(callback(val.rest.as_slice(), &mut shell), &mut shell)
            }
        }
    }
}

pub fn process_executed<'a,
                        T: Encodable<json::Encoder<'a>, io::IoError>>(
                            result: CliResult<Option<T>>,
                            shell: &mut MultiShell) {
    match result {
        Err(e) => handle_error(e, shell),
        Ok(encodable) => {
            encodable.map(|encodable| {
                let encoded = json::encode(&encodable);
                println!("{}", encoded);
            });
        }
    }
}

pub fn shell(verbose: bool) -> MultiShell {
    let tty = stderr_raw().isatty();
    let stderr = box stderr() as Box<Writer>;

    let config = ShellConfig { color: true, verbose: verbose, tty: tty };
    let err = Shell::create(stderr, config);

    let tty = stdout_raw().isatty();
    let stdout = box stdout() as Box<Writer>;

    let config = ShellConfig { color: true, verbose: verbose, tty: tty };
    let out = Shell::create(stdout, config);

    MultiShell::new(out, err, verbose)
}

pub fn handle_error(err: CliError, shell: &mut MultiShell) {
    log!(4, "handle_error; err={}", err);

    let CliError { error, exit_code, unknown, .. } = err;

    if unknown {
        let _ = shell.error("An unknown error occurred");
    } else {
        let _ = shell.error(error.to_string());
    }

    if error.cause().is_some() || unknown {
        let _ = shell.concise(|shell| {
            shell.err().say("\nTo learn more, run the command again with --verbose.", BLACK)
        });
    }

    let _ = shell.verbose(|shell| {
        if unknown {
            let _ = shell.error(error.to_string());
        }
        error.cause().map(|err| {
            let _ = handle_cause(err, shell);
        });
        Ok(())
      });

    std::os::set_exit_status(exit_code as int);
}

fn handle_cause(err: &CargoError, shell: &mut MultiShell) {
    let _ = shell.err().say("\nCaused by:", BLACK);
    let _ = shell.err().say(format!("  {}", err.description()), BLACK);

    err.cause().map(|e| handle_cause(e, shell));
}

fn args() -> Vec<String> {
    std::os::args()
}

fn flags_from_args<T: Flags>(args: &[String]) -> CliResult<T> {
    decode_args(args).map_err(|e| {
        CliError::new(e.message, 1)
    })
}

fn global_flags() -> CliResult<GlobalFlags> {
    decode_args(args().tail()).map_err(|e| {
        CliError::new(e.message, 1)
    })
}

fn json_from_stdin<T: RepresentsJSON>() -> CliResult<T> {
    let mut reader = io::stdin();
    let input = try!(reader.read_to_string().map_err(|_| {
        CliError::new("Standard in did not exist or was not UTF-8", 1)
    }));

    let json = try!(json::from_str(input.as_slice()).map_err(|_| {
        CliError::new("Could not parse standard in as JSON", 1)
    }));
    let mut decoder = json::Decoder::new(json);

    Decodable::decode(&mut decoder).map_err(|_| {
        CliError::new("Could not process standard in as input", 1)
    })
}
<|endoftext|>


<commit_before>use std::option;
use std::vec;
use {MAX_QOS};
use error::Result;
use mqtt3::{SubscribeTopic, TopicPath, PacketIdentifier, QoS};

#[derive(Debug, Clone)]
pub struct Subscription {
    pub pid: PacketIdentifier,
    pub topic_path: TopicPath,
    pub qos: QoS
}

pub trait ToSubTopics {
    type Iter: Iterator<Item=SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter>;
}

impl ToSubTopics for SubscribeTopic {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(self.clone()).into_iter())
    }
}

impl<'a> ToSubTopics for &'a str {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(SubscribeTopic { topic_path: self.to_string(), qos: MAX_QOS }).into_iter())
    }
}

impl ToSubTopics for (String, QoS) {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        let (ref topic_path, qos): (String, QoS) = *self;
        Ok(Some(SubscribeTopic { topic_path: topic_path.clone(), qos: qos }).into_iter())
    }
}

pub trait ToUnSubTopics {
    type Iter: Iterator<Item=String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter>;
}

impl ToUnSubTopics for Vec<String> {
    type Iter = vec::IntoIter<String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter> {
        Ok(self.clone().into_iter())
    }
}

impl<'a> ToUnSubTopics for &'a str {
    type Iter = option::IntoIter<String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(self.to_string()).into_iter())
    }
}
<commit_msg>mqttc: Added further ToSubTopics converter<commit_after>use std::option;
use std::vec;
use {MAX_QOS};
use error::Result;
use mqtt3::{SubscribeTopic, TopicPath, PacketIdentifier, QoS};

#[derive(Debug, Clone)]
pub struct Subscription {
    pub pid: PacketIdentifier,
    pub topic_path: TopicPath,
    pub qos: QoS
}

pub trait ToSubTopics {
    type Iter: Iterator<Item=SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter>;
}

impl ToSubTopics for SubscribeTopic {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(self.clone()).into_iter())
    }
}

impl ToSubTopics for Vec<SubscribeTopic> {
    type Iter = vec::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        Ok(self.clone().into_iter()) //FIXME:
    }
}

impl<'a> ToSubTopics for &'a str {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(SubscribeTopic { topic_path: self.to_string(), qos: MAX_QOS }).into_iter())
    }
}

impl ToSubTopics for (String, QoS) {
    type Iter = option::IntoIter<SubscribeTopic>;
    fn to_subscribe_topics(&self) -> Result<Self::Iter> {
        let (ref topic_path, qos): (String, QoS) = *self;
        Ok(Some(SubscribeTopic { topic_path: topic_path.clone(), qos: qos }).into_iter())
    }
}

pub trait ToUnSubTopics {
    type Iter: Iterator<Item=String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter>;
}

impl ToUnSubTopics for Vec<String> {
    type Iter = vec::IntoIter<String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter> {
        Ok(self.clone().into_iter())
    }
}

impl<'a> ToUnSubTopics for &'a str {
    type Iter = option::IntoIter<String>;
    fn to_unsubscribe_topics(&self) -> Result<Self::Iter> {
        Ok(Some(self.to_string()).into_iter())
    }
}
<|endoftext|>


<commit_before>// x, y
pub struct Coordinate(pub isize, pub isize);

// In the stencil and the image, the first coordinate is x, the second is y.
// This is computing the convolution between the stencil and the extracted matrix from the
// image
pub fn compute_pixel(stencil: &Vec<Vec<f32>>, coordinate: Coordinate, image_ref: &Vec<Vec<(f32, f32, f32)>>) -> (f32, f32, f32) {
    let stencil_size = (stencil.len(), stencil[0].len());

    let extracted_matrix = extract_matrix(image_ref, coordinate, stencil_size);

    let mut result: (f32, f32, f32) = (0.0, 0.0, 0.0);

    for i in 0..stencil_size.0 {
        for j in 0..stencil_size.1 {
            result.0 += extracted_matrix[i][j].0 * stencil[i][j];
            result.1 += extracted_matrix[i][j].1 * stencil[i][j];
            result.2 += extracted_matrix[i][j].2 * stencil[i][j];
        }
    }
    result
}

// Extracts the pixels values for a given stencil dimensions from an image
pub fn extract_matrix(image_ref: &Vec<Vec<(f32, f32, f32)>>, coordinate: Coordinate, dimensions: (usize, usize))
-> Vec<Vec<(f32, f32, f32)>>{

    let delta_x = ((dimensions.0 - 1) as f32 / 2.0) as isize;
    let delta_y = ((dimensions.1 - 1) as f32 / 2.0) as isize;

    let mut extracted_matrix: Vec<Vec<(f32, f32, f32)>> = Vec::with_capacity(dimensions.0);

    for (i, x) in ((coordinate.0 - delta_x)..(coordinate.0 + delta_x + 1)).enumerate() {
        extracted_matrix.push(Vec::with_capacity(dimensions.1));
        for y in (coordinate.1 - delta_y)..(coordinate.1 + delta_y + 1) {
            if x < 0 || y < 0 || x > (image_ref.len() as isize) || (y > image_ref[0].len() as isize) {
                extracted_matrix[i].push((0.0, 0.0, 0.0));
            } else {
                extracted_matrix[i].push(image_ref[x as usize][y as usize]);
            }
        }
    }

    extracted_matrix
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn matrix_regular_extraction() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let stencil_size = (stencil.len(), stencil[0].len());
        let coord = Coordinate(1, 1);
        let extracted = extract_matrix(&image, coord, stencil_size);

        assert_eq!(extracted.len(), 3);
        assert_eq!(extracted[0].len(), 3);
        assert_eq!(extracted[0][0], 1.0);
        assert_eq!(extracted[0][1], 3.0);
        assert_eq!(extracted[1][0], 1.0);
        assert_eq!(extracted[1][1], 3.0);
        assert_eq!(extracted[2][2], 4.0);
    }

    #[test]
    fn compute_pixel_test() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let coord = Coordinate(1, 1);
        let pixel_value = compute_pixel(stencil, coord, &image);

        assert_eq!(pixel_value, 78.0);
    }
}
<commit_msg>fixed conflicts<commit_after>// x, y
pub struct Coordinate(pub isize, pub isize);

// In the stencil and the image, the first coordinate is x, the second is y.
// This is computing the convolution between the stencil and the extracted matrix from the
// image
pub fn compute_pixel(stencil: &Vec<Vec<f32>>, coordinate: Coordinate, image_ref: &Vec<Vec<(f32, f32, f32)>>) -> (f32, f32, f32) {
    let stencil_size = (stencil.len(), stencil[0].len());

    let extracted_matrix = extract_matrix(image_ref, coordinate, stencil_size);

    let mut result: (f32, f32, f32) = (0.0, 0.0, 0.0);

    for i in 0..stencil_size.1 {
        for j in 0..stencil_size.0 {
            result.0 += extracted_matrix[i][j].0 * stencil[i][j];
            result.1 += extracted_matrix[i][j].1 * stencil[i][j];
            result.2 += extracted_matrix[i][j].2 * stencil[i][j];
        }
    }
    result
}

// Extracts the pixels values for a given stencil dimensions from an image
pub fn extract_matrix(image_ref: &Vec<Vec<(f32, f32, f32)>>, coordinate: Coordinate, dimensions: (usize, usize))
-> Vec<Vec<(f32, f32, f32)>>{

    let delta_x = ((dimensions.0 - 1) as f32 / 2.0) as isize;
    let delta_y = ((dimensions.1 - 1) as f32 / 2.0) as isize;

    let width = image_ref[0].len() as isize;
    let height = image_ref.len() as isize;

    let mut extracted_matrix: Vec<Vec<(f32, f32, f32)>> = Vec::with_capacity(dimensions.1);

    for (i, y) in ((coordinate.1 - delta_y)..(coordinate.1 + delta_y + 1)).enumerate() {
        extracted_matrix.push(Vec::with_capacity(dimensions.0));
        for x in (coordinate.0 - delta_x)..(coordinate.0 + delta_x + 1) {
            if x < 0 || y < 0 || x >= width || y >= height {
                extracted_matrix[i].push((0.0, 0.0, 0.0));
            } else {
                extracted_matrix[i].push(image_ref[y as usize][x as usize]);
            }
        }
    }

    extracted_matrix
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn matrix_regular_extraction() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let stencil_size = (stencil.len(), stencil[0].len());
        let coord = Coordinate(1, 1);
        let extracted = extract_matrix(&image, coord, stencil_size);

        assert_eq!(extracted.len(), 3);
        assert_eq!(extracted[0].len(), 3);
        assert_eq!(extracted[0][0], (1.0,3.0,4.0));
        assert_eq!(extracted[0][1], (5.0,6.0,3.0));
        assert_eq!(extracted[1][0], (1.0,3.0,4.0));
        assert_eq!(extracted[1][1], (5.0,6.0,3.0));
        assert_eq!(extracted[2][2], (4.0,5.0,6.0));
    }

    #[test]
    fn matrix_lower_border_extraction() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let stencil_size = (stencil.len(), stencil[0].len());
        let coord = Coordinate(0, 0);
        let extracted = extract_matrix(&image, coord, stencil_size);

        assert_eq!(extracted.len(), 3);
        assert_eq!(extracted[0].len(), 3);
        assert_eq!(extracted[0][0], (0.0,0.0,0.0));
        assert_eq!(extracted[0][1], (0.0,0.0,0.0));
        assert_eq!(extracted[1][0], (0.0,0.0,0.0));
        assert_eq!(extracted[1][1], (1.0,3.0,4.0));
        assert_eq!(extracted[2][2], (5.0,6.0,3.0));
    }

    #[test]
    fn matrix_higher_border_extraction() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let stencil_size = (stencil.len(), stencil[0].len());
        let coord = Coordinate(2, 8);
        let extracted = extract_matrix(&image, coord, stencil_size);

        assert_eq!(extracted.len(), 3);
        assert_eq!(extracted[0].len(), 3);
        assert_eq!(extracted[0][0], (5.0, 6.0, 3.0));
        assert_eq!(extracted[0][1], (4.0, 5.0, 6.0));
        assert_eq!(extracted[1][0], (5.0, 6.0, 3.0));
        assert_eq!(extracted[1][1], (4.0, 5.0, 6.0));
        assert_eq!(extracted[2][2], (0.0, 0.0, 0.0));
    }

    #[test]
    fn compute_pixel_test() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0)),
            vec!((1.0,3.0,4.0), (5.0,6.0,3.0), (4.0,5.0,6.0))
        );

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let coord = Coordinate(1, 1);
        let pixel_value = compute_pixel(&stencil, coord, &image);

        assert_eq!(pixel_value, (96.0, 123.0, 111.0));
    }

    #[test]
    fn compute_mono_pixel_test() {
        let image: Vec<Vec<(f32, f32, f32)>>= vec!(vec!((1.0,3.0,4.0)));

        let stencil: Vec<Vec<f32>> = vec!(
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0),
            vec!(1.0,3.0,4.0)
        );
        let coord = Coordinate(0, 0);
        let pixel_value = compute_pixel(&stencil, coord, &image);

        assert_eq!(pixel_value, (3.0, 9.0, 12.0));
    }
}
<|endoftext|>


<commit_before>// This benchmark suite contains some benchmarks along a set of dimensions:
//   Hasher: std default (SipHash) and crate default (AHash).
//   Int key distribution: low bit heavy, top bit heavy, and random.
//   Task: basic functionality: insert, insert_erase, lookup, lookup_fail, iter
#![feature(test)]

extern crate test;

use test::{black_box, Bencher};

use hashbrown::hash_map::DefaultHashBuilder;
use hashbrown::HashMap;
use std::collections::hash_map::RandomState;

const SIZE: usize = 1000;

// The default hashmap when using this crate directly.
type AHashMap<K, V> = HashMap<K, V, DefaultHashBuilder>;
// This uses the hashmap from this crate with the default hasher of the stdlib.
type StdHashMap<K, V> = HashMap<K, V, RandomState>;

// A random key iterator.
#[derive(Clone, Copy)]
struct RandomKeys {
    state: usize,
}

impl RandomKeys {
    fn new() -> Self {
        RandomKeys { state: 0 }
    }
}

impl Iterator for RandomKeys {
    type Item = usize;
    fn next(&mut self) -> Option<usize> {
        // Add 1 then multiply by some 32 bit prime.
        self.state = self.state.wrapping_add(1).wrapping_mul(3787392781);
        Some(self.state)
    }
}

macro_rules! bench_suite {
    ($bench_macro:ident, $bench_ahash_serial:ident, $bench_std_serial:ident,
     $bench_ahash_highbits:ident, $bench_std_highbits:ident,
     $bench_ahash_random:ident, $bench_std_random:ident) => {
        $bench_macro!($bench_ahash_serial, AHashMap, 0..);
        $bench_macro!($bench_std_serial, StdHashMap, 0..);
        $bench_macro!(
            $bench_ahash_highbits,
            AHashMap,
            (0..).map(usize::swap_bytes)
        );
        $bench_macro!(
            $bench_std_highbits,
            StdHashMap,
            (0..).map(usize::swap_bytes)
        );
        $bench_macro!($bench_ahash_random, AHashMap, RandomKeys::new());
        $bench_macro!($bench_std_random, StdHashMap, RandomKeys::new());
    };
}

macro_rules! bench_insert {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::with_capacity_and_hasher(SIZE, Default::default());
            b.iter(|| {
                m.clear();
                for i in ($keydist).take(SIZE) {
                    m.insert(i, i);
                }
                black_box(&mut m);
            })
        }
    };
}

bench_suite!(
    bench_insert,
    insert_ahash_serial,
    insert_std_serial,
    insert_ahash_highbits,
    insert_std_highbits,
    insert_ahash_random,
    insert_std_random
);

macro_rules! bench_insert_erase {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut base = $maptype::default();
            for i in ($keydist).take(SIZE) {
                base.insert(i, i);
            }
            let skip = $keydist.skip(SIZE);
            b.iter(|| {
                let mut m = base.clone();
                let mut add_iter = skip.clone();
                let mut remove_iter = $keydist;
                // While keeping the size constant,
                // replace the first keydist with the second.
                for (add, remove) in (&mut add_iter).zip(&mut remove_iter).take(SIZE) {
                    m.insert(add, add);
                    black_box(m.remove(&remove));
                }
                black_box(m);
            })
        }
    };
}

bench_suite!(
    bench_insert_erase,
    insert_erase_ahash_serial,
    insert_erase_std_serial,
    insert_erase_ahash_highbits,
    insert_erase_std_highbits,
    insert_erase_ahash_random,
    insert_erase_std_random
);

macro_rules! bench_lookup {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            for i in $keydist.take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in $keydist.take(SIZE) {
                    black_box(m.get(&i));
                }
            })
        }
    };
}

bench_suite!(
    bench_lookup,
    lookup_ahash_serial,
    lookup_std_serial,
    lookup_ahash_highbits,
    lookup_std_highbits,
    lookup_ahash_random,
    lookup_std_random
);

macro_rules! bench_lookup_fail {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            let mut iter = $keydist;
            for i in (&mut iter).take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in (&mut iter).take(SIZE) {
                    black_box(m.get(&i));
                }
            })
        }
    };
}

bench_suite!(
    bench_lookup_fail,
    lookup_fail_ahash_serial,
    lookup_fail_std_serial,
    lookup_fail_ahash_highbits,
    lookup_fail_std_highbits,
    lookup_fail_ahash_random,
    lookup_fail_std_random
);

macro_rules! bench_iter {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            for i in ($keydist).take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in &m {
                    black_box(i);
                }
            })
        }
    };
}

bench_suite!(
    bench_iter,
    iter_ahash_serial,
    iter_std_serial,
    iter_ahash_highbits,
    iter_std_highbits,
    iter_ahash_random,
    iter_std_random
);
<commit_msg>Add benchmarks for clone<commit_after>// This benchmark suite contains some benchmarks along a set of dimensions:
//   Hasher: std default (SipHash) and crate default (AHash).
//   Int key distribution: low bit heavy, top bit heavy, and random.
//   Task: basic functionality: insert, insert_erase, lookup, lookup_fail, iter
#![feature(test)]

extern crate test;

use test::{black_box, Bencher};

use hashbrown::hash_map::DefaultHashBuilder;
use hashbrown::HashMap;
use std::collections::hash_map::RandomState;

const SIZE: usize = 1000;

// The default hashmap when using this crate directly.
type AHashMap<K, V> = HashMap<K, V, DefaultHashBuilder>;
// This uses the hashmap from this crate with the default hasher of the stdlib.
type StdHashMap<K, V> = HashMap<K, V, RandomState>;

// A random key iterator.
#[derive(Clone, Copy)]
struct RandomKeys {
    state: usize,
}

impl RandomKeys {
    fn new() -> Self {
        RandomKeys { state: 0 }
    }
}

impl Iterator for RandomKeys {
    type Item = usize;
    fn next(&mut self) -> Option<usize> {
        // Add 1 then multiply by some 32 bit prime.
        self.state = self.state.wrapping_add(1).wrapping_mul(3787392781);
        Some(self.state)
    }
}

macro_rules! bench_suite {
    ($bench_macro:ident, $bench_ahash_serial:ident, $bench_std_serial:ident,
     $bench_ahash_highbits:ident, $bench_std_highbits:ident,
     $bench_ahash_random:ident, $bench_std_random:ident) => {
        $bench_macro!($bench_ahash_serial, AHashMap, 0..);
        $bench_macro!($bench_std_serial, StdHashMap, 0..);
        $bench_macro!(
            $bench_ahash_highbits,
            AHashMap,
            (0..).map(usize::swap_bytes)
        );
        $bench_macro!(
            $bench_std_highbits,
            StdHashMap,
            (0..).map(usize::swap_bytes)
        );
        $bench_macro!($bench_ahash_random, AHashMap, RandomKeys::new());
        $bench_macro!($bench_std_random, StdHashMap, RandomKeys::new());
    };
}

macro_rules! bench_insert {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::with_capacity_and_hasher(SIZE, Default::default());
            b.iter(|| {
                m.clear();
                for i in ($keydist).take(SIZE) {
                    m.insert(i, i);
                }
                black_box(&mut m);
            })
        }
    };
}

bench_suite!(
    bench_insert,
    insert_ahash_serial,
    insert_std_serial,
    insert_ahash_highbits,
    insert_std_highbits,
    insert_ahash_random,
    insert_std_random
);

macro_rules! bench_insert_erase {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut base = $maptype::default();
            for i in ($keydist).take(SIZE) {
                base.insert(i, i);
            }
            let skip = $keydist.skip(SIZE);
            b.iter(|| {
                let mut m = base.clone();
                let mut add_iter = skip.clone();
                let mut remove_iter = $keydist;
                // While keeping the size constant,
                // replace the first keydist with the second.
                for (add, remove) in (&mut add_iter).zip(&mut remove_iter).take(SIZE) {
                    m.insert(add, add);
                    black_box(m.remove(&remove));
                }
                black_box(m);
            })
        }
    };
}

bench_suite!(
    bench_insert_erase,
    insert_erase_ahash_serial,
    insert_erase_std_serial,
    insert_erase_ahash_highbits,
    insert_erase_std_highbits,
    insert_erase_ahash_random,
    insert_erase_std_random
);

macro_rules! bench_lookup {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            for i in $keydist.take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in $keydist.take(SIZE) {
                    black_box(m.get(&i));
                }
            })
        }
    };
}

bench_suite!(
    bench_lookup,
    lookup_ahash_serial,
    lookup_std_serial,
    lookup_ahash_highbits,
    lookup_std_highbits,
    lookup_ahash_random,
    lookup_std_random
);

macro_rules! bench_lookup_fail {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            let mut iter = $keydist;
            for i in (&mut iter).take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in (&mut iter).take(SIZE) {
                    black_box(m.get(&i));
                }
            })
        }
    };
}

bench_suite!(
    bench_lookup_fail,
    lookup_fail_ahash_serial,
    lookup_fail_std_serial,
    lookup_fail_ahash_highbits,
    lookup_fail_std_highbits,
    lookup_fail_ahash_random,
    lookup_fail_std_random
);

macro_rules! bench_iter {
    ($name:ident, $maptype:ident, $keydist:expr) => {
        #[bench]
        fn $name(b: &mut Bencher) {
            let mut m = $maptype::default();
            for i in ($keydist).take(SIZE) {
                m.insert(i, i);
            }

            b.iter(|| {
                for i in &m {
                    black_box(i);
                }
            })
        }
    };
}

bench_suite!(
    bench_iter,
    iter_ahash_serial,
    iter_std_serial,
    iter_ahash_highbits,
    iter_std_highbits,
    iter_ahash_random,
    iter_std_random
);

#[bench]
fn clone_small(b: &mut Bencher) {
    let mut m = HashMap::new();
    for i in 0..10 {
        m.insert(i, i);
    }

    b.iter(|| {
        black_box(m.clone());
    })
}

#[bench]
fn clone_from_small(b: &mut Bencher) {
    let mut m = HashMap::new();
    let mut m2 = HashMap::new();
    for i in 0..10 {
        m.insert(i, i);
    }

    b.iter(|| {
        m2.clone_from(&m);
        black_box(&mut m2);
    })
}

#[bench]
fn clone_large(b: &mut Bencher) {
    let mut m = HashMap::new();
    for i in 0..1000 {
        m.insert(i, i);
    }

    b.iter(|| {
        black_box(m.clone());
    })
}

#[bench]
fn clone_from_large(b: &mut Bencher) {
    let mut m = HashMap::new();
    let mut m2 = HashMap::new();
    for i in 0..1000 {
        m.insert(i, i);
    }

    b.iter(|| {
        m2.clone_from(&m);
        black_box(&mut m2);
    })
}
<|endoftext|>


<commit_before>#![feature(test)]
#![allow(non_snake_case)]
#![allow(clippy::cast_lossless)]

extern crate test;

macro_rules! benches {
    (
        $(
            $(#[$attr:meta])*
            $name:ident($value:expr)
        ),*
    ) => {
        mod bench_itoa_write {
            use test::{Bencher, black_box};
            $(
                $(#[$attr])*
                #[bench]
                fn $name(b: &mut Bencher) {
                    let mut buf = Vec::with_capacity(40);

                    b.iter(|| {
                        buf.clear();
                        itoa::write(&mut buf, black_box($value)).unwrap()
                    });
                }
            )*
        }

        mod bench_itoa_fmt {
            use test::{Bencher, black_box};
            $(
                $(#[$attr])*
                #[bench]
                fn $name(b: &mut Bencher) {
                    let mut buf = String::with_capacity(40);

                    b.iter(|| {
                        buf.clear();
                        itoa::fmt(&mut buf, black_box($value)).unwrap()
                    });
                }
            )*
        }

        mod bench_std_fmt {
            use test::{Bencher, black_box};
            $(
                $(#[$attr])*
                #[bench]
                fn $name(b: &mut Bencher) {
                    use std::io::Write;

                    let mut buf = Vec::with_capacity(40);

                    b.iter(|| {
                        buf.clear();
                        write!(&mut buf, "{}", black_box($value)).unwrap()
                    });
                }
            )*
        }
    }
}

benches! {
    bench_u64_0(0u64),
    bench_u64_half(<u32>::max_value() as u64),
    bench_u64_max(<u64>::max_value()),

    bench_i16_0(0i16),
    bench_i16_min(<i16>::min_value()),

    bench_u128_0(0u128),
    bench_u128_max(<u128>::max_value())
}
<commit_msg>Eliminate itoa::write and itoa::fmt from benches<commit_after>#![feature(test)]
#![allow(non_snake_case)]
#![allow(clippy::cast_lossless)]

extern crate test;

macro_rules! benches {
    (
        $(
            $(#[$attr:meta])*
            $name:ident($value:expr)
        ),*
    ) => {
        mod bench_itoa_format {
            use test::{Bencher, black_box};
            $(
                $(#[$attr])*
                #[bench]
                fn $name(b: &mut Bencher) {
                    let mut buffer = itoa::Buffer::new();

                    b.iter(|| {
                        let printed = buffer.format(black_box($value));
                        black_box(printed);
                    });
                }
            )*
        }

        mod bench_std_fmt {
            use test::{Bencher, black_box};
            $(
                $(#[$attr])*
                #[bench]
                fn $name(b: &mut Bencher) {
                    use std::io::Write;

                    let mut buf = Vec::with_capacity(40);

                    b.iter(|| {
                        buf.clear();
                        write!(&mut buf, "{}", black_box($value)).unwrap();
                        black_box(&buf);
                    });
                }
            )*
        }
    }
}

benches! {
    bench_u64_0(0u64),
    bench_u64_half(<u32>::max_value() as u64),
    bench_u64_max(<u64>::max_value()),

    bench_i16_0(0i16),
    bench_i16_min(<i16>::min_value()),

    bench_u128_0(0u128),
    bench_u128_max(<u128>::max_value())
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A wrapper around another PRNG that reseeds it after it
//! generates a certain number of random bytes.

use core::mem::size_of;

use rand_core::{RngCore, CryptoRng, SeedableRng, Error, ErrorKind};
use rand_core::block::{BlockRngCore, BlockRng};

/// A wrapper around any PRNG which reseeds the underlying PRNG after it has
/// generated a certain number of random bytes.
///
/// When the RNG gets cloned, the clone is reseeded on first use.
///
/// Reseeding is never strictly *necessary*. Cryptographic PRNGs don't have a
/// limited number of bytes they can output, or at least not a limit reachable
/// in any practical way. There is no such thing as 'running out of entropy'.
///
/// Some small non-cryptographic PRNGs can have very small periods, for
/// example less than 264. Would reseeding help to ensure that you do
/// not wrap around at the end of the period? A period of 264 still
/// takes several centuries of CPU-years on current hardware. Reseeding will
/// actually make things worse, because the reseeded PRNG will just continue
/// somewhere else *in the same period*, with a high chance of overlapping with
/// previously used parts of it.
///
/// # When should you use `ReseedingRng`?
///
/// - Reseeding can be seen as some form of 'security in depth'. Even if in the
///   future a cryptographic weakness is found in the CSPRNG being used,
///   occasionally reseeding should make exploiting it much more difficult or
///   even impossible.
/// - It can be used as a poor man's cryptography (not recommended, just use a
///   good CSPRNG). Previous implementations of `thread_rng` for example used
///   `ReseedingRng` with the ISAAC RNG. That algorithm, although apparently
///   strong and with no known attack, does not come with any proof of security
///   and does not meet the current standards for a cryptographically secure
///   PRNG. By reseeding it frequently (every 32 kiB) it seems safe to assume
///   there is no attack that can operate on the tiny window between reseeds.
///
/// # Error handling
///
/// Although extremely unlikely, reseeding the wrapped PRNG can fail.
/// `ReseedingRng` will never panic but try to handle the error intelligently
/// through some combination of retrying and delaying reseeding until later.
/// If handling the source error fails `ReseedingRng` will continue generating
/// data from the wrapped PRNG without reseeding.
#[derive(Debug)]
pub struct ReseedingRng<R, Rsdr>(BlockRng<ReseedingCore<R, Rsdr>>)
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore;

impl<R, Rsdr> ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingRng` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr) -> Self {
        ReseedingRng(BlockRng::new(ReseedingCore::new(rng, threshold, reseeder)))
    }

    /// Reseed the internal PRNG.
    pub fn reseed(&mut self) -> Result<(), Error> {
        self.0.core.reseed()
    }
}

// TODO: this should be implemented for any type where the inner type
// implements RngCore, but we can't specify that because ReseedingCore is private
impl<R, Rsdr: RngCore> RngCore for ReseedingRng<R, Rsdr>
where R: BlockRngCore<Item = u32> + SeedableRng,
    <R as BlockRngCore>::Results: AsRef<[u32]> + AsMut<[u32]>
{
    #[inline(always)]
    fn next_u32(&mut self) -> u32 {
        self.0.next_u32()
    }

    #[inline(always)]
    fn next_u64(&mut self) -> u64 {
        self.0.next_u64()
    }

    fn fill_bytes(&mut self, dest: &mut [u8]) {
        self.0.fill_bytes(dest)
    }

    fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
        self.0.try_fill_bytes(dest)
    }
}

impl<R, Rsdr> Clone for ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng + Clone,
      Rsdr: RngCore + Clone
{
    fn clone(&self) -> ReseedingRng<R, Rsdr> {
        // Recreating `BlockRng` seems easier than cloning it and resetting
        // the index.
        ReseedingRng(BlockRng::new(self.0.core.clone()))
    }
}

impl<R, Rsdr> CryptoRng for ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng + CryptoRng,
      Rsdr: RngCore + CryptoRng {}

#[derive(Debug)]
struct ReseedingCore<R, Rsdr> {
    inner: R,
    reseeder: Rsdr,
    threshold: i64,
    bytes_until_reseed: i64,
}

impl<R, Rsdr> BlockRngCore for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    type Item = <R as BlockRngCore>::Item;
    type Results = <R as BlockRngCore>::Results;

    fn generate(&mut self, results: &mut Self::Results) {
        if self.bytes_until_reseed <= 0 {
            // We get better performance by not calling only `auto_reseed` here
            // and continuing with the rest of the function, but by directly
            // returning from a non-inlined function.
            return self.reseed_and_generate(results);
        }
        let num_bytes = results.as_ref().len() * size_of::<Self::Item>();
        self.bytes_until_reseed -= num_bytes as i64;
        self.inner.generate(results);
    }
}

impl<R, Rsdr> ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingCore` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr) -> Self {
        assert!(threshold <= ::core::i64::MAX as u64);
        ReseedingCore {
            inner: rng,
            reseeder,
            threshold: threshold as i64,
            bytes_until_reseed: threshold as i64,
        }
    }

    /// Reseed the internal PRNG.
    fn reseed(&mut self) -> Result<(), Error> {
        R::from_rng(&mut self.reseeder).map(|result| {
            self.bytes_until_reseed = self.threshold;
            self.inner = result
        })
    }

    #[inline(never)]
    fn reseed_and_generate(&mut self,
                           results: &mut <Self as BlockRngCore>::Results)
    {
        trace!("Reseeding RNG after {} generated bytes",
               self.threshold - self.bytes_until_reseed);
        let threshold = if let Err(e) = self.reseed()  {
            let delay = match e.kind {
                ErrorKind::Transient => 0,
                kind @ _ if kind.should_retry() => self.threshold >> 8,
                _ => self.threshold,
            };
            warn!("Reseeding RNG delayed reseeding by {} bytes due to \
                    error from source: {}", delay, e);
            delay
        } else {
            self.threshold
        };

        let num_bytes = results.as_ref().len() * size_of::<<R as BlockRngCore>::Item>();
        self.bytes_until_reseed = threshold - num_bytes as i64;
        self.inner.generate(results);
    }
}

impl<R, Rsdr> Clone for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng + Clone,
      Rsdr: RngCore + Clone
{
    fn clone(&self) -> ReseedingCore<R, Rsdr> {
        ReseedingCore {
            inner: self.inner.clone(),
            reseeder: self.reseeder.clone(),
            threshold: self.threshold,
            bytes_until_reseed: 0, // reseed clone on first use
        }
    }
}

impl<R, Rsdr> CryptoRng for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng + CryptoRng,
      Rsdr: RngCore + CryptoRng {}

#[cfg(test)]
mod test {
    use {Rng, SeedableRng};
    use prng::chacha::ChaChaCore;
    use rngs::mock::StepRng;
    use super::ReseedingRng;

    #[test]
    fn test_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut reseeding = ReseedingRng::new(rng, 32*4, zero);

        // Currently we only support for arrays up to length 32.
        // TODO: cannot generate seq via Rng::gen because it uses different alg
        let mut buf = [0u32; 32]; // Needs to be a multiple of the RNGs result
                                  // size to test exactly.
        reseeding.fill(&mut buf);
        let seq = buf;
        for _ in 0..10 {
            reseeding.fill(&mut buf);
            assert_eq!(buf, seq);
        }
    }

    #[test]
    fn test_clone_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut rng1 = ReseedingRng::new(rng, 32*4, zero);

        let first: u32 = rng1.gen();
        for _ in 0..10 { let _ = rng1.gen::<u32>(); }

        let mut rng2 = rng1.clone();
        assert_eq!(first, rng2.gen::<u32>());
    }
}
<commit_msg>Add Unix fork protection<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A wrapper around another PRNG that reseeds it after it
//! generates a certain number of random bytes.

use core::mem::size_of;

use rand_core::{RngCore, CryptoRng, SeedableRng, Error, ErrorKind};
use rand_core::block::{BlockRngCore, BlockRng};

#[cfg(all(feature="std", unix, not(target_os="emscripten")))]
extern crate libc;

/// A wrapper around any PRNG which reseeds the underlying PRNG after it has
/// generated a certain number of random bytes.
///
/// When the RNG gets cloned, the clone is reseeded on first use.
///
/// Reseeding is never strictly *necessary*. Cryptographic PRNGs don't have a
/// limited number of bytes they can output, or at least not a limit reachable
/// in any practical way. There is no such thing as 'running out of entropy'.
///
/// Some small non-cryptographic PRNGs can have very small periods, for
/// example less than 264. Would reseeding help to ensure that you do
/// not wrap around at the end of the period? A period of 264 still
/// takes several centuries of CPU-years on current hardware. Reseeding will
/// actually make things worse, because the reseeded PRNG will just continue
/// somewhere else *in the same period*, with a high chance of overlapping with
/// previously used parts of it.
///
/// # When should you use `ReseedingRng`?
///
/// - Reseeding can be seen as some form of 'security in depth'. Even if in the
///   future a cryptographic weakness is found in the CSPRNG being used,
///   occasionally reseeding should make exploiting it much more difficult or
///   even impossible.
/// - It can be used as a poor man's cryptography (not recommended, just use a
///   good CSPRNG). Previous implementations of `thread_rng` for example used
///   `ReseedingRng` with the ISAAC RNG. That algorithm, although apparently
///   strong and with no known attack, does not come with any proof of security
///   and does not meet the current standards for a cryptographically secure
///   PRNG. By reseeding it frequently (every 32 kiB) it seems safe to assume
///   there is no attack that can operate on the tiny window between reseeds.
///
/// # Error handling
///
/// Although extremely unlikely, reseeding the wrapped PRNG can fail.
/// `ReseedingRng` will never panic but try to handle the error intelligently
/// through some combination of retrying and delaying reseeding until later.
/// If handling the source error fails `ReseedingRng` will continue generating
/// data from the wrapped PRNG without reseeding.
#[derive(Debug)]
pub struct ReseedingRng<R, Rsdr>(BlockRng<ReseedingCore<R, Rsdr>>)
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore;

impl<R, Rsdr> ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingRng` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr) -> Self {
        ReseedingRng(BlockRng::new(ReseedingCore::new(rng, threshold, reseeder)))
    }

    /// Reseed the internal PRNG.
    pub fn reseed(&mut self) -> Result<(), Error> {
        self.0.core.reseed()
    }
}

// TODO: this should be implemented for any type where the inner type
// implements RngCore, but we can't specify that because ReseedingCore is private
impl<R, Rsdr: RngCore> RngCore for ReseedingRng<R, Rsdr>
where R: BlockRngCore<Item = u32> + SeedableRng,
    <R as BlockRngCore>::Results: AsRef<[u32]> + AsMut<[u32]>
{
    #[inline(always)]
    fn next_u32(&mut self) -> u32 {
        self.0.next_u32()
    }

    #[inline(always)]
    fn next_u64(&mut self) -> u64 {
        self.0.next_u64()
    }

    fn fill_bytes(&mut self, dest: &mut [u8]) {
        self.0.fill_bytes(dest)
    }

    fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
        self.0.try_fill_bytes(dest)
    }
}

impl<R, Rsdr> Clone for ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng + Clone,
      Rsdr: RngCore + Clone
{
    fn clone(&self) -> ReseedingRng<R, Rsdr> {
        // Recreating `BlockRng` seems easier than cloning it and resetting
        // the index.
        ReseedingRng(BlockRng::new(self.0.core.clone()))
    }
}

impl<R, Rsdr> CryptoRng for ReseedingRng<R, Rsdr>
where R: BlockRngCore + SeedableRng + CryptoRng,
      Rsdr: RngCore + CryptoRng {}

#[derive(Debug)]
struct ReseedingCore<R, Rsdr> {
    inner: R,
    reseeder: Rsdr,
    threshold: i64,
    bytes_until_reseed: i64,
    fork_counter: u64,
}

impl<R, Rsdr> BlockRngCore for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    type Item = <R as BlockRngCore>::Item;
    type Results = <R as BlockRngCore>::Results;

    fn generate(&mut self, results: &mut Self::Results) {
        if self.bytes_until_reseed <= 0 ||
           self.fork_counter < get_fork_counter() {
            // We get better performance by not calling only `reseed` here
            // and continuing with the rest of the function, but by directly
            // returning from a non-inlined function.
            return self.reseed_and_generate(results);
        }
        let num_bytes = results.as_ref().len() * size_of::<Self::Item>();
        self.bytes_until_reseed -= num_bytes as i64;
        self.inner.generate(results);
    }
}

impl<R, Rsdr> ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng,
      Rsdr: RngCore
{
    /// Create a new `ReseedingCore` with the given parameters.
    ///
    /// # Arguments
    ///
    /// * `rng`: the random number generator to use.
    /// * `threshold`: the number of generated bytes after which to reseed the RNG.
    /// * `reseeder`: the RNG to use for reseeding.
    pub fn new(rng: R, threshold: u64, reseeder: Rsdr) -> Self {
        assert!(threshold <= ::core::i64::MAX as u64);
        register_fork_handler();

        ReseedingCore {
            inner: rng,
            reseeder,
            threshold: threshold as i64,
            bytes_until_reseed: threshold as i64,
            fork_counter: 0,
        }
    }

    /// Reseed the internal PRNG.
    fn reseed(&mut self) -> Result<(), Error> {
        R::from_rng(&mut self.reseeder).map(|result| {
            self.bytes_until_reseed = self.threshold;
            self.inner = result
        })
    }

    #[inline(never)]
    fn reseed_and_generate(&mut self,
                           results: &mut <Self as BlockRngCore>::Results)
    {
        let fork_counter = get_fork_counter();
        if self.fork_counter < fork_counter {
            warn!("Fork detected, reseeding RNG");
        } else {
            trace!("Reseeding RNG after {} generated bytes",
                   self.threshold - self.bytes_until_reseed);
        }

        let threshold = if let Err(e) = self.reseed() {
            let delay = match e.kind {
                ErrorKind::Transient => 0,
                kind @ _ if kind.should_retry() => self.threshold >> 8,
                _ => self.threshold,
            };
            warn!("Reseeding RNG delayed reseeding by {} bytes due to \
                    error from source: {}", delay, e);
            delay
        } else {
            let num_bytes =
                results.as_ref().len() * size_of::<<R as BlockRngCore>::Item>();
            self.fork_counter = fork_counter;
            self.threshold - num_bytes as i64
        };

        self.bytes_until_reseed = threshold;
        self.inner.generate(results);
    }
}

impl<R, Rsdr> Clone for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng + Clone,
      Rsdr: RngCore + Clone
{
    fn clone(&self) -> ReseedingCore<R, Rsdr> {
        ReseedingCore {
            inner: self.inner.clone(),
            reseeder: self.reseeder.clone(),
            threshold: self.threshold,
            bytes_until_reseed: 0, // reseed clone on first use
            fork_counter: self.fork_counter,
        }
    }
}

impl<R, Rsdr> CryptoRng for ReseedingCore<R, Rsdr>
where R: BlockRngCore + SeedableRng + CryptoRng,
      Rsdr: RngCore + CryptoRng {}

// Fork protection
//
// We implement fork protection on Unix using `pthread_atfork`.
// When the process is forked, we increment `RESEEDING_RNG_FORK_COUNTER`.
// Every `ReseedingRng` stores the last known value of the static in
// `fork_counter`. If the cached `fork_counter` is less than
// `RESEEDING_RNG_FORK_COUNTER`, it is time to reseed this RNG.
//
// If reseeding fails, we don't deal with this by setting a delay, but just
// don't update `fork_counter`, so a reseed is attempted a soon as possible.
#[cfg(all(feature="std", unix, not(target_os="emscripten")))]
static mut RESEEDING_RNG_FORK_COUNTER: u64 = ::core::u64::MAX;

#[cfg(all(feature="std", unix, not(target_os="emscripten")))]
extern fn forkhandler() {
    unsafe { RESEEDING_RNG_FORK_COUNTER += 1; }
}

#[cfg(all(feature="std", unix, not(target_os="emscripten")))]
fn register_fork_handler() {
    if get_fork_counter() == ::core::u64::MAX {
        unsafe {
            RESEEDING_RNG_FORK_COUNTER = 0;
            libc::pthread_atfork(None, None, Some(forkhandler));
        }
    }
}
#[cfg(not(all(feature="std", unix, not(target_os="emscripten"))))]
fn register_fork_handler() {}

#[cfg(all(feature="std", unix, not(target_os="emscripten")))]
fn get_fork_counter() -> u64 { unsafe { RESEEDING_RNG_FORK_COUNTER } }
#[cfg(not(all(feature="std", unix, not(target_os="emscripten"))))]
fn get_fork_counter() -> u64 { 0 }

#[cfg(test)]
mod test {
    use {Rng, SeedableRng};
    use prng::chacha::ChaChaCore;
    use rngs::mock::StepRng;
    use super::ReseedingRng;

    #[test]
    fn test_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut reseeding = ReseedingRng::new(rng, 32*4, zero);

        // Currently we only support for arrays up to length 32.
        // TODO: cannot generate seq via Rng::gen because it uses different alg
        let mut buf = [0u32; 32]; // Needs to be a multiple of the RNGs result
                                  // size to test exactly.
        reseeding.fill(&mut buf);
        let seq = buf;
        for _ in 0..10 {
            reseeding.fill(&mut buf);
            assert_eq!(buf, seq);
        }
    }

    #[test]
    fn test_clone_reseeding() {
        let mut zero = StepRng::new(0, 0);
        let rng = ChaChaCore::from_rng(&mut zero).unwrap();
        let mut rng1 = ReseedingRng::new(rng, 32*4, zero);

        let first: u32 = rng1.gen();
        for _ in 0..10 { let _ = rng1.gen::<u32>(); }

        let mut rng2 = rng1.clone();
        assert_eq!(first, rng2.gen::<u32>());
    }
}
<|endoftext|>