<commit_before>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Build a block to mine: gathers transactions from the pool, assembles
//! them into a block and returns it.

use chrono::prelude::{DateTime, NaiveDateTime, Utc};
use rand::{thread_rng, Rng};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use util::RwLock;

use chain;
use common::types::Error;
use core::core::verifier_cache::VerifierCache;
use core::{consensus, core, ser};
use keychain::{ExtKeychain, Identifier, Keychain};
use pool;
use util;
use wallet::{self, BlockFees};

// Ensure a block suitable for mining is built and returned
// If a wallet listener URL is not provided the reward will be "burnt"
// Warning: This call does not return until/unless a new block can be built
pub fn get_block(
	chain: &Arc<chain::Chain>,
	tx_pool: &Arc<RwLock<pool::TransactionPool>>,
	verifier_cache: Arc<RwLock<VerifierCache>>,
	key_id: Option<Identifier>,
	wallet_listener_url: Option<String>,
) -> (core::Block, BlockFees) {
	let wallet_retry_interval = 5;
	// get the latest chain state and build a block on top of it
	let mut result = build_block(
		chain,
		tx_pool,
		verifier_cache.clone(),
		key_id.clone(),
		wallet_listener_url.clone(),
	);
	while let Err(e) = result {
		match e {
			self::Error::Chain(c) => match c.kind() {
				chain::ErrorKind::DuplicateCommitment(_) => {
					debug!(
						"Duplicate commit for potential coinbase detected. Trying next derivation."
					);
				}
				_ => {
					error!("Chain Error: {}", c);
				}
			},
			self::Error::Wallet(_) => {
				error!(
					"Error building new block: Can't connect to wallet listener at {:?}; will retry",
					wallet_listener_url.as_ref().unwrap()
				);
				thread::sleep(Duration::from_secs(wallet_retry_interval));
			}
			ae => {
				warn!("Error building new block: {:?}. Retrying.", ae);
			}
		}
		thread::sleep(Duration::from_millis(100));
		result = build_block(
			chain,
			tx_pool,
			verifier_cache.clone(),
			key_id.clone(),
			wallet_listener_url.clone(),
		);
	}
	return result.unwrap();
}

/// Builds a new block with the chain head as previous and eligible
/// transactions from the pool.
fn build_block(
	chain: &Arc<chain::Chain>,
	tx_pool: &Arc<RwLock<pool::TransactionPool>>,
	verifier_cache: Arc<RwLock<VerifierCache>>,
	key_id: Option<Identifier>,
	wallet_listener_url: Option<String>,
) -> Result<(core::Block, BlockFees), Error> {
	let head = chain.head_header()?;

	// prepare the block header timestamp
	let mut now_sec = Utc::now().timestamp();
	let head_sec = head.timestamp.timestamp();
	if now_sec <= head_sec {
		now_sec = head_sec + 1;
	}

	// Determine the difficulty our block should be at.
	// Note: do not keep the difficulty_iter in scope (it has an active batch).
	let difficulty = consensus::next_difficulty(1, chain.difficulty_iter());

	// extract current transaction from the pool
	// TODO - we have a lot of unwrap() going on in this fn...
	let txs = tx_pool.read().prepare_mineable_transactions().unwrap();

	// build the coinbase and the block itself
	let fees = txs.iter().map(|tx| tx.fee()).sum();
	let height = head.height + 1;
	let block_fees = BlockFees {
		fees,
		key_id,
		height,
	};

	let (output, kernel, block_fees) = get_coinbase(wallet_listener_url, block_fees)?;
	let mut b = core::Block::with_reward(&head, txs, output, kernel, difficulty.difficulty)?;

	// making sure we're not spending time mining a useless block
	b.validate(&head.total_kernel_offset, verifier_cache)?;

	b.header.pow.nonce = thread_rng().gen();
	b.header.pow.scaling_difficulty = difficulty.secondary_scaling;
	b.header.timestamp = DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(now_sec, 0), Utc);

	let b_difficulty = (b.header.total_difficulty() - head.total_difficulty()).to_num();
	debug!(
		"Built new block with {} inputs and {} outputs, network difficulty: {}, cumulative difficulty {}",
		b.inputs().len(),
		b.outputs().len(),
		b_difficulty,
		b.header.total_difficulty().to_num(),
	);

	// Now set txhashset roots and sizes on the header of the block being built.
	let roots_result = chain.set_txhashset_roots(&mut b, false);

	match roots_result {
		Ok(_) => Ok((b, block_fees)),

		// If it's a duplicate commitment, it's likely trying to use
		// a key that's already been derived but not in the wallet
		// for some reason, allow caller to retry
		Err(e) => {
			match e.kind() {
				chain::ErrorKind::DuplicateCommitment(e) => Err(Error::Chain(
					chain::ErrorKind::DuplicateCommitment(e).into(),
				)),

				//Some other issue, possibly duplicate kernel
				_ => {
					error!("Error setting txhashset root to build a block: {:?}", e);
					Err(Error::Chain(
						chain::ErrorKind::Other(format!("{:?}", e)).into(),
					))
				}
			}
		}
	}
}

///
/// Probably only want to do this when testing.
///
fn burn_reward(block_fees: BlockFees) -> Result<(core::Output, core::TxKernel, BlockFees), Error> {
	warn!("Burning block fees: {:?}", block_fees);
	let keychain = ExtKeychain::from_random_seed().unwrap();
	let key_id = ExtKeychain::derive_key_id(1, 1, 0, 0, 0);
	let (out, kernel) =
		wallet::libtx::reward::output(&keychain, &key_id, block_fees.fees, block_fees.height)
			.unwrap();
	Ok((out, kernel, block_fees))
}

// Connect to the wallet listener and get coinbase.
// Warning: If a wallet listener URL is not provided the reward will be "burnt"
fn get_coinbase(
	wallet_listener_url: Option<String>,
	block_fees: BlockFees,
) -> Result<(core::Output, core::TxKernel, BlockFees), Error> {
	match wallet_listener_url {
		None => {
			// Burn it
			return burn_reward(block_fees);
		}
		Some(wallet_listener_url) => {
			let res = wallet::create_coinbase(&wallet_listener_url, &block_fees)?;
			let out_bin = util::from_hex(res.output).unwrap();
			let kern_bin = util::from_hex(res.kernel).unwrap();
			let key_id_bin = util::from_hex(res.key_id).unwrap();
			let output = ser::deserialize(&mut &out_bin[..]).unwrap();
			let kernel = ser::deserialize(&mut &kern_bin[..]).unwrap();
			let key_id = ser::deserialize(&mut &key_id_bin[..]).unwrap();
			let block_fees = BlockFees {
				key_id: Some(key_id),
				..block_fees
			};

			debug!("get_coinbase: {:?}", block_fees);
			return Ok((output, kernel, block_fees));
		}
	}
}
<commit_msg>Fix scaling diff in blocks for mining (#1796)<commit_after>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Build a block to mine: gathers transactions from the pool, assembles
//! them into a block and returns it.

use chrono::prelude::{DateTime, NaiveDateTime, Utc};
use rand::{thread_rng, Rng};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use util::RwLock;

use chain;
use common::types::Error;
use core::core::verifier_cache::VerifierCache;
use core::{consensus, core, ser};
use keychain::{ExtKeychain, Identifier, Keychain};
use pool;
use util;
use wallet::{self, BlockFees};

// Ensure a block suitable for mining is built and returned
// If a wallet listener URL is not provided the reward will be "burnt"
// Warning: This call does not return until/unless a new block can be built
pub fn get_block(
	chain: &Arc<chain::Chain>,
	tx_pool: &Arc<RwLock<pool::TransactionPool>>,
	verifier_cache: Arc<RwLock<VerifierCache>>,
	key_id: Option<Identifier>,
	wallet_listener_url: Option<String>,
) -> (core::Block, BlockFees) {
	let wallet_retry_interval = 5;
	// get the latest chain state and build a block on top of it
	let mut result = build_block(
		chain,
		tx_pool,
		verifier_cache.clone(),
		key_id.clone(),
		wallet_listener_url.clone(),
	);
	while let Err(e) = result {
		match e {
			self::Error::Chain(c) => match c.kind() {
				chain::ErrorKind::DuplicateCommitment(_) => {
					debug!(
						"Duplicate commit for potential coinbase detected. Trying next derivation."
					);
				}
				_ => {
					error!("Chain Error: {}", c);
				}
			},
			self::Error::Wallet(_) => {
				error!(
					"Error building new block: Can't connect to wallet listener at {:?}; will retry",
					wallet_listener_url.as_ref().unwrap()
				);
				thread::sleep(Duration::from_secs(wallet_retry_interval));
			}
			ae => {
				warn!("Error building new block: {:?}. Retrying.", ae);
			}
		}
		thread::sleep(Duration::from_millis(100));
		result = build_block(
			chain,
			tx_pool,
			verifier_cache.clone(),
			key_id.clone(),
			wallet_listener_url.clone(),
		);
	}
	return result.unwrap();
}

/// Builds a new block with the chain head as previous and eligible
/// transactions from the pool.
fn build_block(
	chain: &Arc<chain::Chain>,
	tx_pool: &Arc<RwLock<pool::TransactionPool>>,
	verifier_cache: Arc<RwLock<VerifierCache>>,
	key_id: Option<Identifier>,
	wallet_listener_url: Option<String>,
) -> Result<(core::Block, BlockFees), Error> {
	let head = chain.head_header()?;

	// prepare the block header timestamp
	let mut now_sec = Utc::now().timestamp();
	let head_sec = head.timestamp.timestamp();
	if now_sec <= head_sec {
		now_sec = head_sec + 1;
	}

	// Determine the difficulty our block should be at.
	// Note: do not keep the difficulty_iter in scope (it has an active batch).
	let difficulty = consensus::next_difficulty(head.height + 1, chain.difficulty_iter());

	// extract current transaction from the pool
	// TODO - we have a lot of unwrap() going on in this fn...
	let txs = tx_pool.read().prepare_mineable_transactions().unwrap();

	// build the coinbase and the block itself
	let fees = txs.iter().map(|tx| tx.fee()).sum();
	let height = head.height + 1;
	let block_fees = BlockFees {
		fees,
		key_id,
		height,
	};

	let (output, kernel, block_fees) = get_coinbase(wallet_listener_url, block_fees)?;
	let mut b = core::Block::with_reward(&head, txs, output, kernel, difficulty.difficulty)?;

	// making sure we're not spending time mining a useless block
	b.validate(&head.total_kernel_offset, verifier_cache)?;

	b.header.pow.nonce = thread_rng().gen();
	b.header.pow.scaling_difficulty = difficulty.secondary_scaling;
	b.header.timestamp = DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(now_sec, 0), Utc);

	let b_difficulty = (b.header.total_difficulty() - head.total_difficulty()).to_num();
	debug!(
		"Built new block with {} inputs and {} outputs, network difficulty: {}, cumulative difficulty {}",
		b.inputs().len(),
		b.outputs().len(),
		b_difficulty,
		b.header.total_difficulty().to_num(),
	);

	// Now set txhashset roots and sizes on the header of the block being built.
	let roots_result = chain.set_txhashset_roots(&mut b, false);

	match roots_result {
		Ok(_) => Ok((b, block_fees)),

		// If it's a duplicate commitment, it's likely trying to use
		// a key that's already been derived but not in the wallet
		// for some reason, allow caller to retry
		Err(e) => {
			match e.kind() {
				chain::ErrorKind::DuplicateCommitment(e) => Err(Error::Chain(
					chain::ErrorKind::DuplicateCommitment(e).into(),
				)),

				//Some other issue, possibly duplicate kernel
				_ => {
					error!("Error setting txhashset root to build a block: {:?}", e);
					Err(Error::Chain(
						chain::ErrorKind::Other(format!("{:?}", e)).into(),
					))
				}
			}
		}
	}
}

///
/// Probably only want to do this when testing.
///
fn burn_reward(block_fees: BlockFees) -> Result<(core::Output, core::TxKernel, BlockFees), Error> {
	warn!("Burning block fees: {:?}", block_fees);
	let keychain = ExtKeychain::from_random_seed().unwrap();
	let key_id = ExtKeychain::derive_key_id(1, 1, 0, 0, 0);
	let (out, kernel) =
		wallet::libtx::reward::output(&keychain, &key_id, block_fees.fees, block_fees.height)
			.unwrap();
	Ok((out, kernel, block_fees))
}

// Connect to the wallet listener and get coinbase.
// Warning: If a wallet listener URL is not provided the reward will be "burnt"
fn get_coinbase(
	wallet_listener_url: Option<String>,
	block_fees: BlockFees,
) -> Result<(core::Output, core::TxKernel, BlockFees), Error> {
	match wallet_listener_url {
		None => {
			// Burn it
			return burn_reward(block_fees);
		}
		Some(wallet_listener_url) => {
			let res = wallet::create_coinbase(&wallet_listener_url, &block_fees)?;
			let out_bin = util::from_hex(res.output).unwrap();
			let kern_bin = util::from_hex(res.kernel).unwrap();
			let key_id_bin = util::from_hex(res.key_id).unwrap();
			let output = ser::deserialize(&mut &out_bin[..]).unwrap();
			let kernel = ser::deserialize(&mut &kern_bin[..]).unwrap();
			let key_id = ser::deserialize(&mut &key_id_bin[..]).unwrap();
			let block_fees = BlockFees {
				key_id: Some(key_id),
				..block_fees
			};

			debug!("get_coinbase: {:?}", block_fees);
			return Ok((output, kernel, block_fees));
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gfx-rs Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate cgmath;
#[macro_use]
extern crate gfx;
extern crate gfx_core;
extern crate gfx_window_glfw;
extern crate glfw;
extern crate time;
extern crate gfx_gl as gl;
extern crate gfx_device_gl;

use time::precise_time_s;
use cgmath::{SquareMatrix, Matrix, Point3, Vector3, Matrix3, Matrix4};
use cgmath::{Transform, Vector4};
pub use gfx::format::{DepthStencil, Rgba8 as ColorFormat};
use glfw::Context;
use gl::Gl;
use gl::types::*;
use std::mem;
use std::ptr;
use std::str;
use std::env;
use std::str::FromStr;
use std::iter::repeat;
use std::ffi::CString;
use gfx_device_gl::{Resources as R, CommandBuffer as CB};
use gfx_core::Device;

gfx_defines!{
    vertex Vertex {
        pos: [f32; 3] = "a_Pos",
    }

    pipeline pipe {
        vbuf: gfx::VertexBuffer<Vertex> = (),
        transform: gfx::Global<[[f32; 4]; 4]> = "u_Transform",
        out_color: gfx::RenderTarget<ColorFormat> = "o_Color",
    }
}

static VERTEX_SRC: &'static [u8] = b"
    #version 150 core
    in vec3 a_Pos;
    uniform mat4 u_Transform;

    void main() {
        gl_Position = u_Transform * vec4(a_Pos, 1.0);
   }
";

static FRAGMENT_SRC: &'static [u8] = b"
    #version 150 core
    out vec4 o_Color;

    void main() {
        o_Color = vec4(1.0, 0.0, 0.0, 1.0);
    }
";

static VERTEX_DATA: &'static [Vertex] = &[
    Vertex { pos: [-1.0, 0.0, -1.0] },
    Vertex { pos: [ 1.0, 0.0, -1.0] },
    Vertex { pos: [-1.0, 0.0,  1.0] },
];

const CLEAR_COLOR: (f32, f32, f32, f32) = (0.3, 0.3, 0.3, 1.0);

//----------------------------------------

fn transform(x: i16, y: i16, proj_view: &Matrix4<f32>) -> Matrix4<f32> {
    let mut model = Matrix4::from(Matrix3::identity() * 0.05);
    model.w = Vector4::new(x as f32 * 0.10,
                           0f32,
                           y as f32 * 0.10,
                           1f32);
    proj_view * model
}

trait Renderer: Drop {
    fn render(&mut self, window: &mut glfw::Window, proj_view: &Matrix4<f32>);
}

struct GFX {
    device:gfx_device_gl::Device,
    encoder: gfx::Encoder<R,CB>,
    data: pipe::Data<R>,
    pso: gfx::PipelineState<R, pipe::Meta>,
    slice: gfx::Slice<R>,
    dimension: i16,
}

impl GFX {
    fn new(window: &mut glfw::Window, dimension: i16) -> Self {
        use gfx::traits::FactoryExt;

        let (device, mut factory, main_color, _) =
            gfx_window_glfw::init(window);
        let encoder: gfx::Encoder<_,_> = factory.create_command_buffer().into();

        let pso = factory.create_pipeline_simple(
            VERTEX_SRC, FRAGMENT_SRC,
            gfx::state::CullFace::Nothing,
            pipe::new()
        ).unwrap();

        let (vbuf, slice) = factory.create_vertex_buffer_with_slice(VERTEX_DATA,());

        let data = pipe::Data {
            vbuf: vbuf,
            transform: cgmath::Matrix4::identity().into(),
            out_color: main_color,
        };

        GFX {
            device: device,
            encoder: encoder,
            data: data,
            pso: pso,
            slice: slice,
            dimension: dimension,
        }
    }
}

impl Renderer for GFX {
    fn render(&mut self, window: &mut glfw::Window, proj_view:&Matrix4<f32>) {
        let start = precise_time_s() * 1000.;
        self.encoder.clear(&self.data.out_color, [CLEAR_COLOR.0,
                                                  CLEAR_COLOR.1,
                                                  CLEAR_COLOR.2,
                                                  CLEAR_COLOR.3]);

        for x in (-self.dimension) ..self.dimension {
            for y in (-self.dimension) ..self.dimension {
                self.data.transform = transform(x, y, proj_view).into();
                self.encoder.draw(&self.slice, &self.pso, &self.data);
            }
        }

        let pre_submit = precise_time_s() * 1000.;
        self.encoder.flush(&mut self.device);
        let post_submit = precise_time_s() * 1000.;
        window.swap_buffers();
        self.device.cleanup();
        let swap = precise_time_s() * 1000.;

        println!("total time:\t\t{0:4.2}ms", swap - start);
        println!("\tcreate list:\t{0:4.2}ms", pre_submit - start);
        println!("\tsubmit:\t\t{0:4.2}ms", post_submit - pre_submit);
        println!("\tgpu wait:\t{0:4.2}ms", swap - post_submit);
    }
}

impl Drop for GFX {
    fn drop(&mut self) {
    }
}

struct GL {
    gl:Gl,
    trans_uniform:GLint,
    vs:GLuint,
    fs:GLuint,
    program:GLuint,
    vbo:GLuint,
    vao:GLuint,
    dimension:i16,
}

impl GL {
    fn new(window: &mut glfw::Window, dimension: i16) -> Self {
        let gl = Gl::load_with(|s| window.get_proc_address(s) as *const _);

        fn compile_shader (gl:&Gl, src: &[u8], ty: GLenum) -> GLuint {
            unsafe {
                let shader = gl.CreateShader(ty);
                // Attempt to compile the shader
                gl.ShaderSource(shader, 1,
                                &(src.as_ptr() as *const i8),
                                &(src.len() as GLint));
                gl.CompileShader(shader);

                // Get the compile status
                let mut status = gl::FALSE as GLint;
                gl.GetShaderiv(shader, gl::COMPILE_STATUS, &mut status);

                // Fail on error
                if status != (gl::TRUE as GLint) {
                    let mut len: GLint = 0;
                    gl.GetShaderiv(shader, gl::INFO_LOG_LENGTH, &mut len);
                    let mut buf: Vec<u8> = repeat(0u8).take((len as isize).saturating_sub(1) as usize).collect();     // subtract 1 to skip the trailing null character
                    gl.GetShaderInfoLog(shader, len, ptr::null_mut(), buf.as_mut_ptr() as *mut GLchar);
                    panic!("{}", str::from_utf8(&buf).ok().expect("ShaderInfoLog not valid utf8"));
                }
                shader
            }
        };

        // Create GLSL shaders
        let vs = compile_shader(&gl, VERTEX_SRC, gl::VERTEX_SHADER);
        let fs = compile_shader(&gl, FRAGMENT_SRC, gl::FRAGMENT_SHADER);

        // Link program
        let program;
        unsafe {
            program = gl.CreateProgram();
            gl.AttachShader(program, vs);
            gl.AttachShader(program, fs);
            gl.LinkProgram(program);
            // Get the link status
            let mut status = gl::FALSE as GLint;
            gl.GetProgramiv(program, gl::LINK_STATUS, &mut status);

            // Fail on error
            if status != (gl::TRUE as GLint) {
                let mut len: GLint = 0;
                gl.GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len);
                let mut buf: Vec<u8> = repeat(0u8).take((len as isize).saturating_sub(1) as usize).collect();     // subtract 1 to skip the trailing null character
                gl.GetProgramInfoLog(program, len, ptr::null_mut(), buf.as_mut_ptr() as *mut GLchar);
                panic!("{}", str::from_utf8(&buf).ok().expect("ProgramInfoLog not valid utf8"));
            }
        }

        let mut vao = 0;
        let mut vbo = 0;

        let trans_uniform;
        unsafe {
            // Create Vertex Array Object
            gl.GenVertexArrays(1, &mut vao);
            gl.BindVertexArray(vao);

            // Create a Vertex Buffer Object and copy the vertex data to it
            gl.GenBuffers(1, &mut vbo);
            gl.BindBuffer(gl::ARRAY_BUFFER, vbo);

            gl.BufferData(gl::ARRAY_BUFFER,
                          (VERTEX_DATA.len() * mem::size_of::<Vertex>()) as GLsizeiptr,
                          mem::transmute(&VERTEX_DATA[0]),
                          gl::STATIC_DRAW);

            // Use shader program
            gl.UseProgram(program);
            let o_color = CString::new("o_Color").unwrap();
            gl.BindFragDataLocation(program, 0, o_color.as_bytes_with_nul().as_ptr() as *const i8);

            // Specify the layout of the vertex data
            let a_pos = CString::new("a_Pos").unwrap();
            gl.BindFragDataLocation(program, 0, a_pos.as_bytes_with_nul().as_ptr() as *const i8);

            let pos_attr = gl.GetAttribLocation(program, a_pos.as_ptr());
            gl.EnableVertexAttribArray(pos_attr as GLuint);
            gl.VertexAttribPointer(pos_attr as GLuint, 3, gl::FLOAT,
                                   gl::FALSE as GLboolean, 0, ptr::null());

            let u_transform = CString::new("u_Transform").unwrap();
            trans_uniform = gl.GetUniformLocation(program, u_transform.as_bytes_with_nul().as_ptr() as *const i8)
        };

        GL {
            gl: gl,
            vs: vs,
            fs: fs,
            program: program,
            vbo: vbo,
            vao: vao,
            trans_uniform: trans_uniform,
            dimension: dimension,
        }
    }
}

impl Renderer for GL {
    fn render(&mut self, window: &mut glfw::Window, proj_view: &Matrix4<f32>) {
        let start = precise_time_s() * 1000.;

        // Clear the screen to black
        unsafe {
            self.gl.ClearColor(CLEAR_COLOR.0, CLEAR_COLOR.1, CLEAR_COLOR.2, CLEAR_COLOR.3);
            self.gl.Clear(gl::COLOR_BUFFER_BIT);
        }

        for x in (-self.dimension) ..self.dimension {
            for y in (-self.dimension) ..self.dimension {
                let mat:Matrix4<f32> = transform(x, y, proj_view).into();

                unsafe {
                    self.gl.UniformMatrix4fv(self.trans_uniform,
                                             1,
                                             gl::FALSE,
                                             mat.as_ptr());
                    self.gl.DrawArrays(gl::TRIANGLES, 0, 3);
                }

            }
        }

        let submit = precise_time_s() * 1000.;

        // Swap buffers
        window.swap_buffers();
        let swap = precise_time_s() * 1000.;

        println!("total time:\t\t{0:4.2}ms", swap - start);
        println!("\tsubmit:\t\t{0:4.2}ms", submit - start);
        println!("\tgpu wait:\t{0:4.2}ms", swap - submit)
    }
}

impl Drop for GL {
    fn drop(&mut self) {
        unsafe {
            self.gl.DeleteProgram(self.program);
            self.gl.DeleteShader(self.fs);
            self.gl.DeleteShader(self.vs);
            self.gl.DeleteBuffers(1, &self.vbo);
            self.gl.DeleteVertexArrays(1, &self.vao);
        }
    }
}

fn main() {
    let ref mut args = env::args();
    let args_count = env::args().count();
    if args_count == 1 {
        println!("gfx-perf [gl|gfx] <size>");
        return;
    }

    let mode = args.nth(1).unwrap();
    let count: i32 = if args_count == 3 {
        FromStr::from_str(&args.next().unwrap()).ok()
    } else {
        None
    }.unwrap_or(10000);

    let count = ((count as f64).sqrt() / 2.) as i16;

    let mut glfw = glfw::init(glfw::FAIL_ON_ERRORS)
        .ok().expect("Failed to initialize glfw-rs");

    glfw.window_hint(glfw::WindowHint::ContextVersion(3, 2));
    glfw.window_hint(glfw::WindowHint::OpenGlForwardCompat(true));
    glfw.window_hint(glfw::WindowHint::OpenGlProfile(glfw::OpenGlProfileHint::Core));

    let (mut window, events) = glfw
        .create_window(640, 480, "Performance example", glfw::WindowMode::Windowed)
        .expect("Failed to create GLFW window.");

    window.make_current();
    glfw.set_error_callback(glfw::FAIL_ON_ERRORS);
    window.set_key_polling(true);

    let proj_view = {
        let view = Matrix4::look_at(
            Point3::new(0f32, 5.0, -5.0),
            Point3::new(0f32, 0.0, 0.0),
            Vector3::unit_z(),
        );

        let proj = {
            let aspect = {
                let (w, h) = window.get_framebuffer_size();
                w as f32 / h as f32
            };
            cgmath::perspective(cgmath::deg(45.0f32), aspect, 1.0, 10.0)
        };
        proj * view
    };

    let mut r: Box<Renderer>;
    match mode.as_ref() {
        "gfx" => r = Box::new(GFX::new(&mut window, count)),
        "gl" => r = Box::new(GL::new(&mut window, count)),
        x => {
            panic!("{} is not a known mode", x)
        }
    }

    println!("count is {}", count*count*4);

    while !window.should_close() {
        // Poll events
        glfw.poll_events();
        for (_, event) in glfw::flush_messages(&events) {
            match event {
                glfw::WindowEvent::Key(glfw::Key::Escape, _, glfw::Action::Press, _) =>
                    window.set_should_close(true),
                _ => {},
            }
        }
        r.render(&mut window, &proj_view);
    }
}
<commit_msg>Port performance example to glutin<commit_after>// Copyright 2014 The Gfx-rs Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate cgmath;
#[macro_use]
extern crate gfx;
extern crate gfx_core;
extern crate gfx_window_glutin;
extern crate glutin;
extern crate time;
extern crate gfx_gl as gl;
extern crate gfx_device_gl;

use time::precise_time_s;
use cgmath::{SquareMatrix, Matrix, Point3, Vector3, Matrix3, Matrix4};
use cgmath::{Transform, Vector4};
pub use gfx::format::{DepthStencil, Rgba8 as ColorFormat};
//use glfw::Context;
use gl::Gl;
use gl::types::*;
use std::mem;
use std::ptr;
use std::str;
use std::env;
use std::str::FromStr;
use std::iter::repeat;
use std::ffi::CString;
use gfx_device_gl::{Resources as R, CommandBuffer as CB};
use gfx_core::Device;

gfx_defines!{
    vertex Vertex {
        pos: [f32; 3] = "a_Pos",
    }

    pipeline pipe {
        vbuf: gfx::VertexBuffer<Vertex> = (),
        transform: gfx::Global<[[f32; 4]; 4]> = "u_Transform",
        out_color: gfx::RenderTarget<ColorFormat> = "o_Color",
    }
}

static VERTEX_SRC: &'static [u8] = b"
    #version 150 core
    in vec3 a_Pos;
    uniform mat4 u_Transform;

    void main() {
        gl_Position = u_Transform * vec4(a_Pos, 1.0);
   }
";

static FRAGMENT_SRC: &'static [u8] = b"
    #version 150 core
    out vec4 o_Color;

    void main() {
        o_Color = vec4(1.0, 0.0, 0.0, 1.0);
    }
";

static VERTEX_DATA: &'static [Vertex] = &[
    Vertex { pos: [-1.0, 0.0, -1.0] },
    Vertex { pos: [ 1.0, 0.0, -1.0] },
    Vertex { pos: [-1.0, 0.0,  1.0] },
];

const CLEAR_COLOR: (f32, f32, f32, f32) = (0.3, 0.3, 0.3, 1.0);

//----------------------------------------

fn transform(x: i16, y: i16, proj_view: &Matrix4<f32>) -> Matrix4<f32> {
    let mut model = Matrix4::from(Matrix3::identity() * 0.05);
    model.w = Vector4::new(x as f32 * 0.10,
                           0f32,
                           y as f32 * 0.10,
                           1f32);
    proj_view * model
}

trait Renderer: Drop {
    fn render(&mut self, proj_view: &Matrix4<f32>);
    fn window(&mut self) -> &glutin::Window;
}

struct GFX {
    dimension: i16,
    window: glutin::Window,
    device:gfx_device_gl::Device,
    encoder: gfx::Encoder<R,CB>,
    data: pipe::Data<R>,
    pso: gfx::PipelineState<R, pipe::Meta>,
    slice: gfx::Slice<R>,
}

struct GL {
    dimension: i16,
    window: glutin::Window,
    gl:Gl,
    trans_uniform:GLint,
    vs:GLuint,
    fs:GLuint,
    program:GLuint,
    vbo:GLuint,
    vao:GLuint,
}

impl GFX {
    fn new(builder: glutin::WindowBuilder, dimension: i16) -> Self {
        use gfx::traits::FactoryExt;

        let (window, device, mut factory, main_color, _) =
            gfx_window_glutin::init::<ColorFormat, DepthStencil>(builder);
        let encoder: gfx::Encoder<_,_> = factory.create_command_buffer().into();

        let pso = factory.create_pipeline_simple(
            VERTEX_SRC, FRAGMENT_SRC,
            gfx::state::CullFace::Nothing,
            pipe::new()
        ).unwrap();

        let (vbuf, slice) = factory.create_vertex_buffer_with_slice(VERTEX_DATA,());

        let data = pipe::Data {
            vbuf: vbuf,
            transform: cgmath::Matrix4::identity().into(),
            out_color: main_color,
        };

        GFX {
            window: window,
            dimension: dimension,
            device: device,
            encoder: encoder,
            data: data,
            pso: pso,
            slice: slice,
        }
    }
}

impl Renderer for GFX {
    fn render(&mut self, proj_view: &Matrix4<f32>) {
        let start = precise_time_s() * 1000.;
        self.encoder.clear(&self.data.out_color, [CLEAR_COLOR.0,
                                                  CLEAR_COLOR.1,
                                                  CLEAR_COLOR.2,
                                                  CLEAR_COLOR.3]);

        for x in (-self.dimension) ..self.dimension {
            for y in (-self.dimension) ..self.dimension {
                self.data.transform = transform(x, y, proj_view).into();
                self.encoder.draw(&self.slice, &self.pso, &self.data);
            }
        }

        let pre_submit = precise_time_s() * 1000.;
        self.encoder.flush(&mut self.device);
        let post_submit = precise_time_s() * 1000.;
        self.window.swap_buffers().unwrap();
        self.device.cleanup();
        let swap = precise_time_s() * 1000.;

        println!("total time:\t\t{0:4.2}ms", swap - start);
        println!("\tcreate list:\t{0:4.2}ms", pre_submit - start);
        println!("\tsubmit:\t\t{0:4.2}ms", post_submit - pre_submit);
        println!("\tgpu wait:\t{0:4.2}ms", swap - post_submit);
    }
    fn window(&mut self) -> &glutin::Window { &self.window }
}

impl Drop for GFX {
    fn drop(&mut self) {
    }
}

impl GL {
    fn new(builder: glutin::WindowBuilder, dimension: i16) -> Self {
        fn compile_shader (gl:&Gl, src: &[u8], ty: GLenum) -> GLuint {
            unsafe {
                let shader = gl.CreateShader(ty);
                // Attempt to compile the shader
                gl.ShaderSource(shader, 1,
                                &(src.as_ptr() as *const i8),
                                &(src.len() as GLint));
                gl.CompileShader(shader);

                // Get the compile status
                let mut status = gl::FALSE as GLint;
                gl.GetShaderiv(shader, gl::COMPILE_STATUS, &mut status);

                // Fail on error
                if status != (gl::TRUE as GLint) {
                    let mut len: GLint = 0;
                    gl.GetShaderiv(shader, gl::INFO_LOG_LENGTH, &mut len);

                    // allocate a buffer of size (len - 1) to skip the trailing null character
                    let mut buf: Vec<u8> = repeat(0u8).take((len as usize).saturating_sub(1)).collect();
                    gl.GetShaderInfoLog(shader, len, ptr::null_mut(), buf.as_mut_ptr() as *mut GLchar);
                    panic!("{}", str::from_utf8(&buf).ok().expect("ShaderInfoLog not valid utf8"));
                }
                shader
            }
        };

        let window = builder.build().unwrap();
        unsafe { window.make_current().unwrap() };
        let gl = Gl::load_with(|s| window.get_proc_address(s) as *const _);

        // Create GLSL shaders
        let vs = compile_shader(&gl, VERTEX_SRC, gl::VERTEX_SHADER);
        let fs = compile_shader(&gl, FRAGMENT_SRC, gl::FRAGMENT_SHADER);

        // Link program
        let program;
        unsafe {
            program = gl.CreateProgram();
            gl.AttachShader(program, vs);
            gl.AttachShader(program, fs);
            gl.LinkProgram(program);
            // Get the link status
            let mut status = gl::FALSE as GLint;
            gl.GetProgramiv(program, gl::LINK_STATUS, &mut status);

            // Fail on error
            if status != (gl::TRUE as GLint) {
                let mut len: GLint = 0;
                gl.GetProgramiv(program, gl::INFO_LOG_LENGTH, &mut len);

                // allocate a buffer of size (len - 1) to skip the trailing null character
                let mut buf: Vec<u8> = repeat(0u8).take((len as usize).saturating_sub(1)).collect();
                gl.GetProgramInfoLog(program, len, ptr::null_mut(), buf.as_mut_ptr() as *mut GLchar);
                panic!("{}", str::from_utf8(&buf).ok().expect("ProgramInfoLog not valid utf8"));
            }
        }

        let mut vao = 0;
        let mut vbo = 0;

        let trans_uniform;
        unsafe {
            // Create Vertex Array Object
            gl.GenVertexArrays(1, &mut vao);
            gl.BindVertexArray(vao);

            // Create a Vertex Buffer Object and copy the vertex data to it
            gl.GenBuffers(1, &mut vbo);
            gl.BindBuffer(gl::ARRAY_BUFFER, vbo);

            gl.BufferData(gl::ARRAY_BUFFER,
                          (VERTEX_DATA.len() * mem::size_of::<Vertex>()) as GLsizeiptr,
                          mem::transmute(&VERTEX_DATA[0]),
                          gl::STATIC_DRAW);

            // Use shader program
            gl.UseProgram(program);
            let o_color = CString::new("o_Color").unwrap();
            gl.BindFragDataLocation(program, 0, o_color.as_bytes_with_nul().as_ptr() as *const i8);

            // Specify the layout of the vertex data
            let a_pos = CString::new("a_Pos").unwrap();
            gl.BindFragDataLocation(program, 0, a_pos.as_bytes_with_nul().as_ptr() as *const i8);

            let pos_attr = gl.GetAttribLocation(program, a_pos.as_ptr());
            gl.EnableVertexAttribArray(pos_attr as GLuint);
            gl.VertexAttribPointer(pos_attr as GLuint, 3, gl::FLOAT,
                                   gl::FALSE as GLboolean, 0, ptr::null());

            let u_transform = CString::new("u_Transform").unwrap();
            trans_uniform = gl.GetUniformLocation(program, u_transform.as_bytes_with_nul().as_ptr() as *const i8)
        };

        GL {
            window: window,
            dimension: dimension,
            gl: gl,
            vs: vs,
            fs: fs,
            program: program,
            vbo: vbo,
            vao: vao,
            trans_uniform: trans_uniform,
        }
    }
}

impl Renderer for GL {
    fn render(&mut self, proj_view: &Matrix4<f32>) {
        let start = precise_time_s() * 1000.;

        // Clear the screen to black
        unsafe {
            self.gl.ClearColor(CLEAR_COLOR.0, CLEAR_COLOR.1, CLEAR_COLOR.2, CLEAR_COLOR.3);
            self.gl.Clear(gl::COLOR_BUFFER_BIT);
        }

        for x in (-self.dimension) ..self.dimension {
            for y in (-self.dimension) ..self.dimension {
                let mat:Matrix4<f32> = transform(x, y, proj_view).into();

                unsafe {
                    self.gl.UniformMatrix4fv(self.trans_uniform,
                                             1,
                                             gl::FALSE,
                                             mat.as_ptr());
                    self.gl.DrawArrays(gl::TRIANGLES, 0, 3);
                }

            }
        }

        let submit = precise_time_s() * 1000.;

        // Swap buffers
        self.window.swap_buffers().unwrap();
        let swap = precise_time_s() * 1000.;

        println!("total time:\t\t{0:4.2}ms", swap - start);
        println!("\tsubmit:\t\t{0:4.2}ms", submit - start);
        println!("\tgpu wait:\t{0:4.2}ms", swap - submit)
    }
    fn window(&mut self) -> &glutin::Window { &self.window }
}

impl Drop for GL {
    fn drop(&mut self) {
        unsafe {
            self.gl.DeleteProgram(self.program);
            self.gl.DeleteShader(self.fs);
            self.gl.DeleteShader(self.vs);
            self.gl.DeleteBuffers(1, &self.vbo);
            self.gl.DeleteVertexArrays(1, &self.vao);
        }
    }
}

fn main() {
    let ref mut args = env::args();
    let args_count = env::args().count();
    if args_count == 1 {
        println!("gfx-perf [gl|gfx] <size>");
        return;
    }

    let mode = args.nth(1).unwrap();
    let count: i32 = if args_count == 3 {
        FromStr::from_str(&args.next().unwrap()).ok()
    } else {
        None
    }.unwrap_or(10000);

    let count = ((count as f64).sqrt() / 2.) as i16;

    let builder = glutin::WindowBuilder::new()
        .with_title("Performance example".to_string())
        .with_dimensions(800, 600)
        .with_vsync();

    let mut r: Box<Renderer>;
    match mode.as_ref() {
        "gfx" => r = Box::new(GFX::new(builder, count)),
        "gl" => r = Box::new(GL::new(builder, count)),
        x => {
            panic!("{} is not a known mode", x)
        }
    }

    let proj_view = {
        let view = Matrix4::look_at(
            Point3::new(0f32, 5.0, -5.0),
            Point3::new(0f32, 0.0, 0.0),
            Vector3::unit_z(),
        );

        let proj = {
            let aspect = {
                let (w, h) = r.window().get_inner_size().unwrap();
                w as f32 / h as f32
            };
            cgmath::perspective(cgmath::deg(45.0f32), aspect, 1.0, 10.0)
        };
        proj * view
    };

    println!("count is {}", count*count*4);

    'main: loop {
        for event in r.window().poll_events() {
            match event {
                glutin::Event::KeyboardInput(_, _, Some(glutin::VirtualKeyCode::Escape)) |
                glutin::Event::Closed => break 'main,
                _ => {},
            }
        }
        r.render(&proj_view);
    }
}
<|endoftext|>


<commit_before>//! A module that provides a high-level interface for working with stored data.
//!
//! # Database
//!
//! A [`Database`] is the underlying type for working with stored data. Internally `Database` is
//! a key-value storage with reading isolation and atomic writing. This is an embedded database,
//! which means that only the Exonum process can access it. You can interact with a `Database` from
//! different threads by cloning its instance.
//!
//! Exonum provides two types of database: [`LevelDB`] and [`MemoryDB`]. However, you can make your
//! own implementations of [`Database`] trait. See its documentation for more.
//!
//! # Snapshot and Fork
//!
//! There is no way to directly interact with data in the database.
//!
//! If you need to read the data, you can create a [`Snapshot`] using method [`snapshot`][1]
//! of the `Database` instance. Snapshots provide a read isolation, so you are guaranteed to work
//! with consistent values even if the data in the database changes between reads. `Snapshot`
//! provides all the necessary methods for reading data from the database, so `&Storage`
//! is used as a storage view for creating read-only [indices] representation.
//!
//! If you need to make any changes to the data, you need to create a [`Fork`] using method
//! [`fork`][2] of the `Database` instance. As well as `Snapshot`, `Fork` provides read isolation
//! and also allows you to create a sequence of changes to the database that are specified
//! as [`Patch`]. Later you can atomically merge a patch into the database using method [`merge`].
//!
//! # StorageKey and StorageValue traits
//!
//! If you need to use your own data types as keys or values in storage, you need to implement
//! the [`StorageKey`] or [`StorageValue`] traits respectively. These traits are already
//! implemented for most standard types. See their documentation for more.
//!
//! # Indices
//!
//! Indexes are structures that represent high-level data collections that are stored in a database.
//! This concept is similar to tables in relational databases. All indexes are stored in a single
//! key-value table, but internally the keys of each index will be prefixed with the unique index
//! prefixes, so the data of the different indexes do not conflict with each other. The interfaces
//! of the indexes are similar to ordinary collections (like arrays, maps and sets), but their data
//! is located in persistent storage.
//!
//! Merklized indices support the ability to generate the cryptographic proofs about the inclusion
//! of data in them. Having such proof external client may verify locally that received data was
//! really authorized by the validators without having to replicate the entire blockchain contents.
//!
//! Exonum implements the following indices:
//!
//! - [`Entry`] is a specific index that stores only one value. Useful for global values, such as
//! configuration. Similar to combination of [`Box`] and [`Option`] from standard library.
//! - [`ListIndex`] is a list of items stored in sequential order. Similar to [`Vec`] collection
//! from standard library.
//! - [`MapIndex`] is a map of keys and values. Similar to [`BTreeMap`] collection from standard
//! library.
//! - [`ProofListIndex`] is a Merklized version of `ListIndex` which supports the cryptographic
//! proofs of existence and is implemented as a Merkle tree.
//! - [`ProofMapIndex`] is a Merklized version of `MapIndex` which supports the cryptographic
//! proofs of existence and is implemented as a Merkle Patricia tree.
//! - [`KeySetIndex`] and [`ValueSetIndex`] is a set of items, similar to [`BTreeSet`] and
//! [`HashSet`] from standard library.
//!
//! If necessary, it's possible to implement new specific types of indexes. To do this, you need
//! to create a wrapper over [`BaseIndex`] structure. See their documentation for more.
//!
//! [`Database`]: trait.Database.html
//! [`LevelDB`]: struct.LevelDB.html
//! [`MemoryDB`]: struct.MemoryDB.html
//! [`Snapshot`]: trait.Snapshot.html
//! [`Fork`]: struct.fork.html
//! [`Patch`]: type.Patch.html
//! [1]: trait.Database.html#tymethod.snapshot
//! [2]: trait.Database.html#method.fork
//! [`merge`]: trait.Database.html#tymethod.merge
//! [`StorageKey`]: trait.StorageKey.html
//! [`StorageValue`]: trait.StorageValue.html
//! [`Entry`]: struct.Entry.html
//! [`ListIndex`]: list_index/struct.ListIndex.html
//! [`MapIndex`]: map_index/struct.MapIndex.html
//! [`ProofListIndex`]: proof_list_index/struct.ProofListIndex.html
//! [`ProofMapIndex`]: proof_map_index/struct.ProofMapIndex.html
//! [`KeySetIndex`]: key_set_index/struct.KeySetIndex.html
//! [`ValueSetIndex`]: value_set_index/struct.ValueSetIndex.html
//! [`BaseIndex`]: base_index/struct.BaseIndex.html
//! [indices]: #indices
//! [`Option`]: https://doc.rust-lang.org/std/option/enum.Option.html
//! [`Box`]: https://doc.rust-lang.org/std/boxed/struct.Box.html
//! [`Vec`]: https://doc.rust-lang.org/std/vec/struct.Vec.html
//! [`BTreeMap`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html
//! [`BTreeSet`]: https://doc.rust-lang.org/std/collections/struct.BTreeSet.html
//! [`HashSet`]: https://doc.rust-lang.org/std/collections/struct.HashSet.html

#![deny(missing_docs)]

pub use self::error::Error;
pub use self::db::{Database, Snapshot, Fork, Patch, Change, Iterator, Iter};

pub use self::leveldb::{LevelDB, LevelDBOptions, LevelDBCache};
pub use self::memorydb::MemoryDB;

pub use self::keys::StorageKey;
pub use self::values::StorageValue;

pub use self::entry::Entry;

pub use self::base_index::{BaseIndex, BaseIndexIter};
pub use self::map_index::MapIndex;
pub use self::list_index::ListIndex;
pub use self::key_set_index::KeySetIndex;
pub use self::value_set_index::ValueSetIndex;
pub use self::proof_list_index::{ProofListIndex, ListProof};
pub use self::proof_map_index::{ProofMapIndex, MapProof};

/// A specialized `Result` type for I/O operations with storage.
pub type Result<T> = ::std::result::Result<T, Error>;

mod error;

mod db;
mod leveldb;
mod memorydb;

mod keys;
mod values;

mod entry;

pub mod base_index;

pub mod map_index;
pub mod list_index;
pub mod key_set_index;
pub mod value_set_index;
pub mod proof_list_index;
pub mod proof_map_index;

#[cfg(test)]
mod tests;
<commit_msg>Fix clippy warnings<commit_after>//! A module that provides a high-level interface for working with stored data.
//!
//! # Database
//!
//! A [`Database`] is the underlying type for working with stored data. Internally `Database` is
//! a key-value storage with reading isolation and atomic writing. This is an embedded database,
//! which means that only the Exonum process can access it. You can interact with a `Database` from
//! different threads by cloning its instance.
//!
//! Exonum provides two types of database: [`LevelDB`] and [`MemoryDB`]. However, you can make your
//! own implementations of [`Database`] trait. See its documentation for more.
//!
//! # Snapshot and Fork
//!
//! There is no way to directly interact with data in the database.
//!
//! If you need to read the data, you can create a [`Snapshot`] using method [`snapshot`][1]
//! of the `Database` instance. Snapshots provide a read isolation, so you are guaranteed to work
//! with consistent values even if the data in the database changes between reads. `Snapshot`
//! provides all the necessary methods for reading data from the database, so `&Storage`
//! is used as a storage view for creating read-only [indices] representation.
//!
//! If you need to make any changes to the data, you need to create a [`Fork`] using method
//! [`fork`][2] of the `Database` instance. As well as `Snapshot`, `Fork` provides read isolation
//! and also allows you to create a sequence of changes to the database that are specified
//! as [`Patch`]. Later you can atomically merge a patch into the database using method [`merge`].
//!
//! # `StorageKey` and `StorageValue` traits
//!
//! If you need to use your own data types as keys or values in storage, you need to implement
//! the [`StorageKey`] or [`StorageValue`] traits respectively. These traits are already
//! implemented for most standard types. See their documentation for more.
//!
//! # Indices
//!
//! Indexes are structures that represent high-level data collections that are stored in a database.
//! This concept is similar to tables in relational databases. All indexes are stored in a single
//! key-value table, but internally the keys of each index will be prefixed with the unique index
//! prefixes, so the data of the different indexes do not conflict with each other. The interfaces
//! of the indexes are similar to ordinary collections (like arrays, maps and sets), but their data
//! is located in persistent storage.
//!
//! Merklized indices support the ability to generate the cryptographic proofs about the inclusion
//! of data in them. Having such proof external client may verify locally that received data was
//! really authorized by the validators without having to replicate the entire blockchain contents.
//!
//! Exonum implements the following indices:
//!
//! - [`Entry`] is a specific index that stores only one value. Useful for global values, such as
//! configuration. Similar to combination of [`Box`] and [`Option`] from standard library.
//! - [`ListIndex`] is a list of items stored in sequential order. Similar to [`Vec`] collection
//! from standard library.
//! - [`MapIndex`] is a map of keys and values. Similar to [`BTreeMap`] collection from standard
//! library.
//! - [`ProofListIndex`] is a Merklized version of `ListIndex` which supports the cryptographic
//! proofs of existence and is implemented as a Merkle tree.
//! - [`ProofMapIndex`] is a Merklized version of `MapIndex` which supports the cryptographic
//! proofs of existence and is implemented as a Merkle Patricia tree.
//! - [`KeySetIndex`] and [`ValueSetIndex`] is a set of items, similar to [`BTreeSet`] and
//! [`HashSet`] from standard library.
//!
//! If necessary, it's possible to implement new specific types of indexes. To do this, you need
//! to create a wrapper over [`BaseIndex`] structure. See their documentation for more.
//!
//! [`Database`]: trait.Database.html
//! [`LevelDB`]: struct.LevelDB.html
//! [`MemoryDB`]: struct.MemoryDB.html
//! [`Snapshot`]: trait.Snapshot.html
//! [`Fork`]: struct.fork.html
//! [`Patch`]: type.Patch.html
//! [1]: trait.Database.html#tymethod.snapshot
//! [2]: trait.Database.html#method.fork
//! [`merge`]: trait.Database.html#tymethod.merge
//! [`StorageKey`]: trait.StorageKey.html
//! [`StorageValue`]: trait.StorageValue.html
//! [`Entry`]: struct.Entry.html
//! [`ListIndex`]: list_index/struct.ListIndex.html
//! [`MapIndex`]: map_index/struct.MapIndex.html
//! [`ProofListIndex`]: proof_list_index/struct.ProofListIndex.html
//! [`ProofMapIndex`]: proof_map_index/struct.ProofMapIndex.html
//! [`KeySetIndex`]: key_set_index/struct.KeySetIndex.html
//! [`ValueSetIndex`]: value_set_index/struct.ValueSetIndex.html
//! [`BaseIndex`]: base_index/struct.BaseIndex.html
//! [indices]: #indices
//! [`Option`]: https://doc.rust-lang.org/std/option/enum.Option.html
//! [`Box`]: https://doc.rust-lang.org/std/boxed/struct.Box.html
//! [`Vec`]: https://doc.rust-lang.org/std/vec/struct.Vec.html
//! [`BTreeMap`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html
//! [`BTreeSet`]: https://doc.rust-lang.org/std/collections/struct.BTreeSet.html
//! [`HashSet`]: https://doc.rust-lang.org/std/collections/struct.HashSet.html

#![deny(missing_docs)]

pub use self::error::Error;
pub use self::db::{Database, Snapshot, Fork, Patch, Change, Iterator, Iter};

pub use self::leveldb::{LevelDB, LevelDBOptions, LevelDBCache};
pub use self::memorydb::MemoryDB;

pub use self::keys::StorageKey;
pub use self::values::StorageValue;

pub use self::entry::Entry;

pub use self::base_index::{BaseIndex, BaseIndexIter};
pub use self::map_index::MapIndex;
pub use self::list_index::ListIndex;
pub use self::key_set_index::KeySetIndex;
pub use self::value_set_index::ValueSetIndex;
pub use self::proof_list_index::{ProofListIndex, ListProof};
pub use self::proof_map_index::{ProofMapIndex, MapProof};

/// A specialized `Result` type for I/O operations with storage.
pub type Result<T> = ::std::result::Result<T, Error>;

mod error;

mod db;
mod leveldb;
mod memorydb;

mod keys;
mod values;

mod entry;

pub mod base_index;

pub mod map_index;
pub mod list_index;
pub mod key_set_index;
pub mod value_set_index;
pub mod proof_list_index;
pub mod proof_map_index;

#[cfg(test)]
mod tests;
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use internal_types::FastHashMap;
use std::fmt::Debug;
use std::hash::Hash;
use std::marker::PhantomData;
use std::mem;
use std::ops;
use std::u64;

/*

 The interning module provides a generic data structure
 interning container. It is similar in concept to a
 traditional string interning container, but it is
 specialized to the WR thread model.

 There is an Interner structure, that lives in the
 scene builder thread, and a DataStore structure
 that lives in the frame builder thread.

 Hashing, interning and handle creation is done by
 the interner structure during scene building.

 Delta changes for the interner are pushed during
 a transaction to the frame builder. The frame builder
 is then able to access the content of the interned
 handles quickly, via array indexing.

 Epoch tracking ensures that the garbage collection
 step which the interner uses to remove items is
 only invoked on items that the frame builder thread
 is no longer referencing.

 Items in the data store are stored in a traditional
 free-list structure, for content access and memory
 usage efficiency.

 */

/// The epoch is incremented each time a scene is
/// built. The most recently used scene epoch is
/// stored inside each item and handle. This is
/// then used for cache invalidation (item) and
/// correctness validation (handle).
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone, PartialEq)]
struct Epoch(u64);

impl Epoch {
    pub const INVALID: Self = Epoch(u64::MAX);
}

/// A list of updates to be applied to the data store,
/// provided by the interning structure.
pub struct UpdateList<S> {
    /// The current epoch of the scene builder.
    epoch: Epoch,
    /// The additions and removals to apply.
    updates: Vec<Update<S>>,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)]
pub struct ItemUid<T> {
    uid: usize,
    _marker: PhantomData<T>,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone)]
pub struct Handle<T> {
    index: u32,
    epoch: Epoch,
    uid: ItemUid<T>,
    _marker: PhantomData<T>,
}

impl <T> Handle<T> where T: Copy {
    pub fn uid(&self) -> ItemUid<T> {
        self.uid
    }
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub enum UpdateKind<S> {
    Insert(S),
    Remove,
    UpdateEpoch,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct Update<S> {
    index: usize,
    kind: UpdateKind<S>,
}

/// The data item is stored with an epoch, for validating
/// correct access patterns.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
struct Item<T> {
    epoch: Epoch,
    data: T,
}

/// The data store lives in the frame builder thread. It
/// contains a free-list of items for fast access.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct DataStore<S, T, M> {
    items: Vec<Item<T>>,
    _source: PhantomData<S>,
    _marker: PhantomData<M>,
}

impl<S, T, M> DataStore<S, T, M> where S: Debug, T: From<S>, M: Debug {
    /// Construct a new data store
    pub fn new() -> Self {
        DataStore {
            items: Vec::new(),
            _source: PhantomData,
            _marker: PhantomData,
        }
    }

    /// Apply any updates from the scene builder thread to
    /// this data store.
    pub fn apply_updates(
        &mut self,
        update_list: UpdateList<S>,
    ) {
        for update in update_list.updates {
            match update.kind {
                UpdateKind::Insert(data) => {
                    let item = Item {
                        data: T::from(data),
                        epoch: update_list.epoch,
                    };
                    if self.items.len() == update.index {
                        self.items.push(item)
                    } else {
                        self.items[update.index] = item;
                    }
                }
                UpdateKind::Remove => {
                    self.items[update.index].epoch = Epoch::INVALID;
                }
                UpdateKind::UpdateEpoch => {
                    self.items[update.index].epoch = update_list.epoch;
                }
            }
        }
    }
}

/// Retrieve an item from the store via handle
impl<S, T, M> ops::Index<Handle<M>> for DataStore<S, T, M> {
    type Output = T;
    fn index(&self, handle: Handle<M>) -> &T {
        let item = &self.items[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &item.data
    }
}

/// Retrieve a mutable item from the store via handle
/// Retrieve an item from the store via handle
impl<S, T, M> ops::IndexMut<Handle<M>> for DataStore<S, T, M> {
    fn index_mut(&mut self, handle: Handle<M>) -> &mut T {
        let item = &mut self.items[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &mut item.data
    }
}

/// The main interning data structure. This lives in the
/// scene builder thread, and handles hashing and interning
/// unique data structures. It also manages a free-list for
/// the items in the data store, which is synchronized via
/// an update list of additions / removals.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct Interner<S : Eq + Hash + Clone + Debug, D, M> {
    /// Uniquely map an interning key to a handle
    map: FastHashMap<S, Handle<M>>,
    /// List of free slots in the data store for re-use.
    free_list: Vec<usize>,
    /// Pending list of updates that need to be applied.
    updates: Vec<Update<S>>,
    /// The current epoch for the interner.
    current_epoch: Epoch,
    /// Incrementing counter for identifying stable values.
    next_uid: usize,
    /// The information associated with each interned
    /// item that can be accessed by the interner.
    local_data: Vec<Item<D>>,
}

impl<S, D, M> Interner<S, D, M> where S: Eq + Hash + Clone + Debug, M: Copy + Debug {
    /// Construct a new interner
    pub fn new() -> Self {
        Interner {
            map: FastHashMap::default(),
            free_list: Vec::new(),
            updates: Vec::new(),
            current_epoch: Epoch(1),
            next_uid: 0,
            local_data: Vec::new(),
        }
    }

    /// Intern a data structure, and return a handle to
    /// that data. The handle can then be stored in the
    /// frame builder, and safely accessed via the data
    /// store that lives in the frame builder thread.
    /// The provided closure is invoked to build the
    /// local data about an interned structure if the
    /// key isn't already interned.
    pub fn intern<F>(
        &mut self,
        data: &S,
        f: F,
    ) -> Handle<M> where F: FnOnce() -> D {
        // Use get_mut rather than entry here to avoid
        // cloning the (sometimes large) key in the common
        // case, where the data already exists in the interner.
        if let Some(handle) = self.map.get_mut(data) {
            // Update the epoch in the data store. This
            // is not strictly needed for correctness, but
            // is used to ensure items are only accessed
            // via valid handles.
            if handle.epoch != self.current_epoch {
                self.updates.push(Update {
                    index: handle.index as usize,
                    kind: UpdateKind::UpdateEpoch,
                });
                self.local_data[handle.index as usize].epoch = self.current_epoch;
            }
            handle.epoch = self.current_epoch;
            return *handle;
        }

        // We need to intern a new data item. First, find out
        // if there is a spare slot in the free-list that we
        // can use. Otherwise, append to the end of the list.
        let index = match self.free_list.pop() {
            Some(index) => index,
            None => self.local_data.len(),
        };

        // Add a pending update to insert the new data.
        self.updates.push(Update {
            index,
            kind: UpdateKind::Insert(data.clone()),
        });

        // Generate a handle for access via the data store.
        let handle = Handle {
            index: index as u32,
            epoch: self.current_epoch,
            uid: ItemUid {
                uid: self.next_uid,
                _marker: PhantomData,
            },
            _marker: PhantomData,
        };

        // Store this handle so the next time it is
        // interned, it gets re-used.
        self.map.insert(data.clone(), handle);
        self.next_uid += 1;

        // Create the local data for this item that is
        // being interned.
        let local_item = Item {
            epoch: self.current_epoch,
            data: f(),
        };
        if self.local_data.len() == index {
            self.local_data.push(local_item);
        } else {
            self.local_data[index] = local_item;
        }

        handle
    }

    /// Retrieve the pending list of updates for an interner
    /// that need to be applied to the data store. Also run
    /// a GC step that removes old entries.
    pub fn end_frame_and_get_pending_updates(&mut self) -> UpdateList<S> {
        let mut updates = mem::replace(&mut self.updates, Vec::new());
        let free_list = &mut self.free_list;
        let current_epoch = self.current_epoch.0;

        // First, run a GC step. Walk through the handles, and
        // if we find any that haven't been used for some time,
        // remove them. If this ever shows up in profiles, we
        // can make the GC step partial (scan only part of the
        // map each frame). It also might make sense in the
        // future to adjust how long items remain in the cache
        // based on the current size of the list.
        self.map.retain(|_, handle| {
            if handle.epoch.0 + 10 < current_epoch {
                // To expire an item:
                //  - Add index to the free-list for re-use.
                //  - Add an update to the data store to invalidate this slow.
                //  - Remove from the hash map.
                free_list.push(handle.index as usize);
                updates.push(Update {
                    index: handle.index as usize,
                    kind: UpdateKind::Remove,
                });
                return false;
            }

            true
        });

        let updates = UpdateList {
            updates,
            epoch: self.current_epoch,
        };

        // Begin the next epoch
        self.current_epoch = Epoch(self.current_epoch.0 + 1);

        updates
    }
}

/// Retrieve the local data for an item from the interner via handle
impl<S, D, M> ops::Index<Handle<M>> for Interner<S, D, M> where S: Eq + Clone + Hash + Debug, M: Copy + Debug {
    type Output = D;
    fn index(&self, handle: Handle<M>) -> &D {
        let item = &self.local_data[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &item.data
    }
}
<commit_msg>Refactor interning update list to not include the keys.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use internal_types::FastHashMap;
use std::fmt::Debug;
use std::hash::Hash;
use std::marker::PhantomData;
use std::{mem, ops, u64};
use util::VecHelper;

/*

 The interning module provides a generic data structure
 interning container. It is similar in concept to a
 traditional string interning container, but it is
 specialized to the WR thread model.

 There is an Interner structure, that lives in the
 scene builder thread, and a DataStore structure
 that lives in the frame builder thread.

 Hashing, interning and handle creation is done by
 the interner structure during scene building.

 Delta changes for the interner are pushed during
 a transaction to the frame builder. The frame builder
 is then able to access the content of the interned
 handles quickly, via array indexing.

 Epoch tracking ensures that the garbage collection
 step which the interner uses to remove items is
 only invoked on items that the frame builder thread
 is no longer referencing.

 Items in the data store are stored in a traditional
 free-list structure, for content access and memory
 usage efficiency.

 */

/// The epoch is incremented each time a scene is
/// built. The most recently used scene epoch is
/// stored inside each item and handle. This is
/// then used for cache invalidation (item) and
/// correctness validation (handle).
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone, PartialEq)]
struct Epoch(u64);

impl Epoch {
    pub const INVALID: Self = Epoch(u64::MAX);
}

/// A list of updates to be applied to the data store,
/// provided by the interning structure.
pub struct UpdateList<S> {
    /// The current epoch of the scene builder.
    epoch: Epoch,
    /// The additions and removals to apply.
    updates: Vec<Update>,
    /// Actual new data to insert.
    data: Vec<S>,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)]
pub struct ItemUid<T> {
    uid: usize,
    _marker: PhantomData<T>,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
#[derive(Debug, Copy, Clone)]
pub struct Handle<T> {
    index: u32,
    epoch: Epoch,
    uid: ItemUid<T>,
    _marker: PhantomData<T>,
}

impl <T> Handle<T> where T: Copy {
    pub fn uid(&self) -> ItemUid<T> {
        self.uid
    }
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub enum UpdateKind {
    Insert,
    Remove,
    UpdateEpoch,
}

#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct Update {
    index: usize,
    kind: UpdateKind,
}

/// The data item is stored with an epoch, for validating
/// correct access patterns.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
struct Item<T> {
    epoch: Epoch,
    data: T,
}

/// The data store lives in the frame builder thread. It
/// contains a free-list of items for fast access.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct DataStore<S, T, M> {
    items: Vec<Item<T>>,
    _source: PhantomData<S>,
    _marker: PhantomData<M>,
}

impl<S, T, M> DataStore<S, T, M> where S: Debug, T: From<S>, M: Debug {
    /// Construct a new data store
    pub fn new() -> Self {
        DataStore {
            items: Vec::new(),
            _source: PhantomData,
            _marker: PhantomData,
        }
    }

    /// Apply any updates from the scene builder thread to
    /// this data store.
    pub fn apply_updates(
        &mut self,
        update_list: UpdateList<S>,
    ) {
        let mut data_iter = update_list.data.into_iter();
        for update in update_list.updates {
            match update.kind {
                UpdateKind::Insert => {
                    let data = data_iter.next().unwrap();
                    let item = Item {
                        data: T::from(data),
                        epoch: update_list.epoch,
                    };
                    if self.items.len() == update.index {
                        self.items.push(item)
                    } else {
                        self.items[update.index] = item;
                    }
                }
                UpdateKind::Remove => {
                    self.items[update.index].epoch = Epoch::INVALID;
                }
                UpdateKind::UpdateEpoch => {
                    self.items[update.index].epoch = update_list.epoch;
                }
            }
        }
    }
}

/// Retrieve an item from the store via handle
impl<S, T, M> ops::Index<Handle<M>> for DataStore<S, T, M> {
    type Output = T;
    fn index(&self, handle: Handle<M>) -> &T {
        let item = &self.items[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &item.data
    }
}

/// Retrieve a mutable item from the store via handle
/// Retrieve an item from the store via handle
impl<S, T, M> ops::IndexMut<Handle<M>> for DataStore<S, T, M> {
    fn index_mut(&mut self, handle: Handle<M>) -> &mut T {
        let item = &mut self.items[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &mut item.data
    }
}

/// The main interning data structure. This lives in the
/// scene builder thread, and handles hashing and interning
/// unique data structures. It also manages a free-list for
/// the items in the data store, which is synchronized via
/// an update list of additions / removals.
#[cfg_attr(feature = "capture", derive(Serialize))]
#[cfg_attr(feature = "replay", derive(Deserialize))]
pub struct Interner<S : Eq + Hash + Clone + Debug, D, M> {
    /// Uniquely map an interning key to a handle
    map: FastHashMap<S, Handle<M>>,
    /// List of free slots in the data store for re-use.
    free_list: Vec<usize>,
    /// Pending list of updates that need to be applied.
    updates: Vec<Update>,
    /// Pending new data to insert.
    update_data: Vec<S>,
    /// The current epoch for the interner.
    current_epoch: Epoch,
    /// The information associated with each interned
    /// item that can be accessed by the interner.
    local_data: Vec<Item<D>>,
}

impl<S, D, M> Interner<S, D, M> where S: Eq + Hash + Clone + Debug, M: Copy + Debug {
    /// Construct a new interner
    pub fn new() -> Self {
        Interner {
            map: FastHashMap::default(),
            free_list: Vec::new(),
            updates: Vec::new(),
            update_data: Vec::new(),
            current_epoch: Epoch(1),
            local_data: Vec::new(),
        }
    }

    /// Intern a data structure, and return a handle to
    /// that data. The handle can then be stored in the
    /// frame builder, and safely accessed via the data
    /// store that lives in the frame builder thread.
    /// The provided closure is invoked to build the
    /// local data about an interned structure if the
    /// key isn't already interned.
    pub fn intern<F>(
        &mut self,
        data: &S,
        f: F,
    ) -> Handle<M> where F: FnOnce() -> D {
        // Use get_mut rather than entry here to avoid
        // cloning the (sometimes large) key in the common
        // case, where the data already exists in the interner.
        if let Some(handle) = self.map.get_mut(data) {
            // Update the epoch in the data store. This
            // is not strictly needed for correctness, but
            // is used to ensure items are only accessed
            // via valid handles.
            if handle.epoch != self.current_epoch {
                self.updates.push(Update {
                    index: handle.index as usize,
                    kind: UpdateKind::UpdateEpoch,
                });
                self.local_data[handle.index as usize].epoch = self.current_epoch;
            }
            handle.epoch = self.current_epoch;
            return *handle;
        }

        // We need to intern a new data item. First, find out
        // if there is a spare slot in the free-list that we
        // can use. Otherwise, append to the end of the list.
        let index = match self.free_list.pop() {
            Some(index) => index,
            None => self.local_data.len(),
        };

        // Add a pending update to insert the new data.
        self.updates.push(Update {
            index,
            kind: UpdateKind::Insert,
        });
        self.update_data.alloc().init(data.clone());

        // Generate a handle for access via the data store.
        let handle = Handle {
            index: index as u32,
            epoch: self.current_epoch,
            uid: ItemUid {
                uid: self.map.len(),
                _marker: PhantomData,
            },
            _marker: PhantomData,
        };

        // Store this handle so the next time it is
        // interned, it gets re-used.
        self.map.insert(data.clone(), handle);

        // Create the local data for this item that is
        // being interned.
        let local_item = Item {
            epoch: self.current_epoch,
            data: f(),
        };
        if self.local_data.len() == index {
            self.local_data.push(local_item);
        } else {
            self.local_data[index] = local_item;
        }

        handle
    }

    /// Retrieve the pending list of updates for an interner
    /// that need to be applied to the data store. Also run
    /// a GC step that removes old entries.
    pub fn end_frame_and_get_pending_updates(&mut self) -> UpdateList<S> {
        let mut updates = mem::replace(&mut self.updates, Vec::new());
        let data = mem::replace(&mut self.update_data, Vec::new());

        let free_list = &mut self.free_list;
        let current_epoch = self.current_epoch.0;

        // First, run a GC step. Walk through the handles, and
        // if we find any that haven't been used for some time,
        // remove them. If this ever shows up in profiles, we
        // can make the GC step partial (scan only part of the
        // map each frame). It also might make sense in the
        // future to adjust how long items remain in the cache
        // based on the current size of the list.
        self.map.retain(|_, handle| {
            if handle.epoch.0 + 10 < current_epoch {
                // To expire an item:
                //  - Add index to the free-list for re-use.
                //  - Add an update to the data store to invalidate this slow.
                //  - Remove from the hash map.
                free_list.push(handle.index as usize);
                updates.push(Update {
                    index: handle.index as usize,
                    kind: UpdateKind::Remove,
                });
                return false;
            }

            true
        });

        let updates = UpdateList {
            updates,
            data,
            epoch: self.current_epoch,
        };

        // Begin the next epoch
        self.current_epoch = Epoch(self.current_epoch.0 + 1);

        updates
    }
}

/// Retrieve the local data for an item from the interner via handle
impl<S, D, M> ops::Index<Handle<M>> for Interner<S, D, M> where S: Eq + Clone + Hash + Debug, M: Copy + Debug {
    type Output = D;
    fn index(&self, handle: Handle<M>) -> &D {
        let item = &self.local_data[handle.index as usize];
        assert_eq!(item.epoch, handle.epoch);
        &item.data
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use api::{ApiMsg, FrameMsg};
use bincode::{serialize, Infinite};
use byteorder::{LittleEndian, WriteBytesExt};
use std::any::TypeId;
use std::fmt::Debug;
use std::fs::File;
use std::io::Write;
use std::mem;
use std::path::PathBuf;

pub static WEBRENDER_RECORDING_HEADER: u64 = 0xbeefbeefbeefbe01u64;

pub trait ApiRecordingReceiver: Send + Debug {
    fn write_msg(&mut self, frame: u32, msg: &ApiMsg);
    fn write_payload(&mut self, frame: u32, data: &[u8]);
}

#[derive(Debug)]
pub struct BinaryRecorder {
    file: File,
}

impl BinaryRecorder {
    pub fn new(dest: &PathBuf) -> BinaryRecorder {
        let mut file = File::create(dest).unwrap();

        // write the header
        let apimsg_type_id = unsafe {
            assert!(mem::size_of::<TypeId>() == mem::size_of::<u64>());
            mem::transmute::<TypeId, u64>(TypeId::of::<ApiMsg>())
        };
        file.write_u64::<LittleEndian>(WEBRENDER_RECORDING_HEADER)
            .ok();
        file.write_u64::<LittleEndian>(apimsg_type_id).ok();

        BinaryRecorder { file }
    }

    fn write_length_and_data(&mut self, data: &[u8]) {
        self.file.write_u32::<LittleEndian>(data.len() as u32).ok();
        self.file.write(data).ok();
    }
}

impl ApiRecordingReceiver for BinaryRecorder {
    fn write_msg(&mut self, _: u32, msg: &ApiMsg) {
        if should_record_msg(msg) {
            let buf = serialize(msg, Infinite).unwrap();
            self.write_length_and_data(&buf);
        }
    }

    fn write_payload(&mut self, _: u32, data: &[u8]) {
        // signal payload with a 0 length
        self.file.write_u32::<LittleEndian>(0).ok();
        self.write_length_and_data(data);
    }
}

pub fn should_record_msg(msg: &ApiMsg) -> bool {
    match *msg {
        ApiMsg::UpdateResources(..) |
        ApiMsg::AddDocument { .. } |
        ApiMsg::DeleteDocument(..) => true,
        ApiMsg::UpdateDocument(_, ref msgs) => {
            for msg in &msgs.frame_ops {
                match *msg {
                    FrameMsg::GetScrollNodeState(..) |
                    FrameMsg::HitTest(..) => {}
                    _ => { return true; }
                }
            }

            false
        }
        _ => false,
    }
}
<commit_msg>Fix binary recording<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use api::{ApiMsg, FrameMsg, SceneMsg};
use bincode::{serialize, Infinite};
use byteorder::{LittleEndian, WriteBytesExt};
use std::any::TypeId;
use std::fmt::Debug;
use std::fs::File;
use std::io::Write;
use std::mem;
use std::path::PathBuf;

pub static WEBRENDER_RECORDING_HEADER: u64 = 0xbeefbeefbeefbe01u64;

pub trait ApiRecordingReceiver: Send + Debug {
    fn write_msg(&mut self, frame: u32, msg: &ApiMsg);
    fn write_payload(&mut self, frame: u32, data: &[u8]);
}

#[derive(Debug)]
pub struct BinaryRecorder {
    file: File,
}

impl BinaryRecorder {
    pub fn new(dest: &PathBuf) -> BinaryRecorder {
        let mut file = File::create(dest).unwrap();

        // write the header
        let apimsg_type_id = unsafe {
            assert!(mem::size_of::<TypeId>() == mem::size_of::<u64>());
            mem::transmute::<TypeId, u64>(TypeId::of::<ApiMsg>())
        };
        file.write_u64::<LittleEndian>(WEBRENDER_RECORDING_HEADER)
            .ok();
        file.write_u64::<LittleEndian>(apimsg_type_id).ok();

        BinaryRecorder { file }
    }

    fn write_length_and_data(&mut self, data: &[u8]) {
        self.file.write_u32::<LittleEndian>(data.len() as u32).ok();
        self.file.write(data).ok();
    }
}

impl ApiRecordingReceiver for BinaryRecorder {
    fn write_msg(&mut self, _: u32, msg: &ApiMsg) {
        if should_record_msg(msg) {
            let buf = serialize(msg, Infinite).unwrap();
            self.write_length_and_data(&buf);
        }
    }

    fn write_payload(&mut self, _: u32, data: &[u8]) {
        // signal payload with a 0 length
        self.file.write_u32::<LittleEndian>(0).ok();
        self.write_length_and_data(data);
    }
}

pub fn should_record_msg(msg: &ApiMsg) -> bool {
    match *msg {
        ApiMsg::UpdateResources(..) |
        ApiMsg::AddDocument { .. } |
        ApiMsg::DeleteDocument(..) => true,
        ApiMsg::UpdateDocument(_, ref msgs) => {
            if msgs.generate_frame {
                return true;
            }

            for msg in &msgs.scene_ops {
                match *msg {
                    SceneMsg::SetDisplayList { .. } |
                    SceneMsg::SetRootPipeline { .. } => return true,
                    _ => {}
                }
            }

            for msg in &msgs.frame_ops {
                match *msg {
                    FrameMsg::GetScrollNodeState(..) |
                    FrameMsg::HitTest(..) => {}
                    _ => return true,
                }
            }

            false
        }
        _ => false,
    }
}
<|endoftext|>


<commit_before>// See LICENSE file for copyright and license details.

use cgmath::vector::{Vector, Vec3, Vec2};
use core::map::MapPosIter;
use core::types::{MInt, Size2, MapPos, UnitId};
use visualizer::gl_helpers::{
    set_clear_color,
    clear_screen,
    read_pixel_bytes,
};
use visualizer::camera::Camera;
use visualizer::geom::Geom;
use visualizer::mesh::Mesh;
use visualizer::types::{Color3, MFloat, MatId, Scene, NodeId, WorldPos};
use visualizer::shader::Shader;

fn i_to_f(n: MInt) -> f32 {
    n as MFloat / 255.0
}

fn get_mesh(geom: &Geom, map_size: Size2<MInt>, shader: &Shader) -> Mesh {
    let mut c_data = Vec::new();
    let mut v_data = Vec::new();
    for tile_pos in MapPosIter::new(map_size) {
        let pos3d = geom.map_pos_to_world_pos(tile_pos);
        for num in range(0 as MInt, 6) {
            let vertex = geom.index_to_hex_vertex(num);
            let next_vertex = geom.index_to_hex_vertex(num + 1);
            let col_x = i_to_f(tile_pos.x);
            let col_y = i_to_f(tile_pos.y);
            let color = Color3{r: col_x, g: col_y, b: i_to_f(1)};
            v_data.push(pos3d + vertex);
            c_data.push(color);
            v_data.push(pos3d + next_vertex);
            c_data.push(color);
            v_data.push(pos3d + Vec3::zero());
            c_data.push(color);
        }
    }
    let mut mesh = Mesh::new(v_data.as_slice());
    mesh.set_color(c_data.as_slice());
    mesh.prepare(shader);
    mesh
}

pub enum PickResult {
    PickedMapPos(MapPos),
    PickedUnitId(UnitId),
    PickedNothing
}

pub struct TilePicker {
    shader: Shader,
    map_mesh: Mesh,
    units_mesh: Option<Mesh>,
    mvp_mat_id: MatId,
    win_size: Size2<MInt>,
}

impl TilePicker {
    pub fn new(
        win_size: Size2<MInt>,
        geom: &Geom,
        map_size: Size2<MInt>
    ) -> ~TilePicker {
        let shader = Shader::new("pick.vs.glsl", "pick.fs.glsl");
        let mvp_mat_id = MatId(shader.get_uniform("mvp_mat"));
        let map_mesh = get_mesh(geom, map_size, &shader);
        let tile_picker = ~TilePicker {
            map_mesh: map_mesh,
            units_mesh: None,
            shader: shader,
            mvp_mat_id: mvp_mat_id,
            win_size: win_size,
        };
        tile_picker
    }

    pub fn set_win_size(&mut self, win_size: Size2<MInt>) {
        self.win_size = win_size;
    }

    pub fn update_units( &mut self, geom: &Geom, scene: &Scene) {
        fn get_hex_vertex(geom: &Geom, n: MInt) -> WorldPos {
            let scale_factor = 0.5;
            geom.index_to_hex_vertex(n).mul_s(scale_factor)
        }
        let last_unit_node_id = 1000; // TODO
        let mut c_data = Vec::new();
        let mut v_data = Vec::new();
        for (node_id, node) in scene.iter() {
            let NodeId(id) = *node_id;
            if id >= last_unit_node_id {
                continue;
            }
            let color = Color3 {r: i_to_f(id), g: 0.0, b: i_to_f(2)};
            for num in range(0 as MInt, 6) {
                v_data.push(node.pos + get_hex_vertex(geom, num));
                c_data.push(color);
                v_data.push(node.pos + get_hex_vertex(geom, num + 1));
                c_data.push(color);
                v_data.push(node.pos + Vec3::zero());
                c_data.push(color);
            }
        }
        for v in v_data.mut_iter() {
            v.z += 0.01; // TODO
        }
        let mut mesh = Mesh::new(v_data.as_slice());
        mesh.set_color(c_data.as_slice());
        mesh.prepare(&self.shader);
        self.units_mesh = Some(mesh);
    }

    pub fn pick_tile(
        &mut self,
        camera: &Camera,
        mouse_pos: Vec2<MInt>
    ) -> PickResult {
        self.shader.activate();
        self.shader.uniform_mat4f(self.mvp_mat_id, &camera.mat());
        set_clear_color(0.0, 0.0, 0.0);
        clear_screen();
        self.map_mesh.draw(&self.shader);
        match self.units_mesh {
            Some(ref units) => units.draw(&self.shader),
            None => {},
        };
        let (r, g, b, _) = read_pixel_bytes(self.win_size, mouse_pos);
        match b {
            0 => PickedNothing,
            1 => PickedMapPos(Vec2{x: r, y: g}),
            2 => PickedUnitId(UnitId(r)),
            _ => fail!(),
        }
    }
}

// vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:
<commit_msg>picker.rs: Updated to latest Rust<commit_after>// See LICENSE file for copyright and license details.

use cgmath::vector::{Vector, Vec3, Vec2};
use core::map::MapPosIter;
use core::types::{MInt, Size2, MapPos, UnitId};
use visualizer::gl_helpers::{
    set_clear_color,
    clear_screen,
    read_pixel_bytes,
};
use visualizer::camera::Camera;
use visualizer::geom::Geom;
use visualizer::mesh::Mesh;
use visualizer::types::{Color3, MFloat, MatId, Scene, NodeId, WorldPos};
use visualizer::shader::Shader;

fn i_to_f(n: MInt) -> f32 {
    n as MFloat / 255.0
}

fn get_mesh(geom: &Geom, map_size: Size2<MInt>, shader: &Shader) -> Mesh {
    use std::slice::Vector;
    let mut c_data = Vec::new();
    let mut v_data = Vec::new();
    for tile_pos in MapPosIter::new(map_size) {
        let pos3d = geom.map_pos_to_world_pos(tile_pos);
        for num in range(0 as MInt, 6) {
            let vertex = geom.index_to_hex_vertex(num);
            let next_vertex = geom.index_to_hex_vertex(num + 1);
            let col_x = i_to_f(tile_pos.x);
            let col_y = i_to_f(tile_pos.y);
            let color = Color3{r: col_x, g: col_y, b: i_to_f(1)};
            v_data.push(pos3d + vertex);
            c_data.push(color);
            v_data.push(pos3d + next_vertex);
            c_data.push(color);
            v_data.push(pos3d + Vec3::zero());
            c_data.push(color);
        }
    }
    let mut mesh = Mesh::new(v_data.as_slice());
    mesh.set_color(c_data.as_slice());
    mesh.prepare(shader);
    mesh
}

pub enum PickResult {
    PickedMapPos(MapPos),
    PickedUnitId(UnitId),
    PickedNothing
}

pub struct TilePicker {
    shader: Shader,
    map_mesh: Mesh,
    units_mesh: Option<Mesh>,
    mvp_mat_id: MatId,
    win_size: Size2<MInt>,
}

impl TilePicker {
    pub fn new(
        win_size: Size2<MInt>,
        geom: &Geom,
        map_size: Size2<MInt>
    ) -> ~TilePicker {
        let shader = Shader::new("pick.vs.glsl", "pick.fs.glsl");
        let mvp_mat_id = MatId(shader.get_uniform("mvp_mat"));
        let map_mesh = get_mesh(geom, map_size, &shader);
        let tile_picker = ~TilePicker {
            map_mesh: map_mesh,
            units_mesh: None,
            shader: shader,
            mvp_mat_id: mvp_mat_id,
            win_size: win_size,
        };
        tile_picker
    }

    pub fn set_win_size(&mut self, win_size: Size2<MInt>) {
        self.win_size = win_size;
    }

    pub fn update_units( &mut self, geom: &Geom, scene: &Scene) {
        use std::slice::Vector;
        fn get_hex_vertex(geom: &Geom, n: MInt) -> WorldPos {
            let scale_factor = 0.5;
            geom.index_to_hex_vertex(n).mul_s(scale_factor)
        }
        let last_unit_node_id = 1000; // TODO
        let mut c_data = Vec::new();
        let mut v_data = Vec::new();
        for (node_id, node) in scene.iter() {
            let NodeId(id) = *node_id;
            if id >= last_unit_node_id {
                continue;
            }
            let color = Color3 {r: i_to_f(id), g: 0.0, b: i_to_f(2)};
            for num in range(0 as MInt, 6) {
                v_data.push(node.pos + get_hex_vertex(geom, num));
                c_data.push(color);
                v_data.push(node.pos + get_hex_vertex(geom, num + 1));
                c_data.push(color);
                v_data.push(node.pos + Vec3::zero());
                c_data.push(color);
            }
        }
        for v in v_data.mut_iter() {
            v.z += 0.01; // TODO
        }
        let mut mesh = Mesh::new(v_data.as_slice());
        mesh.set_color(c_data.as_slice());
        mesh.prepare(&self.shader);
        self.units_mesh = Some(mesh);
    }

    pub fn pick_tile(
        &mut self,
        camera: &Camera,
        mouse_pos: Vec2<MInt>
    ) -> PickResult {
        self.shader.activate();
        self.shader.uniform_mat4f(self.mvp_mat_id, &camera.mat());
        set_clear_color(0.0, 0.0, 0.0);
        clear_screen();
        self.map_mesh.draw(&self.shader);
        match self.units_mesh {
            Some(ref units) => units.draw(&self.shader),
            None => {},
        };
        let (r, g, b, _) = read_pixel_bytes(self.win_size, mouse_pos);
        match b {
            0 => PickedNothing,
            1 => PickedMapPos(Vec2{x: r, y: g}),
            2 => PickedUnitId(UnitId(r)),
            _ => fail!(),
        }
    }
}

// vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:
<|endoftext|>


<commit_before>// Copyright 2016 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

//! This library exposes a low-level API for creating your own efficient
//! synchronization primitives.
//!
//! # The parking lot
//!
//! To keep synchronization primitives small, all thread queuing and suspending
//! functionality is offloaded to the *parking lot*. The idea behind this is based
//! on the Webkit [`WTF::ParkingLot`](https://webkit.org/blog/6161/locking-in-webkit/)
//! class, which essentially consists of a hash table mapping of lock addresses
//! to queues of parked (sleeping) threads. The Webkit parking lot was itself
//! inspired by Linux [futexes](http://man7.org/linux/man-pages/man2/futex.2.html),
//! but it is more powerful since it allows invoking callbacks while holding a
//! queue lock.
//!
//! There are two main operations that can be performed on the parking lot:
//!
//!  - *Parking* refers to suspending the thread while simultaneously enqueuing it
//! on a queue keyed by some address.
//! - *Unparking* refers to dequeuing a thread from a queue keyed by some address
//! and resuming it.
//!
//! See the documentation of the individual functions for more details.
//!
//! # Building custom synchronization primitives
//!
//! Building custom synchronization primitives is very simple since the parking
//! lot takes care of all the hard parts for you. A simple example for a
//! custom primitive would be to integrate a `Mutex` inside another data type.
//! Since a mutex only requires 2 bits, it can share space with other data.
//! For example, one could create an `ArcMutex` type that combines the atomic
//! reference count and the two mutex bits in the same atomic word.

#![warn(missing_docs)]
#![warn(rust_2018_idioms)]
#![cfg_attr(
    all(target_env = "sgx", target_vendor = "fortanix"),
    feature(sgx_platform)
)]
#![cfg_attr(
    all(
        feature = "nightly",
        target_arch = "wasm32",
        target_feature = "atomics"
    ),
    feature(checked_duration_since, stdsimd)
)]
#![cfg_attr(
    all(feature = "nightly", target_os = "cloudabi",),
    feature(thread_local, checked_duration_since)
)]

mod parking_lot;
mod spinwait;
mod thread_parker;
mod util;
mod word_lock;

pub use self::parking_lot::deadlock;
pub use self::parking_lot::{park, unpark_all, unpark_filter, unpark_one, unpark_requeue};
pub use self::parking_lot::{
    FilterOp, ParkResult, ParkToken, RequeueOp, UnparkResult, UnparkToken,
};
pub use self::parking_lot::{DEFAULT_PARK_TOKEN, DEFAULT_UNPARK_TOKEN};
pub use self::spinwait::SpinWait;
<commit_msg>checked_duration_since is stable. Remove from feature gate<commit_after>// Copyright 2016 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

//! This library exposes a low-level API for creating your own efficient
//! synchronization primitives.
//!
//! # The parking lot
//!
//! To keep synchronization primitives small, all thread queuing and suspending
//! functionality is offloaded to the *parking lot*. The idea behind this is based
//! on the Webkit [`WTF::ParkingLot`](https://webkit.org/blog/6161/locking-in-webkit/)
//! class, which essentially consists of a hash table mapping of lock addresses
//! to queues of parked (sleeping) threads. The Webkit parking lot was itself
//! inspired by Linux [futexes](http://man7.org/linux/man-pages/man2/futex.2.html),
//! but it is more powerful since it allows invoking callbacks while holding a
//! queue lock.
//!
//! There are two main operations that can be performed on the parking lot:
//!
//!  - *Parking* refers to suspending the thread while simultaneously enqueuing it
//! on a queue keyed by some address.
//! - *Unparking* refers to dequeuing a thread from a queue keyed by some address
//! and resuming it.
//!
//! See the documentation of the individual functions for more details.
//!
//! # Building custom synchronization primitives
//!
//! Building custom synchronization primitives is very simple since the parking
//! lot takes care of all the hard parts for you. A simple example for a
//! custom primitive would be to integrate a `Mutex` inside another data type.
//! Since a mutex only requires 2 bits, it can share space with other data.
//! For example, one could create an `ArcMutex` type that combines the atomic
//! reference count and the two mutex bits in the same atomic word.

#![warn(missing_docs)]
#![warn(rust_2018_idioms)]
#![cfg_attr(
    all(target_env = "sgx", target_vendor = "fortanix"),
    feature(sgx_platform)
)]
#![cfg_attr(
    all(
        feature = "nightly",
        target_arch = "wasm32",
        target_feature = "atomics"
    ),
    feature(stdsimd)
)]
#![cfg_attr(
    all(feature = "nightly", target_os = "cloudabi",),
    feature(thread_local)
)]

mod parking_lot;
mod spinwait;
mod thread_parker;
mod util;
mod word_lock;

pub use self::parking_lot::deadlock;
pub use self::parking_lot::{park, unpark_all, unpark_filter, unpark_one, unpark_requeue};
pub use self::parking_lot::{
    FilterOp, ParkResult, ParkToken, RequeueOp, UnparkResult, UnparkToken,
};
pub use self::parking_lot::{DEFAULT_PARK_TOKEN, DEFAULT_UNPARK_TOKEN};
pub use self::spinwait::SpinWait;
<|endoftext|>