<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use crate::data_type::AsBytes;

/// Computes hash value for `data`, with a seed value `seed`.
/// The data type `T` must implement the `AsBytes` trait.
pub fn hash<T: AsBytes>(data: &T, seed: u32) -> u32 {
    hash_(data.as_bytes(), seed)
}

fn hash_(data: &[u8], seed: u32) -> u32 {
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    unsafe {
        if is_x86_feature_detected!("sse4.2") {
            crc32_hash(data, seed)
        } else {
            murmur_hash2_64a(data, seed as u64) as u32
        }
    }
}

const MURMUR_PRIME: u64 = 0xc6a4a7935bd1e995;
const MURMUR_R: i32 = 47;

/// Rust implementation of MurmurHash2, 64-bit version for 64-bit platforms
///
/// SAFTETY Only safe on platforms which support unaligned loads (like x86_64)
unsafe fn murmur_hash2_64a(data_bytes: &[u8], seed: u64) -> u64 {
    let len = data_bytes.len();
    let len_64 = (len / 8) * 8;
    let data_bytes_64 = std::slice::from_raw_parts(
        &data_bytes[0..len_64] as *const [u8] as *const u64,
        len / 8,
    );

    let mut h = seed ^ (MURMUR_PRIME.wrapping_mul(data_bytes.len() as u64));
    for v in data_bytes_64 {
        let mut k = *v;
        k = k.wrapping_mul(MURMUR_PRIME);
        k ^= k >> MURMUR_R;
        k = k.wrapping_mul(MURMUR_PRIME);
        h ^= k;
        h = h.wrapping_mul(MURMUR_PRIME);
    }

    let data2 = &data_bytes[len_64..];

    let v = len & 7;
    if v == 7 {
        h ^= (data2[6] as u64) << 48;
    }
    if v >= 6 {
        h ^= (data2[5] as u64) << 40;
    }
    if v >= 5 {
        h ^= (data2[4] as u64) << 32;
    }
    if v >= 4 {
        h ^= (data2[3] as u64) << 24;
    }
    if v >= 3 {
        h ^= (data2[2] as u64) << 16;
    }
    if v >= 2 {
        h ^= (data2[1] as u64) << 8;
    }
    if v >= 1 {
        h ^= data2[0] as u64;
    }
    if v > 0 {
        h = h.wrapping_mul(MURMUR_PRIME);
    }

    h ^= h >> MURMUR_R;
    h = h.wrapping_mul(MURMUR_PRIME);
    h ^= h >> MURMUR_R;
    h
}

/// CRC32 hash implementation using SSE4 instructions. Borrowed from Impala.
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[target_feature(enable = "sse4.2")]
unsafe fn crc32_hash(bytes: &[u8], seed: u32) -> u32 {
    #[cfg(target_arch = "x86")]
    use std::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use std::arch::x86_64::*;

    let u32_num_bytes = std::mem::size_of::<u32>();
    let mut num_bytes = bytes.len();
    let num_words = num_bytes / u32_num_bytes;
    num_bytes %= u32_num_bytes;

    let bytes_u32: &[u32] = std::slice::from_raw_parts(
        &bytes[0..num_words * u32_num_bytes] as *const [u8] as *const u32,
        num_words,
    );

    let mut offset = 0;
    let mut hash = seed;
    while offset < num_words {
        hash = _mm_crc32_u32(hash, bytes_u32[offset]);
        offset += 1;
    }

    offset = num_words * u32_num_bytes;
    while offset < num_bytes {
        hash = _mm_crc32_u8(hash, bytes[offset]);
        offset += 1;
    }

    // The lower half of the CRC hash has poor uniformity, so swap the halves
    // for anyone who only uses the first several bits of the hash.
    hash = (hash << 16) | (hash >> 16);
    hash
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_murmur2_64a() {
        unsafe {
            let result = murmur_hash2_64a(b"hello", 123);
            assert_eq!(result, 2597646618390559622);

            let result = murmur_hash2_64a(b"helloworld", 123);
            assert_eq!(result, 4934371746140206573);

            let result = murmur_hash2_64a(b"helloworldparquet", 123);
            assert_eq!(result, 2392198230801491746);
        }
    }

    #[test]
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    fn test_crc32() {
        if is_x86_feature_detected!("sse4.2") {
            unsafe {
                let result = crc32_hash(b"hello", 123);
                assert_eq!(result, 2927487359);

                let result = crc32_hash(b"helloworld", 123);
                assert_eq!(result, 314229527);

                let result = crc32_hash(b"helloworldparquet", 123);
                assert_eq!(result, 667078870);
            }
        }
    }
}
<commit_msg>ARROW-8622: [Rust] Allow the parquet crate to be compiled on aarch64 platforms<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use crate::data_type::AsBytes;

/// Computes hash value for `data`, with a seed value `seed`.
/// The data type `T` must implement the `AsBytes` trait.
pub fn hash<T: AsBytes>(data: &T, seed: u32) -> u32 {
    hash_(data.as_bytes(), seed)
}

fn hash_(data: &[u8], seed: u32) -> u32 {
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    unsafe {
        if is_x86_feature_detected!("sse4.2") {
            crc32_hash(data, seed)
        } else {
            murmur_hash2_64a(data, seed as u64) as u32
        }
    }

    #[cfg(target_arch = "aarch64")]
    unsafe {
        murmur_hash2_64a(data, seed as u64) as u32
    }
}

const MURMUR_PRIME: u64 = 0xc6a4a7935bd1e995;
const MURMUR_R: i32 = 47;

/// Rust implementation of MurmurHash2, 64-bit version for 64-bit platforms
///
/// SAFTETY Only safe on platforms which support unaligned loads (like x86_64)
unsafe fn murmur_hash2_64a(data_bytes: &[u8], seed: u64) -> u64 {
    let len = data_bytes.len();
    let len_64 = (len / 8) * 8;
    let data_bytes_64 = std::slice::from_raw_parts(
        &data_bytes[0..len_64] as *const [u8] as *const u64,
        len / 8,
    );

    let mut h = seed ^ (MURMUR_PRIME.wrapping_mul(data_bytes.len() as u64));
    for v in data_bytes_64 {
        let mut k = *v;
        k = k.wrapping_mul(MURMUR_PRIME);
        k ^= k >> MURMUR_R;
        k = k.wrapping_mul(MURMUR_PRIME);
        h ^= k;
        h = h.wrapping_mul(MURMUR_PRIME);
    }

    let data2 = &data_bytes[len_64..];

    let v = len & 7;
    if v == 7 {
        h ^= (data2[6] as u64) << 48;
    }
    if v >= 6 {
        h ^= (data2[5] as u64) << 40;
    }
    if v >= 5 {
        h ^= (data2[4] as u64) << 32;
    }
    if v >= 4 {
        h ^= (data2[3] as u64) << 24;
    }
    if v >= 3 {
        h ^= (data2[2] as u64) << 16;
    }
    if v >= 2 {
        h ^= (data2[1] as u64) << 8;
    }
    if v >= 1 {
        h ^= data2[0] as u64;
    }
    if v > 0 {
        h = h.wrapping_mul(MURMUR_PRIME);
    }

    h ^= h >> MURMUR_R;
    h = h.wrapping_mul(MURMUR_PRIME);
    h ^= h >> MURMUR_R;
    h
}

/// CRC32 hash implementation using SSE4 instructions. Borrowed from Impala.
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[target_feature(enable = "sse4.2")]
unsafe fn crc32_hash(bytes: &[u8], seed: u32) -> u32 {
    #[cfg(target_arch = "x86")]
    use std::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use std::arch::x86_64::*;

    let u32_num_bytes = std::mem::size_of::<u32>();
    let mut num_bytes = bytes.len();
    let num_words = num_bytes / u32_num_bytes;
    num_bytes %= u32_num_bytes;

    let bytes_u32: &[u32] = std::slice::from_raw_parts(
        &bytes[0..num_words * u32_num_bytes] as *const [u8] as *const u32,
        num_words,
    );

    let mut offset = 0;
    let mut hash = seed;
    while offset < num_words {
        hash = _mm_crc32_u32(hash, bytes_u32[offset]);
        offset += 1;
    }

    offset = num_words * u32_num_bytes;
    while offset < num_bytes {
        hash = _mm_crc32_u8(hash, bytes[offset]);
        offset += 1;
    }

    // The lower half of the CRC hash has poor uniformity, so swap the halves
    // for anyone who only uses the first several bits of the hash.
    hash = (hash << 16) | (hash >> 16);
    hash
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_murmur2_64a() {
        unsafe {
            let result = murmur_hash2_64a(b"hello", 123);
            assert_eq!(result, 2597646618390559622);

            let result = murmur_hash2_64a(b"helloworld", 123);
            assert_eq!(result, 4934371746140206573);

            let result = murmur_hash2_64a(b"helloworldparquet", 123);
            assert_eq!(result, 2392198230801491746);
        }
    }

    #[test]
    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
    fn test_crc32() {
        if is_x86_feature_detected!("sse4.2") {
            unsafe {
                let result = crc32_hash(b"hello", 123);
                assert_eq!(result, 2927487359);

                let result = crc32_hash(b"helloworld", 123);
                assert_eq!(result, 314229527);

                let result = crc32_hash(b"helloworldparquet", 123);
                assert_eq!(result, 667078870);
            }
        }
    }
}
<|endoftext|>


<commit_before>use image::ImageResult;
use mli::{Backward, Forward, Graph, Train};
use mli_conv::{Conv2, Conv2n, Conv3};
use mli_ndarray::{Map2One, Map3One, Reshape3to2};
use mli_relu::Blu;
use ndarray::{array, s, Array, Array2, OwnedRepr};
use ndarray_image::{open_gray_image, save_gray_image};
use rand_core::{RngCore, SeedableRng};
use rand_distr::{Distribution, Normal};
use rand_pcg::Pcg64;
use std::path::{Path, PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "image", about = "Loads and convolves an image as a test")]
struct Opt {
    /// Number of epochs
    #[structopt(short = "t", default_value = "4000")]
    epochs: usize,
    /// Number of epochs per output image
    #[structopt(short = "s", default_value = "1")]
    show_every: usize,
    /// Initial learning rate
    #[structopt(short = "i", default_value = "0.0000000005")]
    initial_learning_rate: f32,
    /// Learning rate multiplier per epoch
    #[structopt(short = "m", default_value = "1.001")]
    learning_rate_multiplier: f32,
    /// Seed
    #[structopt(short = "z", default_value = "42")]
    seed: u32,
    /// Beta value for NAG
    #[structopt(short = "b", default_value = "0.9")]
    beta: f32,
    /// File to load
    #[structopt(parse(from_os_str))]
    file: PathBuf,
    /// Output directory
    #[structopt(parse(from_os_str))]
    output_dir: PathBuf,
}

fn sobel(image: &Array2<f32>) -> Array2<f32> {
    let down_filter = array![[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0],];
    let right_filter = down_filter.t().to_owned();
    let down = Conv2::new(down_filter).forward(&image).1;
    let right = Conv2::new(right_filter).forward(&image).1;
    (down.map(|f| f.powi(2)) + right.map(|f| f.powi(2))).map(|f| f.sqrt())
}

fn open_image(path: impl AsRef<Path>) -> ImageResult<Array2<f32>> {
    let image = open_gray_image(path)?;
    Ok(image.map(|&n| f32::from(n) / 255.0))
}

fn save_image(path: impl AsRef<Path>, image: &Array2<f32>) -> ImageResult<()> {
    let image = image.map(|&n| num::clamp(n * 255.0, 0.0, 255.0) as u8);
    save_gray_image(path, image.view())
}

fn main() -> ImageResult<()> {
    let opt = Opt::from_args();
    let image = open_image(opt.file)?;
    let sobel_image = sobel(&image);
    let conv_layers = 4;
    let filter_radius = 1usize;
    let filter_depth = 24usize;
    let filter_area = (filter_radius * 2 + 1).pow(2);
    let filter_volume = filter_area * filter_depth;
    let padding = (conv_layers * filter_radius - 1) as i32;
    #[allow(clippy::deref_addrof)]
    let expected = sobel_image.slice(s![padding..-padding, padding..-padding]);
    save_image(opt.output_dir.join("actual.png"), &expected.to_owned())?;
    let mut prng = make_prng(opt.seed);
    let mut prng_2filter = make_prng(prng.next_u32());
    let mut random_2filter = |mean: f32, variance: f32| -> Conv2<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2filter)
                    .take(filter_area),
            )
            .into_shape((filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_2nfilter = make_prng(prng.next_u32());
    let mut random_2nfilter = |mean: f32, variance: f32| -> Conv2n<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2n::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2nfilter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_3filter = make_prng(prng.next_u32());
    let mut random_3filter = |mean: f32, variance: f32| -> Conv3<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the volume of the filter.
        Conv3::new(
            Array::from_iter(
                Normal::new(mean / filter_volume as f32, variance / filter_volume as f32)
                    .unwrap()
                    .sample_iter(&mut prng_3filter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut random_blu = |mean: f32, variance: f32| -> Blu {
        // Xavier initialize by changing the variance to be 1/N where N is the number of neurons.
        let distr = Normal::new(mean, variance).unwrap();
        Blu::new(distr.sample(&mut prng), distr.sample(&mut prng))
    };
    let mut generate_filter = || {
        random_2nfilter(0.0, 2.0)
            .chain(Map3One(random_blu(0.0, 0.5)))
            .chain(random_3filter(0.0, 2.0))
            .chain(Reshape3to2::new())
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 2.0))
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(4.0, 1.0))
    };

    loop {
        let mut train_filter = generate_filter();
        let mut learn_rate = opt.initial_learning_rate;
        // Weird hack to initialize the momentum to zero without knowing the shape of the ndarray in advance.
        let mut momentum = {
            let (internal, output) = train_filter.forward(&image);
            train_filter.backward_train(&image, &internal, &output)
        };
        momentum *= 0.0f32;
        for i in 0..opt.epochs {
            // Compute beta * momentum.
            momentum *= opt.beta;
            train_filter.train(&momentum);
            let (internal, output) = train_filter.forward(&image);
            // Show the image if the frame is divisible by show_every.
            if i % opt.show_every == 0 {
                save_image(opt.output_dir.join(format!("epoch{:04}.png", i)), &output)?;
            }
            if i == opt.epochs - 1 {
                eprintln!("Finished!");
                return Ok(());
            }
            // Compute the loss for display only (we don't actually need the loss itself for backprop, just its derivative).
            let loss = (output.clone() - expected.view()).map(|n| n.powi(2)).sum();
            eprintln!("epoch {:04} loss: {}, learn_rate: {}", i, loss, learn_rate);
            if !loss.is_normal() {
                eprintln!("abnormal loss at epoch {}; starting over", i);
                break;
            }
            // The loss function is (n - t)^2, so 2*(n - t) is dE/df where
            // `E` is loss and `f` is output.
            let delta_loss = (output - expected).map(|n| 2.0 * n);
            // Compute the output delta.
            let output_delta = -learn_rate * delta_loss;
            // Compute the trainable variable delta.
            let mut train_delta = train_filter.backward_train(&image, &internal, &output_delta);
            // Make the train delta a small component.
            train_delta *= 1.0 - opt.beta;
            train_filter.train(&train_delta);
            // Add the small component to the momentum.
            momentum += train_delta;
            learn_rate *= opt.learning_rate_multiplier;
        }
    }
}

fn make_prng(seed: u32) -> Pcg64 {
    Pcg64::from_seed([
        (seed & 0xFF) as u8,
        ((seed >> 8) & 0xFF) as u8,
        ((seed >> 16) & 0xFF) as u8,
        ((seed >> 24) & 0xFF) as u8,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
    ])
}
<commit_msg>modify loss calculation to take the average across all outputs instead of the sum<commit_after>use image::ImageResult;
use mli::{Backward, Forward, Graph, Train};
use mli_conv::{Conv2, Conv2n, Conv3};
use mli_ndarray::{Map2One, Map3One, Reshape3to2};
use mli_relu::Blu;
use ndarray::{array, s, Array, Array2, OwnedRepr};
use ndarray_image::{open_gray_image, save_gray_image};
use rand_core::{RngCore, SeedableRng};
use rand_distr::{Distribution, Normal};
use rand_pcg::Pcg64;
use std::path::{Path, PathBuf};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "image", about = "Loads and convolves an image as a test")]
struct Opt {
    /// Number of epochs
    #[structopt(short = "t", default_value = "4000")]
    epochs: usize,
    /// Number of epochs per output image
    #[structopt(short = "s", default_value = "1")]
    show_every: usize,
    /// Initial learning rate
    #[structopt(short = "i", default_value = "0.0005")]
    initial_learning_rate: f32,
    /// Learning rate multiplier per epoch
    #[structopt(short = "m", default_value = "1.001")]
    learning_rate_multiplier: f32,
    /// Seed
    #[structopt(short = "z", default_value = "42")]
    seed: u32,
    /// Beta value for NAG
    #[structopt(short = "b", default_value = "0.9")]
    beta: f32,
    /// File to load
    #[structopt(parse(from_os_str))]
    file: PathBuf,
    /// Output directory
    #[structopt(parse(from_os_str))]
    output_dir: PathBuf,
}

fn sobel(image: &Array2<f32>) -> Array2<f32> {
    let down_filter = array![[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0],];
    let right_filter = down_filter.t().to_owned();
    let down = Conv2::new(down_filter).forward(&image).1;
    let right = Conv2::new(right_filter).forward(&image).1;
    (down.map(|f| f.powi(2)) + right.map(|f| f.powi(2))).map(|f| f.sqrt())
}

fn open_image(path: impl AsRef<Path>) -> ImageResult<Array2<f32>> {
    let image = open_gray_image(path)?;
    Ok(image.map(|&n| f32::from(n) / 255.0))
}

fn save_image(path: impl AsRef<Path>, image: &Array2<f32>) -> ImageResult<()> {
    let image = image.map(|&n| num::clamp(n * 255.0, 0.0, 255.0) as u8);
    save_gray_image(path, image.view())
}

fn main() -> ImageResult<()> {
    let opt = Opt::from_args();
    let image = open_image(opt.file)?;
    let sobel_image = sobel(&image);
    let conv_layers = 4;
    let filter_radius = 1usize;
    let filter_depth = 24usize;
    let filter_area = (filter_radius * 2 + 1).pow(2);
    let filter_volume = filter_area * filter_depth;
    let padding = (conv_layers * filter_radius - 1) as i32;
    #[allow(clippy::deref_addrof)]
    let expected = sobel_image.slice(s![padding..-padding, padding..-padding]);
    save_image(opt.output_dir.join("actual.png"), &expected.to_owned())?;
    let mut prng = make_prng(opt.seed);
    let mut prng_2filter = make_prng(prng.next_u32());
    let mut random_2filter = |mean: f32, variance: f32| -> Conv2<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2filter)
                    .take(filter_area),
            )
            .into_shape((filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_2nfilter = make_prng(prng.next_u32());
    let mut random_2nfilter = |mean: f32, variance: f32| -> Conv2n<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the area of the filter.
        Conv2n::new(
            Array::from_iter(
                Normal::new(mean / filter_area as f32, variance / filter_area as f32)
                    .unwrap()
                    .sample_iter(&mut prng_2nfilter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut prng_3filter = make_prng(prng.next_u32());
    let mut random_3filter = |mean: f32, variance: f32| -> Conv3<OwnedRepr<f32>> {
        // Xavier initialize by changing the variance to be 1/N where N is the volume of the filter.
        Conv3::new(
            Array::from_iter(
                Normal::new(mean / filter_volume as f32, variance / filter_volume as f32)
                    .unwrap()
                    .sample_iter(&mut prng_3filter)
                    .take(filter_volume),
            )
            .into_shape((filter_depth, filter_radius * 2 + 1, filter_radius * 2 + 1))
            .unwrap(),
        )
    };
    let mut random_blu = |mean: f32, variance: f32| -> Blu {
        // Xavier initialize by changing the variance to be 1/N where N is the number of neurons.
        let distr = Normal::new(mean, variance).unwrap();
        Blu::new(distr.sample(&mut prng), distr.sample(&mut prng))
    };
    let mut generate_filter = || {
        random_2nfilter(0.0, 2.0)
            .chain(Map3One(random_blu(0.0, 0.5)))
            .chain(random_3filter(0.0, 2.0))
            .chain(Reshape3to2::new())
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(0.0, 2.0))
            .chain(Map2One(random_blu(0.0, 0.5)))
            .chain(random_2filter(4.0, 1.0))
    };

    loop {
        let mut train_filter = generate_filter();
        let mut learn_rate = opt.initial_learning_rate;
        // Weird hack to initialize the momentum to zero without knowing the shape of the ndarray in advance.
        let mut momentum = {
            let (internal, output) = train_filter.forward(&image);
            train_filter.backward_train(&image, &internal, &output)
        };
        momentum *= 0.0f32;
        for i in 0..opt.epochs {
            // Compute beta * momentum.
            momentum *= opt.beta;
            train_filter.train(&momentum);
            let (internal, output) = train_filter.forward(&image);
            // Show the image if the frame is divisible by show_every.
            if i % opt.show_every == 0 {
                save_image(opt.output_dir.join(format!("epoch{:04}.png", i)), &output)?;
            }
            if i == opt.epochs - 1 {
                eprintln!("Finished!");
                return Ok(());
            }
            let output_len = output.len() as f32;
            // Compute the loss for display only (we don't actually need the loss itself for backprop, just its derivative).
            let loss = (output.clone() - expected.view()).map(|n| n.powi(2)).sum() / output_len;
            eprintln!("epoch {:04} loss: {}, learn_rate: {}", i, loss, learn_rate);
            if !loss.is_normal() {
                eprintln!("abnormal loss at epoch {}; starting over", i);
                break;
            }
            // The loss function is (n - t)^2, so 2*(n - t) is dE/df where
            // `E` is loss and `f` is output.
            let delta_loss = (output - expected).map(|n| 2.0 * n / output_len);
            // Compute the output delta.
            let output_delta = -learn_rate * delta_loss;
            // Compute the trainable variable delta.
            let mut train_delta = train_filter.backward_train(&image, &internal, &output_delta);
            // Make the train delta a small component.
            train_delta *= 1.0 - opt.beta;
            train_filter.train(&train_delta);
            // Add the small component to the momentum.
            momentum += train_delta;
            learn_rate *= opt.learning_rate_multiplier;
        }
    }
}

fn make_prng(seed: u32) -> Pcg64 {
    Pcg64::from_seed([
        (seed & 0xFF) as u8,
        ((seed >> 8) & 0xFF) as u8,
        ((seed >> 16) & 0xFF) as u8,
        ((seed >> 24) & 0xFF) as u8,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
    ])
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py
use avro_rs::{from_avro_datum, to_avro_datum, types::Value, Error, Schema};
use lazy_static::lazy_static;
use std::io::Cursor;

lazy_static! {
    static ref SCHEMAS_TO_VALIDATE: Vec<(&'static str, Value)> = vec![
        (r#""null""#, Value::Null),
        (r#""boolean""#, Value::Boolean(true)),
        (r#""string""#, Value::String("adsfasdf09809dsf-=adsf".to_string())),
        (r#""bytes""#, Value::Bytes("12345abcd".to_string().into_bytes())),
        (r#""int""#, Value::Int(1234)),
        (r#""long""#, Value::Long(1234)),
        (r#""float""#, Value::Float(1234.0)),
        (r#""double""#, Value::Double(1234.0)),
        (r#"{"type": "fixed", "name": "Test", "size": 1}"#, Value::Fixed(1, vec![b'B'])),
        (r#"{"type": "enum", "name": "Test", "symbols": ["A", "B"]}"#, Value::Enum(1, "B".to_string())),
        (r#"{"type": "array", "items": "long"}"#, Value::Array(vec![Value::Long(1), Value::Long(3), Value::Long(2)])),
        (r#"{"type": "map", "values": "long"}"#, Value::Map([("a".to_string(), Value::Long(1i64)), ("b".to_string(), Value::Long(3i64)), ("c".to_string(), Value::Long(2i64))].iter().cloned().collect())),
        (r#"["string", "null", "long"]"#, Value::Union(1, Box::new(Value::Null))),
        (r#"{"type": "record", "name": "Test", "fields": [{"name": "f", "type": "long"}]}"#, Value::Record(vec![("f".to_string(), Value::Long(1))]))
    ];

    static ref BINARY_ENCODINGS: Vec<(i64, Vec<u8>)> = vec![
        (0, vec![0x00]),
        (-1, vec![0x01]),
        (1, vec![0x02]),
        (-2, vec![0x03]),
        (2, vec![0x04]),
        (-64, vec![0x7f]),
        (64, vec![0x80, 0x01]),
        (8192, vec![0x80, 0x80, 0x01]),
        (-8193, vec![0x81, 0x80, 0x01]),
    ];

    static ref DEFAULT_VALUE_EXAMPLES: Vec<(&'static str, &'static str, Value)> = vec![
        (r#""null""#, "null", Value::Null),
        (r#""boolean""#, "true", Value::Boolean(true)),
        (r#""string""#, r#""foo""#, Value::String("foo".to_string())),
        // TODO: (#96) investigate why this is failing
        //(r#""bytes""#, r#""\u00FF\u00FF""#, Value::Bytes(vec![0xff, 0xff])),
        (r#""int""#, "5", Value::Int(5)),
        (r#""long""#, "5", Value::Long(5)),
        (r#""float""#, "1.1", Value::Float(1.1)),
        (r#""double""#, "1.1", Value::Double(1.1)),
        // TODO: (#96) investigate why this is failing
        //(r#"{"type": "fixed", "name": "F", "size": 2}"#, r#""\u00FF\u00FF""#, Value::Bytes(vec![0xff, 0xff])),
        (r#"{"type": "enum", "name": "F", "symbols": ["FOO", "BAR"]}"#, r#""FOO""#, Value::Enum(0, "FOO".to_string())),
        (r#"{"type": "array", "items": "int"}"#, "[1, 2, 3]", Value::Array(vec![Value::Int(1), Value::Int(2), Value::Int(3)])),
        (r#"{"type": "map", "values": "int"}"#, r#"{"a": 1, "b": 2}"#, Value::Map([("a".to_string(), Value::Int(1)), ("b".to_string(), Value::Int(2))].iter().cloned().collect())),
        (r#"["int", "null"]"#, "5", Value::Union(0, Box::new(Value::Int(5)))),
        (r#"{"type": "record", "name": "F", "fields": [{"name": "A", "type": "int"}]}"#, r#"{"A": 5}"#,Value::Record(vec![("A".to_string(), Value::Int(5))])),
        (r#"["null", "int"]"#, "null", Value::Union(0, Box::new(Value::Null))),
    ];

    static ref LONG_RECORD_SCHEMA: Schema = Schema::parse_str(r#"
    {
        "type": "record",
        "name": "Test",
        "fields": [
            {"name": "A", "type": "int"},
            {"name": "B", "type": "int"},
            {"name": "C", "type": "int"},
            {"name": "D", "type": "int"},
            {"name": "E", "type": "int"},
            {"name": "F", "type": "int"},
            {"name": "G", "type": "int"}
        ]
    }
    "#).unwrap();

    static ref LONG_RECORD_DATUM: Value = Value::Record(vec![
        ("A".to_string(), Value::Int(1)),
        ("B".to_string(), Value::Int(2)),
        ("C".to_string(), Value::Int(3)),
        ("D".to_string(), Value::Int(4)),
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::Int(6)),
        ("G".to_string(), Value::Int(7)),
    ]);
}

#[test]
fn test_validate() {
    for (raw_schema, value) in SCHEMAS_TO_VALIDATE.iter() {
        let schema = Schema::parse_str(raw_schema).unwrap();
        assert!(
            value.validate(&schema),
            "value {:?} does not validate schema: {}",
            value,
            raw_schema
        );
    }
}

#[test]
fn test_round_trip() {
    for (raw_schema, value) in SCHEMAS_TO_VALIDATE.iter() {
        let schema = Schema::parse_str(raw_schema).unwrap();
        let encoded = to_avro_datum(&schema, value.clone()).unwrap();
        let decoded = from_avro_datum(&schema, &mut Cursor::new(encoded), None).unwrap();
        assert_eq!(value, &decoded);
    }
}

#[test]
fn test_binary_int_encoding() {
    for (number, hex_encoding) in BINARY_ENCODINGS.iter() {
        let encoded = to_avro_datum(&Schema::Int, Value::Int(*number as i32)).unwrap();
        assert_eq!(&encoded, hex_encoding);
    }
}

#[test]
fn test_binary_long_encoding() {
    for (number, hex_encoding) in BINARY_ENCODINGS.iter() {
        let encoded = to_avro_datum(&Schema::Long, Value::Long(*number as i64)).unwrap();
        assert_eq!(&encoded, hex_encoding);
    }
}

#[test]
fn test_schema_promotion() {
    // Each schema is present in order of promotion (int -> long, long -> float, float -> double)
    // Each value represents the expected decoded value when promoting a value previously encoded with a promotable schema
    let promotable_schemas = vec![r#""int""#, r#""long""#, r#""float""#, r#""double""#];
    let promotable_values = vec![
        Value::Int(219),
        Value::Long(219),
        Value::Float(219.0),
        Value::Double(219.0),
    ];
    for (i, writer_raw_schema) in promotable_schemas.iter().enumerate() {
        let writer_schema = Schema::parse_str(writer_raw_schema).unwrap();
        let original_value = &promotable_values[i];
        for (j, reader_raw_schema) in promotable_schemas.iter().enumerate().skip(i + 1) {
            let reader_schema = Schema::parse_str(reader_raw_schema).unwrap();
            let encoded = to_avro_datum(&writer_schema, original_value.clone()).unwrap();
            let decoded = from_avro_datum(
                &writer_schema,
                &mut Cursor::new(encoded),
                Some(&reader_schema),
            )
            .unwrap_or_else(|_| {
                panic!(
                    "failed to decode {:?} with schema: {:?}",
                    original_value, reader_raw_schema,
                )
            });
            assert_eq!(decoded, promotable_values[j]);
        }
    }
}

#[test]
fn test_unknown_symbol() {
    let writer_schema =
        Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["FOO", "BAR"]}"#)
            .unwrap();
    let reader_schema =
        Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["BAR", "BAZ"]}"#)
            .unwrap();
    let original_value = Value::Enum(0, "FOO".to_string());
    let encoded = to_avro_datum(&writer_schema, original_value).unwrap();
    let decoded = from_avro_datum(
        &writer_schema,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    );
    assert!(decoded.is_err());
}

#[test]
fn test_default_value() {
    for (field_type, default_json, default_datum) in DEFAULT_VALUE_EXAMPLES.iter() {
        let reader_schema = Schema::parse_str(&format!(
            r#"{{
                "type": "record",
                "name": "Test",
                "fields": [
                    {{"name": "H", "type": {}, "default": {}}}
                ]
            }}"#,
            field_type, default_json
        ))
        .unwrap();
        let datum_to_read = Value::Record(vec![("H".to_string(), default_datum.clone())]);
        let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
        let datum_read = from_avro_datum(
            &LONG_RECORD_SCHEMA,
            &mut Cursor::new(encoded),
            Some(&reader_schema),
        )
        .unwrap();
        assert_eq!(
            datum_read, datum_to_read,
            "{} -> {}",
            *field_type, *default_json
        );
    }
}

#[test]
fn test_no_default_value() -> Result<(), Error> {
    let reader_schema = Schema::parse_str(
        r#"{
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "H", "type": "int"}
            ]
        }"#,
    )
    .unwrap();
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let result = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    );
    assert!(result.is_err());
    Ok(())
}

#[test]
fn test_projection() {
    let reader_schema = Schema::parse_str(
        r#"
        {
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "E", "type": "int"},
                {"name": "F", "type": "int"}
            ]
        }
    "#,
    )
    .unwrap();
    let datum_to_read = Value::Record(vec![
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::Int(6)),
    ]);
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let datum_read = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    )
    .unwrap();
    assert_eq!(datum_to_read, datum_read);
}

#[test]
fn test_field_order() {
    let reader_schema = Schema::parse_str(
        r#"
        {
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "F", "type": "int"},
                {"name": "E", "type": "int"}
            ]
        }
    "#,
    )
    .unwrap();
    let datum_to_read = Value::Record(vec![
        ("F".to_string(), Value::Int(6)),
        ("E".to_string(), Value::Int(5)),
    ]);
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let datum_read = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    )
    .unwrap();
    assert_eq!(datum_to_read, datum_read);
}

#[test]
fn test_type_exception() -> Result<(), String> {
    let writer_schema = Schema::parse_str(
        r#"
        {
             "type": "record",
             "name": "Test",
             "fields": [
                {"name": "F", "type": "int"},
                {"name": "E", "type": "int"}
             ]
        }
    "#,
    )
    .unwrap();
    let datum_to_write = Value::Record(vec![
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::String(String::from("Bad"))),
    ]);
    let encoded = to_avro_datum(&writer_schema, datum_to_write);
    match encoded {
        Ok(_) => Err(String::from("Expected ValidationError, got Ok")),
        Err(Error::Validation) => Ok(()),
        Err(ref e) => Err(format!("Expected ValidationError, got {:?}", e)),
    }
}
<commit_msg>AVRO-3316 [Rust] build breaks in docker build<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Port of https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py
use avro_rs::{from_avro_datum, to_avro_datum, types::Value, Error, Schema};
use lazy_static::lazy_static;
use std::io::Cursor;

lazy_static! {
    static ref SCHEMAS_TO_VALIDATE: Vec<(&'static str, Value)> = vec![
        (r#""null""#, Value::Null),
        (r#""boolean""#, Value::Boolean(true)),
        (r#""string""#, Value::String("adsfasdf09809dsf-=adsf".to_string())),
        (r#""bytes""#, Value::Bytes("12345abcd".to_string().into_bytes())),
        (r#""int""#, Value::Int(1234)),
        (r#""long""#, Value::Long(1234)),
        (r#""float""#, Value::Float(1234.0)),
        (r#""double""#, Value::Double(1234.0)),
        (r#"{"type": "fixed", "name": "Test", "size": 1}"#, Value::Fixed(1, vec![b'B'])),
        (r#"{"type": "enum", "name": "Test", "symbols": ["A", "B"]}"#, Value::Enum(1, "B".to_string())),
        (r#"{"type": "array", "items": "long"}"#, Value::Array(vec![Value::Long(1), Value::Long(3), Value::Long(2)])),
        (r#"{"type": "map", "values": "long"}"#, Value::Map([("a".to_string(), Value::Long(1i64)), ("b".to_string(), Value::Long(3i64)), ("c".to_string(), Value::Long(2i64))].iter().cloned().collect())),
        (r#"["string", "null", "long"]"#, Value::Union(1, Box::new(Value::Null))),
        (r#"{"type": "record", "name": "Test", "fields": [{"name": "f", "type": "long"}]}"#, Value::Record(vec![("f".to_string(), Value::Long(1))]))
    ];

    static ref BINARY_ENCODINGS: Vec<(i64, Vec<u8>)> = vec![
        (0, vec![0x00]),
        (-1, vec![0x01]),
        (1, vec![0x02]),
        (-2, vec![0x03]),
        (2, vec![0x04]),
        (-64, vec![0x7f]),
        (64, vec![0x80, 0x01]),
        (8192, vec![0x80, 0x80, 0x01]),
        (-8193, vec![0x81, 0x80, 0x01]),
    ];

    static ref DEFAULT_VALUE_EXAMPLES: Vec<(&'static str, &'static str, Value)> = vec![
        (r#""null""#, "null", Value::Null),
        (r#""boolean""#, "true", Value::Boolean(true)),
        (r#""string""#, r#""foo""#, Value::String("foo".to_string())),
        // TODO: (#96) investigate why this is failing
        //(r#""bytes""#, r#""\u00FF\u00FF""#, Value::Bytes(vec![0xff, 0xff])),
        (r#""int""#, "5", Value::Int(5)),
        (r#""long""#, "5", Value::Long(5)),
        (r#""float""#, "1.1", Value::Float(1.1)),
        (r#""double""#, "1.1", Value::Double(1.1)),
        // TODO: (#96) investigate why this is failing
        //(r#"{"type": "fixed", "name": "F", "size": 2}"#, r#""\u00FF\u00FF""#, Value::Bytes(vec![0xff, 0xff])),
        (r#"{"type": "enum", "name": "F", "symbols": ["FOO", "BAR"]}"#, r#""FOO""#, Value::Enum(0, "FOO".to_string())),
        (r#"{"type": "array", "items": "int"}"#, "[1, 2, 3]", Value::Array(vec![Value::Int(1), Value::Int(2), Value::Int(3)])),
        (r#"{"type": "map", "values": "int"}"#, r#"{"a": 1, "b": 2}"#, Value::Map([("a".to_string(), Value::Int(1)), ("b".to_string(), Value::Int(2))].iter().cloned().collect())),
        (r#"["int", "null"]"#, "5", Value::Union(0, Box::new(Value::Int(5)))),
        (r#"{"type": "record", "name": "F", "fields": [{"name": "A", "type": "int"}]}"#, r#"{"A": 5}"#,Value::Record(vec![("A".to_string(), Value::Int(5))])),
        (r#"["null", "int"]"#, "null", Value::Union(0, Box::new(Value::Null))),
    ];

    static ref LONG_RECORD_SCHEMA: Schema = Schema::parse_str(r#"
    {
        "type": "record",
        "name": "Test",
        "fields": [
            {"name": "A", "type": "int"},
            {"name": "B", "type": "int"},
            {"name": "C", "type": "int"},
            {"name": "D", "type": "int"},
            {"name": "E", "type": "int"},
            {"name": "F", "type": "int"},
            {"name": "G", "type": "int"}
        ]
    }
    "#).unwrap();

    static ref LONG_RECORD_DATUM: Value = Value::Record(vec![
        ("A".to_string(), Value::Int(1)),
        ("B".to_string(), Value::Int(2)),
        ("C".to_string(), Value::Int(3)),
        ("D".to_string(), Value::Int(4)),
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::Int(6)),
        ("G".to_string(), Value::Int(7)),
    ]);
}

#[test]
fn test_validate() {
    for (raw_schema, value) in SCHEMAS_TO_VALIDATE.iter() {
        let schema = Schema::parse_str(raw_schema).unwrap();
        assert!(
            value.validate(&schema),
            "value {:?} does not validate schema: {}",
            value,
            raw_schema
        );
    }
}

#[test]
fn test_round_trip() {
    for (raw_schema, value) in SCHEMAS_TO_VALIDATE.iter() {
        let schema = Schema::parse_str(raw_schema).unwrap();
        let encoded = to_avro_datum(&schema, value.clone()).unwrap();
        let decoded = from_avro_datum(&schema, &mut Cursor::new(encoded), None).unwrap();
        assert_eq!(value, &decoded);
    }
}

#[test]
fn test_binary_int_encoding() {
    for (number, hex_encoding) in BINARY_ENCODINGS.iter() {
        let encoded = to_avro_datum(&Schema::Int, Value::Int(*number as i32)).unwrap();
        assert_eq!(&encoded, hex_encoding);
    }
}

#[test]
fn test_binary_long_encoding() {
    for (number, hex_encoding) in BINARY_ENCODINGS.iter() {
        let encoded = to_avro_datum(&Schema::Long, Value::Long(*number as i64)).unwrap();
        assert_eq!(&encoded, hex_encoding);
    }
}

#[test]
fn test_schema_promotion() {
    // Each schema is present in order of promotion (int -> long, long -> float, float -> double)
    // Each value represents the expected decoded value when promoting a value previously encoded with a promotable schema
    let promotable_schemas = vec![r#""int""#, r#""long""#, r#""float""#, r#""double""#];
    let promotable_values = vec![
        Value::Int(219),
        Value::Long(219),
        Value::Float(219.0),
        Value::Double(219.0),
    ];
    for (i, writer_raw_schema) in promotable_schemas.iter().enumerate() {
        let writer_schema = Schema::parse_str(writer_raw_schema).unwrap();
        let original_value = &promotable_values[i];
        for (j, reader_raw_schema) in promotable_schemas.iter().enumerate().skip(i + 1) {
            let reader_schema = Schema::parse_str(reader_raw_schema).unwrap();
            let encoded = to_avro_datum(&writer_schema, original_value.clone()).unwrap();
            let decoded = from_avro_datum(
                &writer_schema,
                &mut Cursor::new(encoded),
                Some(&reader_schema),
            )
            .unwrap_or_else(|_| {
                panic!(
                    "failed to decode {:?} with schema: {:?}",
                    original_value, reader_raw_schema,
                )
            });
            assert_eq!(decoded, promotable_values[j]);
        }
    }
}

#[test]
fn test_unknown_symbol() {
    let writer_schema =
        Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["FOO", "BAR"]}"#)
            .unwrap();
    let reader_schema =
        Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["BAR", "BAZ"]}"#)
            .unwrap();
    let original_value = Value::Enum(0, "FOO".to_string());
    let encoded = to_avro_datum(&writer_schema, original_value).unwrap();
    let decoded = from_avro_datum(
        &writer_schema,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    );
    assert!(decoded.is_err());
}

#[test]
fn test_default_value() {
    for (field_type, default_json, default_datum) in DEFAULT_VALUE_EXAMPLES.iter() {
        let reader_schema = Schema::parse_str(&format!(
            r#"{{
                "type": "record",
                "name": "Test",
                "fields": [
                    {{"name": "H", "type": {}, "default": {}}}
                ]
            }}"#,
            field_type, default_json
        ))
        .unwrap();
        let datum_to_read = Value::Record(vec![("H".to_string(), default_datum.clone())]);
        let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
        let datum_read = from_avro_datum(
            &LONG_RECORD_SCHEMA,
            &mut Cursor::new(encoded),
            Some(&reader_schema),
        )
        .unwrap();
        assert_eq!(
            datum_read, datum_to_read,
            "{} -> {}",
            *field_type, *default_json
        );
    }
}

#[test]
fn test_no_default_value() {
    let reader_schema = Schema::parse_str(
        r#"{
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "H", "type": "int"}
            ]
        }"#,
    )
    .unwrap();
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let result = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    );
    assert!(result.is_err());
}

#[test]
fn test_projection() {
    let reader_schema = Schema::parse_str(
        r#"
        {
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "E", "type": "int"},
                {"name": "F", "type": "int"}
            ]
        }
    "#,
    )
    .unwrap();
    let datum_to_read = Value::Record(vec![
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::Int(6)),
    ]);
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let datum_read = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    )
    .unwrap();
    assert_eq!(datum_to_read, datum_read);
}

#[test]
fn test_field_order() {
    let reader_schema = Schema::parse_str(
        r#"
        {
            "type": "record",
            "name": "Test",
            "fields": [
                {"name": "F", "type": "int"},
                {"name": "E", "type": "int"}
            ]
        }
    "#,
    )
    .unwrap();
    let datum_to_read = Value::Record(vec![
        ("F".to_string(), Value::Int(6)),
        ("E".to_string(), Value::Int(5)),
    ]);
    let encoded = to_avro_datum(&LONG_RECORD_SCHEMA, LONG_RECORD_DATUM.clone()).unwrap();
    let datum_read = from_avro_datum(
        &LONG_RECORD_SCHEMA,
        &mut Cursor::new(encoded),
        Some(&reader_schema),
    )
    .unwrap();
    assert_eq!(datum_to_read, datum_read);
}

#[test]
fn test_type_exception() -> Result<(), String> {
    let writer_schema = Schema::parse_str(
        r#"
        {
             "type": "record",
             "name": "Test",
             "fields": [
                {"name": "F", "type": "int"},
                {"name": "E", "type": "int"}
             ]
        }
    "#,
    )
    .unwrap();
    let datum_to_write = Value::Record(vec![
        ("E".to_string(), Value::Int(5)),
        ("F".to_string(), Value::String(String::from("Bad"))),
    ]);
    let encoded = to_avro_datum(&writer_schema, datum_to_write);
    match encoded {
        Ok(_) => Err(String::from("Expected ValidationError, got Ok")),
        Err(Error::Validation) => Ok(()),
        Err(ref e) => Err(format!("Expected ValidationError, got {:?}", e)),
    }
}
<|endoftext|>


<commit_before>use ::crypto::Hash;
use ::messages::{RawMessage, Precommit, BlockProof};
use ::storage::{StorageValue, ListTable, MapTable, MerkleTable, MerklePatriciaTable, HeightBytes,
                Error, Map, List, RootProofNode};

use super::{Block, Blockchain};
use super::config::StoredConfiguration;

pub type ConfigurationData = Vec<u8>;

type StorageView = ::storage::View;

pub struct Schema<'a> {
    view: &'a StorageView,
}

impl<'a> Schema<'a> {
    pub fn new(view: &'a StorageView) -> Schema {
        Schema { view: view }
    }

    pub fn transactions(&self) -> MapTable<StorageView, Hash, RawMessage> {
        MapTable::new(vec![00], self.view)
    }

    pub fn blocks(&self) -> MapTable<StorageView, Hash, Block> {
        MapTable::new(vec![01], self.view)
    }

    pub fn heights(&self) -> ListTable<MapTable<StorageView, [u8], Vec<u8>>, u64, Hash> {
        ListTable::new(MapTable::new(vec![02], self.view))
    }

    pub fn block_txs(&self,
                     height: u64)
                     -> MerkleTable<MapTable<StorageView, [u8], Vec<u8>>, u32, Hash> {
        MerkleTable::new(MapTable::new([&[03u8] as &[u8], &height.serialize()].concat(), self.view))
    }

    pub fn precommits(&self,
                      hash: &Hash)
                      -> ListTable<MapTable<StorageView, [u8], Vec<u8>>, u32, Precommit> {
        ListTable::new(MapTable::new([&[03], hash.as_ref()].concat(), self.view))
    }

    pub fn block_and_precommits(&self, height: u64) -> Result<Option<BlockProof>, Error> {
        let block_hash = match self.heights().get(height)? {
            None => return Ok(None),
            Some(block_hash) => block_hash,
        };
        let block = self.blocks().get(&block_hash)?.unwrap();
        let precommits_table = self.precommits(&block_hash);
        let precommits = precommits_table.values()?;
        let res = BlockProof {
            block: block,
            precommits: precommits,
        };
        Ok(Some(res))
    }

    pub fn configs
        (&self)
         -> MerklePatriciaTable<MapTable<StorageView, [u8], Vec<u8>>, HeightBytes, ConfigurationData> {
        // configs patricia merkletree <высота блока> json
        MerklePatriciaTable::new(MapTable::new(vec![06], self.view))
    }

    // TODO: consider List index to reduce storage volume
    pub fn configs_heights(&self)
                           -> ListTable<MapTable<StorageView, [u8], Vec<u8>>, u64, HeightBytes> {
        ListTable::new(MapTable::new(vec![07], self.view))
    }

    pub fn state_hash_aggregator(&self)
                            -> MerklePatriciaTable<MapTable<StorageView, [u8], Vec<u8>>, Hash, Hash> {
        MerklePatriciaTable::new(MapTable::new(vec![08], self.view))
    }

    pub fn last_block(&self) -> Result<Option<Block>, Error> {
        Ok(match self.heights().last()? {
            Some(hash) => Some(self.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }

    pub fn last_height(&self) -> Result<u64, Error> {
        self.last_block().map(|block| block.unwrap().height())
    }

    pub fn commit_actual_configuration(&self,
                                       actual_from: u64,
                                       config_data: &[u8])
                                       -> Result<(), Error> {
        let height_bytecode = actual_from.into();
        self.configs().put(&height_bytecode, config_data.to_vec())?;
        self.configs_heights().append(height_bytecode)?;
        // TODO: clear storages
        Ok(())
    }

    pub fn get_actual_configurations_height(&self) -> Result<u64, Error> {
        // TODO improve perfomance
        let h = self.last_height()? + 1;
        let heights = self.configs_heights();
        let height_values = heights.values().unwrap();

        let idx = height_values.into_iter()
            .rposition(|r| u64::from(r) <= h)
            .unwrap();
        Ok(idx as u64)
    }

    pub fn get_actual_configuration(&self) -> Result<StoredConfiguration, Error> {
        let height = self.get_actual_configurations_height()?;
        self.get_configuration_at_height(height).map(|x| x.unwrap())
    }

    // FIXME Replace by result?
    pub fn get_configuration_at_height(&self,
                                       height: u64)
                                       -> Result<Option<StoredConfiguration>, Error> {
        let configs = self.configs();
        if let Some(config) = configs.get(&height.into())? {
            match StoredConfiguration::deserialize(&config) {
                Ok(configuration) => {
                    return Ok(Some(configuration));
                }
                Err(_) => {
                    error!("Can't parse found configuration at height: {}", height);
                }
            }
        }
        Ok(None)
    }

    pub fn core_state_hash(&self) -> Result<Vec<Hash>, Error> {
        Ok(vec![self.configs().root_hash()?])
    }

    pub fn get_proof_to_service_table(&self, service_id: u16, table_idx: usize) -> Result<RootProofNode<Hash>, Error>
    {
        let key = Blockchain::service_table_unique_key(service_id, table_idx);
        let sum_table = self.state_hash_aggregator();
        sum_table.construct_path_to_key(key)
    }
}<commit_msg>Add method to check following configuration<commit_after>use crypto::Hash;
use messages::{RawMessage, Precommit, BlockProof};
use storage::{StorageValue, ListTable, MapTable, MerkleTable, MerklePatriciaTable, HeightBytes,
              Error, Map, List, RootProofNode, View};

use super::{Block, Blockchain};
use super::config::StoredConfiguration;

pub type ConfigurationData = Vec<u8>;

pub struct Schema<'a> {
    view: &'a View,
}

impl<'a> Schema<'a> {
    pub fn new(view: &'a View) -> Schema {
        Schema { view: view }
    }

    pub fn transactions(&self) -> MapTable<View, Hash, RawMessage> {
        MapTable::new(vec![00], self.view)
    }

    pub fn blocks(&self) -> MapTable<View, Hash, Block> {
        MapTable::new(vec![01], self.view)
    }

    pub fn heights(&self) -> ListTable<MapTable<View, [u8], Vec<u8>>, u64, Hash> {
        ListTable::new(MapTable::new(vec![02], self.view))
    }

    pub fn block_txs(&self,
                     height: u64)
                     -> MerkleTable<MapTable<View, [u8], Vec<u8>>, u32, Hash> {
        MerkleTable::new(MapTable::new([&[03u8] as &[u8], &height.serialize()].concat(), self.view))
    }

    pub fn precommits(&self,
                      hash: &Hash)
                      -> ListTable<MapTable<View, [u8], Vec<u8>>, u32, Precommit> {
        ListTable::new(MapTable::new([&[03], hash.as_ref()].concat(), self.view))
    }

    pub fn block_and_precommits(&self, height: u64) -> Result<Option<BlockProof>, Error> {
        let block_hash = match self.heights().get(height)? {
            None => return Ok(None),
            Some(block_hash) => block_hash,
        };
        let block = self.blocks().get(&block_hash)?.unwrap();
        let precommits_table = self.precommits(&block_hash);
        let precommits = precommits_table.values()?;
        let res = BlockProof {
            block: block,
            precommits: precommits,
        };
        Ok(Some(res))
    }

    pub fn configs
        (&self)
         -> MerklePatriciaTable<MapTable<View, [u8], Vec<u8>>, HeightBytes, ConfigurationData> {
        // configs patricia merkletree <высота блока> json
        MerklePatriciaTable::new(MapTable::new(vec![06], self.view))
    }

    // TODO: consider List index to reduce storage volume
    pub fn configs_heights(&self)
                           -> ListTable<MapTable<View, [u8], Vec<u8>>, u64, HeightBytes> {
        ListTable::new(MapTable::new(vec![07], self.view))
    }

    pub fn state_hash_aggregator
        (&self)
         -> MerklePatriciaTable<MapTable<View, [u8], Vec<u8>>, Hash, Hash> {
        MerklePatriciaTable::new(MapTable::new(vec![08], self.view))
    }

    pub fn last_block(&self) -> Result<Option<Block>, Error> {
        Ok(match self.heights().last()? {
            Some(hash) => Some(self.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }

    pub fn last_height(&self) -> Result<u64, Error> {
        self.last_block().map(|block| block.unwrap().height())
    }

    pub fn commit_actual_configuration(&self,
                                       actual_from: u64,
                                       config_data: &[u8])
                                       -> Result<(), Error> {
        let height_bytecode = actual_from.into();
        self.configs().put(&height_bytecode, config_data.to_vec())?;
        self.configs_heights().append(height_bytecode)?;
        // TODO: clear storages
        Ok(())
    }

    pub fn get_actual_configurations_index(&self) -> Result<u64, Error> {
        // TODO improve perfomance, use iterators and binary search
        let h = self.last_height()? + 1;
        let heights = self.configs_heights();
        let height_values = heights.values().unwrap();

        let idx = height_values.into_iter()
            .rposition(|r| u64::from(r) <= h)
            .unwrap();
        Ok(idx as u64)
    }

    pub fn get_actual_configuration(&self) -> Result<StoredConfiguration, Error> {
        let idx = self.get_actual_configurations_index()?;
        let height = self.configs_heights()
            .get(idx)?
            .unwrap();
        self.get_configuration_at_height(height).map(|x| x.unwrap())
    }

    pub fn get_following_configuration(&self) -> Result<Option<StoredConfiguration>, Error> {
        let idx = self.get_actual_configurations_index()?;
        if let Some(height) = self.configs_heights().get(idx + 1)? {
            self.get_configuration_at_height(height)
        } else {
            Ok(None)
        }
    }

    pub fn get_configuration_at_height<H>(&self,
                                          height: H)
                                          -> Result<Option<StoredConfiguration>, Error>
        where H: Into<HeightBytes>
    {
        let configs = self.configs();
        let height = height.into();
        if let Some(config) = configs.get(&height)? {
            match StoredConfiguration::deserialize(&config) {
                Ok(configuration) => {
                    return Ok(Some(configuration));
                }
                Err(_) => {
                    error!("Can't parse found configuration at height: {}", u64::from(height));
                }
            }
        }
        Ok(None)
    }

    pub fn core_state_hash(&self) -> Result<Vec<Hash>, Error> {
        Ok(vec![self.configs().root_hash()?])
    }

    pub fn get_proof_to_service_table(&self,
                                      service_id: u16,
                                      table_idx: usize)
                                      -> Result<RootProofNode<Hash>, Error> {
        let key = Blockchain::service_table_unique_key(service_id, table_idx);
        let sum_table = self.state_hash_aggregator();
        sum_table.construct_path_to_key(key)
    }
}<|endoftext|>


<commit_before>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crypto::Hash;
use messages::{RawMessage, Precommit};
use storage::{Snapshot, Fork, StorageKey, StorageValue, ListIndex, MapIndex, ProofListIndex,
              ProofMapIndex, MapProof};
use helpers::Height;
use super::{Block, BlockProof, Blockchain};
use super::config::StoredConfiguration;

/// Generates an array of bytes from the `prefix`.
pub fn gen_prefix<K: StorageKey>(prefix: &K) -> Vec<u8> {
    let mut res = vec![0; prefix.size()];
    prefix.write(&mut res[..]);
    res
}

encoding_struct! (
    /// Configuration index.
    struct ConfigReference {
        const SIZE = 40;
        /// The height, starting from which this configuration becomes actual.
        field actual_from: Height [00 => 08]
        /// Hash of the configuration contents that serialized as raw bytes vec.
        field cfg_hash:    &Hash  [08 => 40]
    }
);

encoding_struct! (
    /// Transaction location in block.
    struct TxLocation {
        const SIZE = 16;
        /// Height of block in the blockchain.
        field block_height:         Height  [00 => 08]
        /// Index in block.
        field position_in_block:    u64     [08 => 16]
    }
);

/// Information schema for `exonum-core`.
#[derive(Debug)]
pub struct Schema<T> {
    view: T,
}

impl<T> Schema<T>
where
    T: AsRef<Snapshot>,
{
    /// Constructs information schema for the given `snapshot`.
    pub fn new(snapshot: T) -> Schema<T> {
        Schema { view: snapshot }
    }

    /// Returns table that represents a map from transaction hash into raw transaction message.
    pub fn transactions(&self) -> MapIndex<&T, Hash, RawMessage> {
        MapIndex::new("core.transactions", &self.view)
    }

    /// Returns table that keeps the block height and tx position inside block for every
    /// transaction hash.
    pub fn tx_location_by_tx_hash(&self) -> MapIndex<&T, Hash, TxLocation> {
        MapIndex::new("core.tx_location_by_tx_hash", &self.view)
    }

    /// Returns table that stores block object for every block height.
    pub fn blocks(&self) -> MapIndex<&T, Hash, Block> {
        MapIndex::new("core.blocks", &self.view)
    }

    /// Returns table that keeps block hash for the corresponding height.
    pub fn block_hashes_by_height(&self) -> ListIndex<&T, Hash> {
        ListIndex::new("core.block_hashes_by_height", &self.view)
    }

    /// Returns table that keeps a list of transactions for the each block.
    pub fn block_txs(&self, height: Height) -> ProofListIndex<&T, Hash> {
        let height: u64 = height.into();
        ProofListIndex::with_prefix("core.block_txs", gen_prefix(&height), &self.view)
    }

    /// Returns table that saves a list of precommits for block with given hash.
    pub fn precommits(&self, hash: &Hash) -> ListIndex<&T, Precommit> {
        ListIndex::with_prefix("core.precommits", gen_prefix(hash), &self.view)
    }

    /// Returns table that represents a map from configuration hash into contents.
    pub fn configs(&self) -> ProofMapIndex<&T, Hash, StoredConfiguration> {
        // configs patricia merkletree <block height> json
        ProofMapIndex::new("core.configs", &self.view)
    }

    /// Returns auxiliary table that keeps hash references to configurations in order
    /// of increasing their `actual_from` height.
    pub fn configs_actual_from(&self) -> ListIndex<&T, ConfigReference> {
        ListIndex::new("core.configs_actual_from", &self.view)
    }

    /// Returns the accessory `ProofMapIndex` for calculating
    /// patches in the DBView layer.
    ///
    /// Table calculates "aggregation" of root hashes of individual
    /// service tables, in effect summing the state of various entities,
    /// scattered across distinct services and their tables. Sum is performed by
    /// means of computing root hash of this table.
    ///
    /// - Table **key** is 32 bytes of normalized coordinates of a service
    /// table, as returned by `service_table_unique_key` helper function.
    /// - Table **value** is root hash of a service table, which contributes
    /// to the resulting block's `state_hash`.
    ///
    /// Core tables participate in resulting state_hash with `CORE_SERVICE`
    /// service_id. Their vector is returned by `core_state_hash` method.
    pub fn state_hash_aggregator(&self) -> ProofMapIndex<&T, Hash, Hash> {
        ProofMapIndex::new("core.state_hash_aggregator", &self.view)
    }

    /// Returns block hash for the given height.
    pub fn block_hash_by_height(&self, height: Height) -> Option<Hash> {
        self.block_hashes_by_height().get(height.into())
    }

    /// Returns the block for the given height with the proof of its inclusion.
    pub fn block_and_precommits(&self, height: Height) -> Option<BlockProof> {
        let block_hash = match self.block_hash_by_height(height) {
            None => return None,
            Some(block_hash) => block_hash,
        };
        let block = self.blocks().get(&block_hash).unwrap();
        let precommits_table = self.precommits(&block_hash);
        let precommits = precommits_table.iter().collect();
        let res = BlockProof { block, precommits };
        Some(res)
    }

    /// Returns latest committed block.
    pub fn last_block(&self) -> Block {
        let hash = self.block_hashes_by_height().last().expect(
            "Genesis block was not created.",
        );
        self.blocks().get(&hash).unwrap()
    }

    /// Returns height of the latest committed block.
    pub fn height(&self) -> Height {
        self.last_block().height()
    }

    /// Returns configuration for the latest height of blockchain.
    pub fn actual_configuration(&self) -> StoredConfiguration {
        let current_height = self.height().next();
        let res = self.configuration_by_height(current_height);
        trace!("Retrieved actual_config: {:?}", res);
        res
    }

    /// Returns the nearest following configuration if it exists.
    pub fn following_configuration(&self) -> Option<StoredConfiguration> {
        let current_height = self.height().next();
        let idx = self.find_configurations_index_by_height(current_height);
        match self.configs_actual_from().get(idx + 1) {
            Some(cfg_ref) => {
                let cfg_hash = cfg_ref.cfg_hash();
                let cfg = self.configuration_by_hash(cfg_hash).expect(&format!(
                    "Config with hash {:?} is absent in configs table",
                    cfg_hash
                ));
                Some(cfg)
            }
            None => None,
        }
    }

    /// Returns the previous configuration if it exists.
    pub fn previous_configuration(&self) -> Option<StoredConfiguration> {
        let current_height = self.height().next();
        let idx = self.find_configurations_index_by_height(current_height);
        if idx > 0 {
            let cfg_ref = self.configs_actual_from().get(idx - 1).expect(&format!(
                "Configuration at index {} not found",
                idx
            ));
            let cfg_hash = cfg_ref.cfg_hash();
            let cfg = self.configuration_by_hash(cfg_hash).expect(&format!(
                "Config with hash {:?} is absent in configs table",
                cfg_hash
            ));
            Some(cfg)
        } else {
            None
        }
    }

    /// Returns the configuration that is the actual for the given height.
    pub fn configuration_by_height(&self, height: Height) -> StoredConfiguration {
        let idx = self.find_configurations_index_by_height(height);
        let cfg_ref = self.configs_actual_from().get(idx).expect(&format!(
            "Configuration at index {} not found",
            idx
        ));
        let cfg_hash = cfg_ref.cfg_hash();
        self.configuration_by_hash(cfg_hash).expect(&format!(
            "Config with hash {:?} is absent in configs table",
            cfg_hash
        ))
    }

    /// Returns configuration for given configuration hash.
    pub fn configuration_by_hash(&self, hash: &Hash) -> Option<StoredConfiguration> {
        self.configs().get(hash)
    }

    /// Returns the `state_hash` table for core tables.
    pub fn core_state_hash(&self) -> Vec<Hash> {
        vec![self.configs().root_hash()]
    }

    /// Constructs a proof of inclusion of root hash of a specific service
    /// table into block's `state_hash`.
    ///
    /// Searched key for proof is uniquely identified by (`u16`, `u16`) tuple
    /// of table's coordinates.
    ///
    /// If found, root hash is returned as a value of proof's leaf
    /// corresponding to searched key. Otherwise, partial path to searched key
    /// is returned, which proves its exclusion.
    ///
    /// The returned proof is used as a component of proof of state of any
    /// entity, stored in `exonum` db at specific height, as identified
    /// by corresponding block's `state_hash`. State of some meta tables
    /// of core and services isn't tracked.
    ///
    /// # Arguments
    ///
    /// * `service_id` - `service_id` as returned by instance of type of
    /// `Service` trait
    /// * `table_idx` - index of service table in `Vec`, returned by
    /// `state_hash` method of instance of type of `Service` trait
    pub fn get_proof_to_service_table(&self, service_id: u16, table_idx: usize) -> MapProof<Hash> {
        let key = Blockchain::service_table_unique_key(service_id, table_idx);
        let sum_table = self.state_hash_aggregator();
        sum_table.get_proof(&key)
    }

    fn find_configurations_index_by_height(&self, height: Height) -> u64 {
        let actual_from = self.configs_actual_from();
        for i in (0..actual_from.len()).rev() {
            if actual_from.get(i).unwrap().actual_from() <= height {
                return i as u64;
            }
        }
        panic!(
            "Couldn't not find any config for height {}, \
                that means that genesis block was created incorrectly.",
            height
        )
    }
}

impl<'a> Schema<&'a mut Fork> {
    /// Mutable reference to the [`transactions`][1] index.
    ///
    /// [1]: struct.Schema.html#method.transactions
    pub fn transactions_mut(&mut self) -> MapIndex<&mut Fork, Hash, RawMessage> {
        MapIndex::new("core.transactions", &mut self.view)
    }

    /// Mutable reference to the [`tx_location_by_tx_hash`][1] index.
    ///
    /// [1]: struct.Schema.html#method.tx_location_by_tx_hash
    pub fn tx_location_by_tx_hash_mut(&mut self) -> MapIndex<&mut Fork, Hash, TxLocation> {
        MapIndex::new("core.tx_location_by_tx_hash", &mut self.view)
    }

    /// Mutable reference to the [`blocks][1] index.
    ///
    /// [1]: struct.Schema.html#method.blocks
    pub fn blocks_mut(&mut self) -> MapIndex<&mut Fork, Hash, Block> {
        MapIndex::new("core.blocks", &mut self.view)
    }

    /// Mutable reference to the [`block_hashes_by_height_mut`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_hashes_by_height_mut
    pub fn block_hashes_by_height_mut(&mut self) -> ListIndex<&mut Fork, Hash> {
        ListIndex::new("core.block_hashes_by_height", &mut self.view)
    }

    /// Mutable reference to the [`block_hash_by_height`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_hash_by_height
    pub fn block_hash_by_height_mut(&mut self, height: Height) -> Option<Hash> {
        self.block_hashes_by_height().get(height.into())
    }

    /// Mutable reference to the [`block_txs`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_txs
    pub fn block_txs_mut(&mut self, height: Height) -> ProofListIndex<&mut Fork, Hash> {
        let height: u64 = height.into();
        ProofListIndex::with_prefix("core.block_txs", gen_prefix(&height), &mut self.view)
    }

    /// Mutable reference to the [`precommits`][1] index.
    ///
    /// [1]: struct.Schema.html#method.precommits
    pub fn precommits_mut(&mut self, hash: &Hash) -> ListIndex<&mut Fork, Precommit> {
        ListIndex::with_prefix("core.precommits", gen_prefix(hash), &mut self.view)
    }

    /// Mutable reference to the [`configs`][1] index.
    ///
    /// [1]: struct.Schema.html#method.configs
    pub fn configs_mut(&mut self) -> ProofMapIndex<&mut Fork, Hash, StoredConfiguration> {
        ProofMapIndex::new("core.configs", &mut self.view)
    }

    /// Mutable reference to the [`configs_actual_from`][1] index.
    ///
    /// [1]: struct.Schema.html#method.configs_actual_from
    pub fn configs_actual_from_mut(&mut self) -> ListIndex<&mut Fork, ConfigReference> {
        ListIndex::new("core.configs_actual_from", &mut self.view)
    }

    /// Mutable reference to the [`state_hash_aggregator`][1] index.
    ///
    /// [1]: struct.Schema.html#method.state_hash_aggregator
    pub fn state_hash_aggregator_mut(&mut self) -> ProofMapIndex<&mut Fork, Hash, Hash> {
        ProofMapIndex::new("core.state_hash_aggregator", &mut self.view)
    }

    /// Adds a new configuration to the blockchain, which will become an actual at
    /// the `actual_from` height in `config_data`.
    pub fn commit_configuration(&mut self, config_data: StoredConfiguration) {
        let actual_from = config_data.actual_from;
        if let Some(last_cfg) = self.configs_actual_from().last() {
            if last_cfg.cfg_hash() != &config_data.previous_cfg_hash {
                // TODO: Replace panic with errors (ECR-123).
                panic!(
                    "Attempting to commit configuration with incorrect previous hash: {:?}, \
                    expected: {:?}",
                    config_data.previous_cfg_hash,
                    last_cfg.cfg_hash()
                );
            }

            if actual_from <= last_cfg.actual_from() {
                panic!(
                    "Attempting to commit configuration with actual_from {} less than the last \
                    committed the last committed actual_from {}",
                    actual_from,
                    last_cfg.actual_from()
                );
            }
        }

        info!(
            "Scheduled the following configuration for acceptance: {:?}",
            &config_data
        );

        let cfg_hash = config_data.hash();
        self.configs_mut().put(&cfg_hash, config_data);

        let cfg_ref = ConfigReference::new(actual_from, &cfg_hash);
        self.configs_actual_from_mut().push(cfg_ref);
        // TODO: clear storages
    }
}
<commit_msg>Fix `actual_configuration` behaviour.<commit_after>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crypto::Hash;
use messages::{Precommit, RawMessage};
use storage::{Fork, ListIndex, MapIndex, MapProof, ProofListIndex, ProofMapIndex, Snapshot,
              StorageKey, StorageValue};
use helpers::Height;
use super::{Block, BlockProof, Blockchain};
use super::config::StoredConfiguration;

/// Generates an array of bytes from the `prefix`.
pub fn gen_prefix<K: StorageKey>(prefix: &K) -> Vec<u8> {
    let mut res = vec![0; prefix.size()];
    prefix.write(&mut res[..]);
    res
}

encoding_struct! (
    /// Configuration index.
    struct ConfigReference {
        const SIZE = 40;
        /// The height, starting from which this configuration becomes actual.
        field actual_from: Height [00 => 08]
        /// Hash of the configuration contents that serialized as raw bytes vec.
        field cfg_hash:    &Hash  [08 => 40]
    }
);

encoding_struct! (
    /// Transaction location in block.
    struct TxLocation {
        const SIZE = 16;
        /// Height of block in the blockchain.
        field block_height:         Height  [00 => 08]
        /// Index in block.
        field position_in_block:    u64     [08 => 16]
    }
);

/// Information schema for `exonum-core`.
#[derive(Debug)]
pub struct Schema<T> {
    view: T,
}

impl<T> Schema<T>
where
    T: AsRef<Snapshot>,
{
    /// Constructs information schema for the given `snapshot`.
    pub fn new(snapshot: T) -> Schema<T> {
        Schema { view: snapshot }
    }

    /// Returns table that represents a map from transaction hash into raw transaction message.
    pub fn transactions(&self) -> MapIndex<&T, Hash, RawMessage> {
        MapIndex::new("core.transactions", &self.view)
    }

    /// Returns table that keeps the block height and tx position inside block for every
    /// transaction hash.
    pub fn tx_location_by_tx_hash(&self) -> MapIndex<&T, Hash, TxLocation> {
        MapIndex::new("core.tx_location_by_tx_hash", &self.view)
    }

    /// Returns table that stores block object for every block height.
    pub fn blocks(&self) -> MapIndex<&T, Hash, Block> {
        MapIndex::new("core.blocks", &self.view)
    }

    /// Returns table that keeps block hash for the corresponding height.
    pub fn block_hashes_by_height(&self) -> ListIndex<&T, Hash> {
        ListIndex::new("core.block_hashes_by_height", &self.view)
    }

    /// Returns table that keeps a list of transactions for the each block.
    pub fn block_txs(&self, height: Height) -> ProofListIndex<&T, Hash> {
        let height: u64 = height.into();
        ProofListIndex::with_prefix("core.block_txs", gen_prefix(&height), &self.view)
    }

    /// Returns table that saves a list of precommits for block with given hash.
    pub fn precommits(&self, hash: &Hash) -> ListIndex<&T, Precommit> {
        ListIndex::with_prefix("core.precommits", gen_prefix(hash), &self.view)
    }

    /// Returns table that represents a map from configuration hash into contents.
    pub fn configs(&self) -> ProofMapIndex<&T, Hash, StoredConfiguration> {
        // configs patricia merkletree <block height> json
        ProofMapIndex::new("core.configs", &self.view)
    }

    /// Returns auxiliary table that keeps hash references to configurations in order
    /// of increasing their `actual_from` height.
    pub fn configs_actual_from(&self) -> ListIndex<&T, ConfigReference> {
        ListIndex::new("core.configs_actual_from", &self.view)
    }

    /// Returns the accessory `ProofMapIndex` for calculating
    /// patches in the DBView layer.
    ///
    /// Table calculates "aggregation" of root hashes of individual
    /// service tables, in effect summing the state of various entities,
    /// scattered across distinct services and their tables. Sum is performed by
    /// means of computing root hash of this table.
    ///
    /// - Table **key** is 32 bytes of normalized coordinates of a service
    /// table, as returned by `service_table_unique_key` helper function.
    /// - Table **value** is root hash of a service table, which contributes
    /// to the resulting block's `state_hash`.
    ///
    /// Core tables participate in resulting state_hash with `CORE_SERVICE`
    /// service_id. Their vector is returned by `core_state_hash` method.
    pub fn state_hash_aggregator(&self) -> ProofMapIndex<&T, Hash, Hash> {
        ProofMapIndex::new("core.state_hash_aggregator", &self.view)
    }

    /// Returns block hash for the given height.
    pub fn block_hash_by_height(&self, height: Height) -> Option<Hash> {
        self.block_hashes_by_height().get(height.into())
    }

    /// Returns the block for the given height with the proof of its inclusion.
    pub fn block_and_precommits(&self, height: Height) -> Option<BlockProof> {
        let block_hash = match self.block_hash_by_height(height) {
            None => return None,
            Some(block_hash) => block_hash,
        };
        let block = self.blocks().get(&block_hash).unwrap();
        let precommits_table = self.precommits(&block_hash);
        let precommits = precommits_table.iter().collect();
        let res = BlockProof { block, precommits };
        Some(res)
    }

    /// Returns latest committed block.
    pub fn last_block(&self) -> Block {
        let hash = self.block_hashes_by_height().last().expect(
            "Genesis block was not created.",
        );
        self.blocks().get(&hash).unwrap()
    }

    /// Returns height of the latest committed block.
    pub fn height(&self) -> Height {
        self.last_block().height()
    }

    /// Returns configuration for the latest height of blockchain.
    pub fn actual_configuration(&self) -> StoredConfiguration {
        let current_height = self.current_height();
        let res = self.configuration_by_height(current_height);
        trace!("Retrieved actual_config: {:?}", res);
        res
    }

    /// Returns the nearest following configuration if it exists.
    pub fn following_configuration(&self) -> Option<StoredConfiguration> {
        let current_height = self.current_height();
        let idx = self.find_configurations_index_by_height(current_height);
        match self.configs_actual_from().get(idx + 1) {
            Some(cfg_ref) => {
                let cfg_hash = cfg_ref.cfg_hash();
                let cfg = self.configuration_by_hash(cfg_hash).expect(&format!(
                    "Config with hash {:?} is absent in configs table",
                    cfg_hash
                ));
                Some(cfg)
            }
            None => None,
        }
    }

    /// Returns the previous configuration if it exists.
    pub fn previous_configuration(&self) -> Option<StoredConfiguration> {
        let current_height = self.current_height();
        let idx = self.find_configurations_index_by_height(current_height);
        if idx > 0 {
            let cfg_ref = self.configs_actual_from().get(idx - 1).expect(&format!(
                "Configuration at index {} not found",
                idx
            ));
            let cfg_hash = cfg_ref.cfg_hash();
            let cfg = self.configuration_by_hash(cfg_hash).expect(&format!(
                "Config with hash {:?} is absent in configs table",
                cfg_hash
            ));
            Some(cfg)
        } else {
            None
        }
    }

    /// Returns the configuration that is the actual for the given height.
    pub fn configuration_by_height(&self, height: Height) -> StoredConfiguration {
        let idx = self.find_configurations_index_by_height(height);
        let cfg_ref = self.configs_actual_from().get(idx).expect(&format!(
            "Configuration at index {} not found",
            idx
        ));
        let cfg_hash = cfg_ref.cfg_hash();
        self.configuration_by_hash(cfg_hash).expect(&format!(
            "Config with hash {:?} is absent in configs table",
            cfg_hash
        ))
    }

    /// Returns configuration for given configuration hash.
    pub fn configuration_by_hash(&self, hash: &Hash) -> Option<StoredConfiguration> {
        self.configs().get(hash)
    }

    /// Returns the `state_hash` table for core tables.
    pub fn core_state_hash(&self) -> Vec<Hash> {
        vec![self.configs().root_hash()]
    }

    /// Constructs a proof of inclusion of root hash of a specific service
    /// table into block's `state_hash`.
    ///
    /// Searched key for proof is uniquely identified by (`u16`, `u16`) tuple
    /// of table's coordinates.
    ///
    /// If found, root hash is returned as a value of proof's leaf
    /// corresponding to searched key. Otherwise, partial path to searched key
    /// is returned, which proves its exclusion.
    ///
    /// The returned proof is used as a component of proof of state of any
    /// entity, stored in `exonum` db at specific height, as identified
    /// by corresponding block's `state_hash`. State of some meta tables
    /// of core and services isn't tracked.
    ///
    /// # Arguments
    ///
    /// * `service_id` - `service_id` as returned by instance of type of
    /// `Service` trait
    /// * `table_idx` - index of service table in `Vec`, returned by
    /// `state_hash` method of instance of type of `Service` trait
    pub fn get_proof_to_service_table(&self, service_id: u16, table_idx: usize) -> MapProof<Hash> {
        let key = Blockchain::service_table_unique_key(service_id, table_idx);
        let sum_table = self.state_hash_aggregator();
        sum_table.get_proof(&key)
    }

    fn find_configurations_index_by_height(&self, height: Height) -> u64 {
        let actual_from = self.configs_actual_from();
        for i in (0..actual_from.len()).rev() {
            if actual_from.get(i).unwrap().actual_from() <= height {
                return i as u64;
            }
        }
        panic!(
            "Couldn't not find any config for height {}, \
             that means that genesis block was created incorrectly.",
            height
        )
    }

    fn current_height(&self) -> Height {
        self.block_hashes_by_height()
            .last()
            .map(|hash| self.blocks().get(&hash).unwrap().height().next())
            .unwrap_or_else(Height::zero)
    }
}

impl<'a> Schema<&'a mut Fork> {
    /// Mutable reference to the [`transactions`][1] index.
    ///
    /// [1]: struct.Schema.html#method.transactions
    pub fn transactions_mut(&mut self) -> MapIndex<&mut Fork, Hash, RawMessage> {
        MapIndex::new("core.transactions", &mut self.view)
    }

    /// Mutable reference to the [`tx_location_by_tx_hash`][1] index.
    ///
    /// [1]: struct.Schema.html#method.tx_location_by_tx_hash
    pub fn tx_location_by_tx_hash_mut(&mut self) -> MapIndex<&mut Fork, Hash, TxLocation> {
        MapIndex::new("core.tx_location_by_tx_hash", &mut self.view)
    }

    /// Mutable reference to the [`blocks][1] index.
    ///
    /// [1]: struct.Schema.html#method.blocks
    pub fn blocks_mut(&mut self) -> MapIndex<&mut Fork, Hash, Block> {
        MapIndex::new("core.blocks", &mut self.view)
    }

    /// Mutable reference to the [`block_hashes_by_height_mut`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_hashes_by_height_mut
    pub fn block_hashes_by_height_mut(&mut self) -> ListIndex<&mut Fork, Hash> {
        ListIndex::new("core.block_hashes_by_height", &mut self.view)
    }

    /// Mutable reference to the [`block_hash_by_height`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_hash_by_height
    pub fn block_hash_by_height_mut(&mut self, height: Height) -> Option<Hash> {
        self.block_hashes_by_height().get(height.into())
    }

    /// Mutable reference to the [`block_txs`][1] index.
    ///
    /// [1]: struct.Schema.html#method.block_txs
    pub fn block_txs_mut(&mut self, height: Height) -> ProofListIndex<&mut Fork, Hash> {
        let height: u64 = height.into();
        ProofListIndex::with_prefix("core.block_txs", gen_prefix(&height), &mut self.view)
    }

    /// Mutable reference to the [`precommits`][1] index.
    ///
    /// [1]: struct.Schema.html#method.precommits
    pub fn precommits_mut(&mut self, hash: &Hash) -> ListIndex<&mut Fork, Precommit> {
        ListIndex::with_prefix("core.precommits", gen_prefix(hash), &mut self.view)
    }

    /// Mutable reference to the [`configs`][1] index.
    ///
    /// [1]: struct.Schema.html#method.configs
    pub fn configs_mut(&mut self) -> ProofMapIndex<&mut Fork, Hash, StoredConfiguration> {
        ProofMapIndex::new("core.configs", &mut self.view)
    }

    /// Mutable reference to the [`configs_actual_from`][1] index.
    ///
    /// [1]: struct.Schema.html#method.configs_actual_from
    pub fn configs_actual_from_mut(&mut self) -> ListIndex<&mut Fork, ConfigReference> {
        ListIndex::new("core.configs_actual_from", &mut self.view)
    }

    /// Mutable reference to the [`state_hash_aggregator`][1] index.
    ///
    /// [1]: struct.Schema.html#method.state_hash_aggregator
    pub fn state_hash_aggregator_mut(&mut self) -> ProofMapIndex<&mut Fork, Hash, Hash> {
        ProofMapIndex::new("core.state_hash_aggregator", &mut self.view)
    }

    /// Adds a new configuration to the blockchain, which will become an actual at
    /// the `actual_from` height in `config_data`.
    pub fn commit_configuration(&mut self, config_data: StoredConfiguration) {
        let actual_from = config_data.actual_from;
        if let Some(last_cfg) = self.configs_actual_from().last() {
            if last_cfg.cfg_hash() != &config_data.previous_cfg_hash {
                // TODO: Replace panic with errors (ECR-123).
                panic!(
                    "Attempting to commit configuration with incorrect previous hash: {:?}, \
                     expected: {:?}",
                    config_data.previous_cfg_hash,
                    last_cfg.cfg_hash()
                );
            }

            if actual_from <= last_cfg.actual_from() {
                panic!(
                    "Attempting to commit configuration with actual_from {} less than the last \
                     committed the last committed actual_from {}",
                    actual_from,
                    last_cfg.actual_from()
                );
            }
        }

        info!(
            "Scheduled the following configuration for acceptance: {:?}",
            &config_data
        );

        let cfg_hash = config_data.hash();
        self.configs_mut().put(&cfg_hash, config_data);

        let cfg_ref = ConfigReference::new(actual_from, &cfg_hash);
        self.configs_actual_from_mut().push(cfg_ref);
        // TODO: clear storages
    }
}
<|endoftext|>


<commit_before>extern crate grade_school as school;

fn stringvec_to_strvec(v: &Vec<String>) -> Vec<&str> {
    v.iter().map(|s| s.as_ref()).collect()
}

#[test]
fn test_grades_for_empty_school() {
    let s = school::School::new();
    assert_eq!(s.grades(), vec![]);
}

#[test]
#[ignore]
fn test_grades_for_one_student() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    assert_eq!(s.grades(), vec!(2));
}

#[test]
#[ignore]
fn test_grades_for_several_students_are_sorted() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    s.add(7, "Logan");
    s.add(4, "Blair");
    assert_eq!(s.grades(), vec!(2, 4, 7));
}

#[test]
#[ignore]
fn test_grades_when_several_students_have_the_same_grade() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    s.add(2, "Logan");
    s.add(2, "Blair");
    assert_eq!(s.grades(), vec!(2));
}

#[test]
#[ignore]
fn test_grade_for_empty_school() {
    let s = school::School::new();
    assert_eq!(s.grade(1), None);
}

#[test]
#[ignore]
fn test_grade_when_no_students_have_that_grade() {
    let mut s = school::School::new();
    s.add(7, "Logan");
    assert_eq!(s.grade(1), None);
}

#[test]
#[ignore]
fn test_grade_for_one_student() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    assert_eq!(s.grade(2).map(|v| stringvec_to_strvec(v)),
                Some(vec!("Aimee")))
}

#[test]
#[ignore]
fn test_grade_returns_students_sorted_by_name() {
    let mut s = school::School::new();
    s.add(2, "James");
    s.add(2, "Blair");
    s.add(2, "Paul");
    assert_eq!(s.grade(2).map(|v| stringvec_to_strvec(v)),
               Some(vec!("Blair", "James", "Paul")));
}

#[test]
#[ignore]
fn test_add_students_to_different_grades() {
    let mut s = school::School::new();
    s.add(3, "Chelsea");
    s.add(7, "Logan");
    assert_eq!(s.grades(), vec!(3, 7));
    assert_eq!(s.grade(3).map(|v| stringvec_to_strvec(v)),
               Some(vec!("Chelsea")));
    assert_eq!(s.grade(7).map(|v| stringvec_to_strvec(v)),
               Some(vec!("Logan")));
}
<commit_msg>Use [] instead of () for vec!<commit_after>extern crate grade_school as school;

fn stringvec_to_strvec(v: &Vec<String>) -> Vec<&str> {
    v.iter().map(|s| s.as_ref()).collect()
}

#[test]
fn test_grades_for_empty_school() {
    let s = school::School::new();
    assert_eq!(s.grades(), vec![]);
}

#[test]
#[ignore]
fn test_grades_for_one_student() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    assert_eq!(s.grades(), vec![2]);
}

#[test]
#[ignore]
fn test_grades_for_several_students_are_sorted() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    s.add(7, "Logan");
    s.add(4, "Blair");
    assert_eq!(s.grades(), vec![2, 4, 7]);
}

#[test]
#[ignore]
fn test_grades_when_several_students_have_the_same_grade() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    s.add(2, "Logan");
    s.add(2, "Blair");
    assert_eq!(s.grades(), vec![2]);
}

#[test]
#[ignore]
fn test_grade_for_empty_school() {
    let s = school::School::new();
    assert_eq!(s.grade(1), None);
}

#[test]
#[ignore]
fn test_grade_when_no_students_have_that_grade() {
    let mut s = school::School::new();
    s.add(7, "Logan");
    assert_eq!(s.grade(1), None);
}

#[test]
#[ignore]
fn test_grade_for_one_student() {
    let mut s = school::School::new();
    s.add(2, "Aimee");
    assert_eq!(s.grade(2).map(|v| stringvec_to_strvec(v)),
                Some(vec!["Aimee"]))
}

#[test]
#[ignore]
fn test_grade_returns_students_sorted_by_name() {
    let mut s = school::School::new();
    s.add(2, "James");
    s.add(2, "Blair");
    s.add(2, "Paul");
    assert_eq!(s.grade(2).map(|v| stringvec_to_strvec(v)),
               Some(vec!["Blair", "James", "Paul"]));
}

#[test]
#[ignore]
fn test_add_students_to_different_grades() {
    let mut s = school::School::new();
    s.add(3, "Chelsea");
    s.add(7, "Logan");
    assert_eq!(s.grades(), vec!(3, 7));
    assert_eq!(s.grade(3).map(|v| stringvec_to_strvec(v)),
               Some(vec!["Chelsea"]));
    assert_eq!(s.grade(7).map(|v| stringvec_to_strvec(v)),
               Some(vec!["Logan"]));
}
<|endoftext|>