<commit_before>#[allow(missing_doc)];

use std::str;
use std::io::Decorator;
use std::io::mem::{MemWriter, MemReader};
use std::mem;
use std::vec;

use super::types::Oid;

pub static PROTOCOL_VERSION: i32 = 0x0003_0000;
pub static CANCEL_CODE: i32 = 80877102;
pub static SSL_CODE: i32 = 80877103;

#[deriving(ToStr)]
pub enum BackendMessage {
    AuthenticationOk,
    AuthenticationKerberosV5,
    AuthenticationCleartextPassword,
    AuthenticationMD5Password {
        salt: ~[u8]
    },
    AuthenticationSCMCredential,
    AuthenticationGSS,
    AuthenticationSSPI,
    BackendKeyData {
        process_id: i32,
        secret_key: i32
    },
    BindComplete,
    CloseComplete,
    CommandComplete {
        tag: ~str
    },
    DataRow {
        row: ~[Option<~[u8]>]
    },
    EmptyQueryResponse,
    ErrorResponse {
        fields: ~[(u8, ~str)]
    },
    NoData,
    NoticeResponse {
        fields: ~[(u8, ~str)]
    },
    NotificationResponse {
        pid: i32,
        channel: ~str,
        payload: ~str
    },
    ParameterDescription {
        types: ~[Oid]
    },
    ParameterStatus {
        parameter: ~str,
        value: ~str
    },
    ParseComplete,
    PortalSuspended,
    ReadyForQuery {
        state: u8
    },
    RowDescription {
        descriptions: ~[RowDescriptionEntry]
    }
}

#[deriving(ToStr)]
pub struct RowDescriptionEntry {
    name: ~str,
    table_oid: Oid,
    column_id: i16,
    type_oid: Oid,
    type_size: i16,
    type_modifier: i32,
    format: i16
}

pub enum FrontendMessage<'a> {
    Bind {
        portal: &'a str,
        statement: &'a str,
        formats: &'a [i16],
        values: &'a [Option<~[u8]>],
        result_formats: &'a [i16]
    },
    CancelRequest {
        code: i32,
        process_id: i32,
        secret_key: i32,
    },
    Close {
        variant: u8,
        name: &'a str
    },
    Describe {
        variant: u8,
        name: &'a str
    },
    Execute {
        portal: &'a str,
        max_rows: i32
    },
    Parse {
        name: &'a str,
        query: &'a str,
        param_types: &'a [i32]
    },
    PasswordMessage {
        password: &'a str
    },
    Query {
        query: &'a str
    },
    StartupMessage {
        version: i32,
        parameters: &'a [(~str, ~str)]
    },
    SslRequest {
        code: i32
    },
    Sync,
    Terminate
}

trait WriteString {
    fn write_string(&mut self, s: &str);
}

impl<W: Writer> WriteString for W {
    fn write_string(&mut self, s: &str) {
        self.write(s.as_bytes());
        self.write_u8(0);
    }
}

pub trait WriteMessage {
    fn write_message(&mut self, &FrontendMessage);
}

impl<W: Writer> WriteMessage for W {
    fn write_message(&mut self, message: &FrontendMessage) {
        debug!("Writing message {:?}", message);
        let mut buf = MemWriter::new();
        let mut ident = None;

        match *message {
            Bind { portal, statement, formats, values, result_formats } => {
                ident = Some('B');
                buf.write_string(portal);
                buf.write_string(statement);

                buf.write_be_i16(formats.len() as i16);
                for format in formats.iter() {
                    buf.write_be_i16(*format);
                }

                buf.write_be_i16(values.len() as i16);
                for value in values.iter() {
                    match *value {
                        None => {
                            buf.write_be_i32(-1);
                        }
                        Some(ref value) => {
                            buf.write_be_i32(value.len() as i32);
                            buf.write(*value);
                        }
                    }
                }

                buf.write_be_i16(result_formats.len() as i16);
                for format in result_formats.iter() {
                    buf.write_be_i16(*format);
                }
            }
            CancelRequest { code, process_id, secret_key } => {
                buf.write_be_i32(code);
                buf.write_be_i32(process_id);
                buf.write_be_i32(secret_key);
            }
            Close { variant, name } => {
                ident = Some('C');
                buf.write_u8(variant);
                buf.write_string(name);
            }
            Describe { variant, name } => {
                ident = Some('D');
                buf.write_u8(variant);
                buf.write_string(name);
            }
            Execute { portal, max_rows } => {
                ident = Some('E');
                buf.write_string(portal);
                buf.write_be_i32(max_rows);
            }
            Parse { name, query, param_types } => {
                ident = Some('P');
                buf.write_string(name);
                buf.write_string(query);
                buf.write_be_i16(param_types.len() as i16);
                for ty in param_types.iter() {
                    buf.write_be_i32(*ty);
                }
            }
            PasswordMessage { password } => {
                ident = Some('p');
                buf.write_string(password);
            }
            Query { query } => {
                ident = Some('Q');
                buf.write_string(query);
            }
            StartupMessage { version, parameters } => {
                buf.write_be_i32(version);
                for &(ref k, ref v) in parameters.iter() {
                    buf.write_string(k.as_slice());
                    buf.write_string(v.as_slice());
                }
                buf.write_u8(0);
            }
            SslRequest { code } => buf.write_be_i32(code),
            Sync => {
                ident = Some('S');
            }
            Terminate => {
                ident = Some('X');
            }
        }

        match ident {
            Some(ident) => self.write_u8(ident as u8),
            None => ()
        }

        // add size of length value
        self.write_be_i32((buf.inner_ref().len() + mem::size_of::<i32>())
                           as i32);
        self.write(buf.inner());
    }
}

trait ReadString {
    fn read_string(&mut self) -> ~str;
}

impl<R: Buffer> ReadString for R {
    fn read_string(&mut self) -> ~str {
        let mut buf = self.read_until(0).unwrap();
        buf.pop();
        str::from_utf8_owned(buf)
    }
}

pub trait ReadMessage {
    fn read_message(&mut self) -> BackendMessage;
}

impl<R: Reader> ReadMessage for R {
    fn read_message(&mut self) -> BackendMessage {
        debug!("Reading message");

        let ident = self.read_u8();
        // subtract size of length value
        let len = self.read_be_i32() as uint - mem::size_of::<i32>();
        let mut buf = MemReader::new(self.read_bytes(len));

        let ret = match ident as char {
            '1' => ParseComplete,
            '2' => BindComplete,
            '3' => CloseComplete,
            'A' => NotificationResponse {
                pid: buf.read_be_i32(),
                channel: buf.read_string(),
                payload: buf.read_string()
            },
            'C' => CommandComplete { tag: buf.read_string() },
            'D' => read_data_row(&mut buf),
            'E' => ErrorResponse { fields: read_fields(&mut buf) },
            'I' => EmptyQueryResponse,
            'K' => BackendKeyData {
                process_id: buf.read_be_i32(),
                secret_key: buf.read_be_i32()
            },
            'n' => NoData,
            'N' => NoticeResponse { fields: read_fields(&mut buf) },
            'R' => read_auth_message(&mut buf),
            's' => PortalSuspended,
            'S' => ParameterStatus {
                parameter: buf.read_string(),
                value: buf.read_string()
            },
            't' => read_parameter_description(&mut buf),
            'T' => read_row_description(&mut buf),
            'Z' => ReadyForQuery { state: buf.read_u8() },
            ident => fail!("Unknown message identifier `{}`", ident)
        };
        assert!(buf.eof());
        debug!("Read message {:?}", ret);
        ret
    }
}

fn read_fields(buf: &mut MemReader) -> ~[(u8, ~str)] {
    let mut fields = ~[];
    loop {
        let ty = buf.read_u8();
        if ty == 0 {
            break;
        }

        fields.push((ty, buf.read_string()));
    }

    fields
}

fn read_data_row(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut values = vec::with_capacity(len);

    for _ in range(0, len) {
        let val = match buf.read_be_i32() {
            -1 => None,
            len => Some(buf.read_bytes(len as uint))
        };
        values.push(val);
    }

    DataRow { row: values }
}

fn read_auth_message(buf: &mut MemReader) -> BackendMessage {
    match buf.read_be_i32() {
        0 => AuthenticationOk,
        2 => AuthenticationKerberosV5,
        3 => AuthenticationCleartextPassword,
        5 => AuthenticationMD5Password { salt: buf.read_bytes(4) },
        6 => AuthenticationSCMCredential,
        7 => AuthenticationGSS,
        9 => AuthenticationSSPI,
        val => fail!("Invalid authentication identifier `{}`", val)
    }
}

fn read_parameter_description(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut types = vec::with_capacity(len);

    for _ in range(0, len) {
        types.push(buf.read_be_i32());
    }

    ParameterDescription { types: types }
}

fn read_row_description(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut types = vec::with_capacity(len);

    for _ in range(0, len) {
        types.push(RowDescriptionEntry {
            name: buf.read_string(),
            table_oid: buf.read_be_i32(),
            column_id: buf.read_be_i16(),
            type_oid: buf.read_be_i32(),
            type_size: buf.read_be_i16(),
            type_modifier: buf.read_be_i32(),
            format: buf.read_be_i16()
        });
    }

    RowDescription { descriptions: types }
}
<commit_msg>Remove unneeded lint attribute<commit_after>use std::str;
use std::io::Decorator;
use std::io::mem::{MemWriter, MemReader};
use std::mem;
use std::vec;

use super::types::Oid;

pub static PROTOCOL_VERSION: i32 = 0x0003_0000;
pub static CANCEL_CODE: i32 = 80877102;
pub static SSL_CODE: i32 = 80877103;

#[deriving(ToStr)]
pub enum BackendMessage {
    AuthenticationOk,
    AuthenticationKerberosV5,
    AuthenticationCleartextPassword,
    AuthenticationMD5Password {
        salt: ~[u8]
    },
    AuthenticationSCMCredential,
    AuthenticationGSS,
    AuthenticationSSPI,
    BackendKeyData {
        process_id: i32,
        secret_key: i32
    },
    BindComplete,
    CloseComplete,
    CommandComplete {
        tag: ~str
    },
    DataRow {
        row: ~[Option<~[u8]>]
    },
    EmptyQueryResponse,
    ErrorResponse {
        fields: ~[(u8, ~str)]
    },
    NoData,
    NoticeResponse {
        fields: ~[(u8, ~str)]
    },
    NotificationResponse {
        pid: i32,
        channel: ~str,
        payload: ~str
    },
    ParameterDescription {
        types: ~[Oid]
    },
    ParameterStatus {
        parameter: ~str,
        value: ~str
    },
    ParseComplete,
    PortalSuspended,
    ReadyForQuery {
        state: u8
    },
    RowDescription {
        descriptions: ~[RowDescriptionEntry]
    }
}

#[deriving(ToStr)]
pub struct RowDescriptionEntry {
    name: ~str,
    table_oid: Oid,
    column_id: i16,
    type_oid: Oid,
    type_size: i16,
    type_modifier: i32,
    format: i16
}

pub enum FrontendMessage<'a> {
    Bind {
        portal: &'a str,
        statement: &'a str,
        formats: &'a [i16],
        values: &'a [Option<~[u8]>],
        result_formats: &'a [i16]
    },
    CancelRequest {
        code: i32,
        process_id: i32,
        secret_key: i32,
    },
    Close {
        variant: u8,
        name: &'a str
    },
    Describe {
        variant: u8,
        name: &'a str
    },
    Execute {
        portal: &'a str,
        max_rows: i32
    },
    Parse {
        name: &'a str,
        query: &'a str,
        param_types: &'a [i32]
    },
    PasswordMessage {
        password: &'a str
    },
    Query {
        query: &'a str
    },
    StartupMessage {
        version: i32,
        parameters: &'a [(~str, ~str)]
    },
    SslRequest {
        code: i32
    },
    Sync,
    Terminate
}

trait WriteString {
    fn write_string(&mut self, s: &str);
}

impl<W: Writer> WriteString for W {
    fn write_string(&mut self, s: &str) {
        self.write(s.as_bytes());
        self.write_u8(0);
    }
}

pub trait WriteMessage {
    fn write_message(&mut self, &FrontendMessage);
}

impl<W: Writer> WriteMessage for W {
    fn write_message(&mut self, message: &FrontendMessage) {
        debug!("Writing message {:?}", message);
        let mut buf = MemWriter::new();
        let mut ident = None;

        match *message {
            Bind { portal, statement, formats, values, result_formats } => {
                ident = Some('B');
                buf.write_string(portal);
                buf.write_string(statement);

                buf.write_be_i16(formats.len() as i16);
                for format in formats.iter() {
                    buf.write_be_i16(*format);
                }

                buf.write_be_i16(values.len() as i16);
                for value in values.iter() {
                    match *value {
                        None => {
                            buf.write_be_i32(-1);
                        }
                        Some(ref value) => {
                            buf.write_be_i32(value.len() as i32);
                            buf.write(*value);
                        }
                    }
                }

                buf.write_be_i16(result_formats.len() as i16);
                for format in result_formats.iter() {
                    buf.write_be_i16(*format);
                }
            }
            CancelRequest { code, process_id, secret_key } => {
                buf.write_be_i32(code);
                buf.write_be_i32(process_id);
                buf.write_be_i32(secret_key);
            }
            Close { variant, name } => {
                ident = Some('C');
                buf.write_u8(variant);
                buf.write_string(name);
            }
            Describe { variant, name } => {
                ident = Some('D');
                buf.write_u8(variant);
                buf.write_string(name);
            }
            Execute { portal, max_rows } => {
                ident = Some('E');
                buf.write_string(portal);
                buf.write_be_i32(max_rows);
            }
            Parse { name, query, param_types } => {
                ident = Some('P');
                buf.write_string(name);
                buf.write_string(query);
                buf.write_be_i16(param_types.len() as i16);
                for ty in param_types.iter() {
                    buf.write_be_i32(*ty);
                }
            }
            PasswordMessage { password } => {
                ident = Some('p');
                buf.write_string(password);
            }
            Query { query } => {
                ident = Some('Q');
                buf.write_string(query);
            }
            StartupMessage { version, parameters } => {
                buf.write_be_i32(version);
                for &(ref k, ref v) in parameters.iter() {
                    buf.write_string(k.as_slice());
                    buf.write_string(v.as_slice());
                }
                buf.write_u8(0);
            }
            SslRequest { code } => buf.write_be_i32(code),
            Sync => {
                ident = Some('S');
            }
            Terminate => {
                ident = Some('X');
            }
        }

        match ident {
            Some(ident) => self.write_u8(ident as u8),
            None => ()
        }

        // add size of length value
        self.write_be_i32((buf.inner_ref().len() + mem::size_of::<i32>())
                           as i32);
        self.write(buf.inner());
    }
}

trait ReadString {
    fn read_string(&mut self) -> ~str;
}

impl<R: Buffer> ReadString for R {
    fn read_string(&mut self) -> ~str {
        let mut buf = self.read_until(0).unwrap();
        buf.pop();
        str::from_utf8_owned(buf)
    }
}

pub trait ReadMessage {
    fn read_message(&mut self) -> BackendMessage;
}

impl<R: Reader> ReadMessage for R {
    fn read_message(&mut self) -> BackendMessage {
        debug!("Reading message");

        let ident = self.read_u8();
        // subtract size of length value
        let len = self.read_be_i32() as uint - mem::size_of::<i32>();
        let mut buf = MemReader::new(self.read_bytes(len));

        let ret = match ident as char {
            '1' => ParseComplete,
            '2' => BindComplete,
            '3' => CloseComplete,
            'A' => NotificationResponse {
                pid: buf.read_be_i32(),
                channel: buf.read_string(),
                payload: buf.read_string()
            },
            'C' => CommandComplete { tag: buf.read_string() },
            'D' => read_data_row(&mut buf),
            'E' => ErrorResponse { fields: read_fields(&mut buf) },
            'I' => EmptyQueryResponse,
            'K' => BackendKeyData {
                process_id: buf.read_be_i32(),
                secret_key: buf.read_be_i32()
            },
            'n' => NoData,
            'N' => NoticeResponse { fields: read_fields(&mut buf) },
            'R' => read_auth_message(&mut buf),
            's' => PortalSuspended,
            'S' => ParameterStatus {
                parameter: buf.read_string(),
                value: buf.read_string()
            },
            't' => read_parameter_description(&mut buf),
            'T' => read_row_description(&mut buf),
            'Z' => ReadyForQuery { state: buf.read_u8() },
            ident => fail!("Unknown message identifier `{}`", ident)
        };
        assert!(buf.eof());
        debug!("Read message {:?}", ret);
        ret
    }
}

fn read_fields(buf: &mut MemReader) -> ~[(u8, ~str)] {
    let mut fields = ~[];
    loop {
        let ty = buf.read_u8();
        if ty == 0 {
            break;
        }

        fields.push((ty, buf.read_string()));
    }

    fields
}

fn read_data_row(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut values = vec::with_capacity(len);

    for _ in range(0, len) {
        let val = match buf.read_be_i32() {
            -1 => None,
            len => Some(buf.read_bytes(len as uint))
        };
        values.push(val);
    }

    DataRow { row: values }
}

fn read_auth_message(buf: &mut MemReader) -> BackendMessage {
    match buf.read_be_i32() {
        0 => AuthenticationOk,
        2 => AuthenticationKerberosV5,
        3 => AuthenticationCleartextPassword,
        5 => AuthenticationMD5Password { salt: buf.read_bytes(4) },
        6 => AuthenticationSCMCredential,
        7 => AuthenticationGSS,
        9 => AuthenticationSSPI,
        val => fail!("Invalid authentication identifier `{}`", val)
    }
}

fn read_parameter_description(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut types = vec::with_capacity(len);

    for _ in range(0, len) {
        types.push(buf.read_be_i32());
    }

    ParameterDescription { types: types }
}

fn read_row_description(buf: &mut MemReader) -> BackendMessage {
    let len = buf.read_be_i16() as uint;
    let mut types = vec::with_capacity(len);

    for _ in range(0, len) {
        types.push(RowDescriptionEntry {
            name: buf.read_string(),
            table_oid: buf.read_be_i32(),
            column_id: buf.read_be_i16(),
            type_oid: buf.read_be_i32(),
            type_size: buf.read_be_i16(),
            type_modifier: buf.read_be_i32(),
            format: buf.read_be_i16()
        });
    }

    RowDescription { descriptions: types }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Defines aggregations over Arrow arrays.

use std::ops::Add;

use crate::array::{Array, GenericStringArray, PrimitiveArray, StringOffsetSizeTrait};
use crate::datatypes::ArrowNumericType;

/// Helper macro to perform min/max of strings
fn min_max_string<T: StringOffsetSizeTrait, F: Fn(&str, &str) -> bool>(
    array: &GenericStringArray<T>,
    cmp: F,
) -> Option<&str> {
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }
    let mut n = "";
    let mut has_value = false;
    let data = array.data();

    if null_count == 0 {
        for i in 0..data.len() {
            let item = array.value(i);
            if !has_value || cmp(&n, item) {
                has_value = true;
                n = item;
            }
        }
    } else {
        for i in 0..data.len() {
            let item = array.value(i);
            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
                has_value = true;
                n = item;
            }
        }
    }
    Some(n)
}

/// Returns the minimum value in the array, according to the natural order.
pub fn min<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
{
    min_max_helper(array, |a, b| a > b)
}

/// Returns the maximum value in the array, according to the natural order.
pub fn max<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
{
    min_max_helper(array, |a, b| a < b)
}

/// Returns the maximum value in the string array, according to the natural order.
pub fn max_string<T: StringOffsetSizeTrait>(
    array: &GenericStringArray<T>,
) -> Option<&str> {
    min_max_string(array, |a, b| a < b)
}

/// Returns the minimum value in the string array, according to the natural order.
pub fn min_string<T: StringOffsetSizeTrait>(
    array: &GenericStringArray<T>,
) -> Option<&str> {
    min_max_string(array, |a, b| a > b)
}

/// Helper function to perform min/max lambda function on values from a numeric array.
fn min_max_helper<T, F>(array: &PrimitiveArray<T>, cmp: F) -> Option<T::Native>
where
    T: ArrowNumericType,
    F: Fn(&T::Native, &T::Native) -> bool,
{
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }

    let mut n: T::Native = T::default_value();
    let mut has_value = false;
    let data = array.data();
    let m = array.value_slice(0, data.len());

    if null_count == 0 {
        // optimized path for arrays without null values
        for item in m {
            if !has_value || cmp(&n, item) {
                has_value = true;
                n = *item
            }
        }
    } else {
        for (i, item) in m.iter().enumerate() {
            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
                has_value = true;
                n = *item
            }
        }
    }
    Some(n)
}

/// Returns the sum of values in the array.
///
/// Returns `None` if the array is empty or only contains null values.
#[cfg(not(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd")))]
pub fn sum<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
    T::Native: Add<Output = T::Native>,
{
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }

    let data: &[T::Native] = array.value_slice(0, array.len());

    match array.data().null_buffer() {
        None => {
            let sum = data.iter().fold(T::default_value(), |accumulator, value| {
                accumulator + *value
            });

            Some(sum)
        }
        Some(buffer) => {
            let mut sum = T::default_value();
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
            &data_chunks
                .zip(bit_chunks.iter())
                .for_each(|(chunk, mask)| {
                    chunk.iter().enumerate().for_each(|(i, value)| {
                        if (mask & (1 << i)) != 0 {
                            sum = sum + *value;
                        }
                    });
                });

            let remainder_bits = bit_chunks.remainder_bits();

            remainder.iter().enumerate().for_each(|(i, value)| {
                if remainder_bits & (1 << i) != 0 {
                    sum = sum + *value;
                }
            });

            Some(sum)
        }
    }
}

/// Returns the sum of values in the array.
///
/// Returns `None` if the array is empty or only contains null values.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
pub fn sum<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T::Native: Add<Output = T::Native>,
{
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }

    let data: &[T::Native] = array.value_slice(0, array.len());

    let mut vector_sum = T::init(T::default_value());
    let mut remainder_sum = T::default_value();

    match array.data().null_buffer() {
        None => {
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            data_chunks.for_each(|chunk| {
                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
                    let chunk = T::load(&chunk);
                    vector_sum = vector_sum + chunk;
                });
            });

            remainder.iter().for_each(|value| {
                remainder_sum = remainder_sum + *value;
            });
        }
        Some(buffer) => {
            // process data in chunks of 64 elements since we also get 64 bits of validity information at a time
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
            let remainder_bits = bit_chunks.remainder_bits();

            data_chunks.zip(bit_chunks).for_each(|(chunk, mut mask)| {
                // split chunks further into slices corresponding to the vector length
                // the compiler is able to unroll this inner loop and remove bounds checks
                // since the outer chunk size (64) is always a multiple of the number of lanes
                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
                    let zero = T::init(T::default_value());
                    let vecmask = T::mask_from_u64(mask);
                    let chunk = T::load(&chunk);
                    let blended = T::mask_select(vecmask, chunk, zero);

                    vector_sum = vector_sum + blended;

                    mask = mask >> T::lanes();
                });
            });

            remainder.iter().enumerate().for_each(|(i, value)| {
                if remainder_bits & (1 << i) != 0 {
                    remainder_sum = remainder_sum + *value;
                }
            });
        }
    }

    // calculate horizontal sum of accumulator by writing to a temporary
    // this is probably faster than extracting individual lanes
    // the compiler is free to optimize this to something faster
    let tmp = &mut [T::default_value(); 64];
    T::write(vector_sum, &mut tmp[0..T::lanes()]);

    let mut total_sum = T::default_value();
    tmp[0..T::lanes()]
        .iter()
        .for_each(|lane| total_sum = total_sum + *lane);

    total_sum = total_sum + remainder_sum;

    Some(total_sum)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::array::*;

    #[test]
    fn test_primitive_array_sum() {
        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
        assert_eq!(15, sum(&a).unwrap());
    }

    #[test]
    fn test_primitive_array_float_sum() {
        let a = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]);
        assert!(16.5 - sum(&a).unwrap() < f64::EPSILON);
    }

    #[test]
    fn test_primitive_array_sum_with_nulls() {
        let a = Int32Array::from(vec![None, Some(2), Some(3), None, Some(5)]);
        assert_eq!(10, sum(&a).unwrap());
    }

    #[test]
    fn test_primitive_array_sum_all_nulls() {
        let a = Int32Array::from(vec![None, None, None]);
        assert_eq!(None, sum(&a));
    }

    #[test]
    fn test_buffer_array_min_max() {
        let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
        assert_eq!(5, min(&a).unwrap());
        assert_eq!(9, max(&a).unwrap());
    }

    #[test]
    fn test_buffer_array_min_max_with_nulls() {
        let a = Int32Array::from(vec![Some(5), None, None, Some(8), Some(9)]);
        assert_eq!(5, min(&a).unwrap());
        assert_eq!(9, max(&a).unwrap());
    }

    #[test]
    fn test_buffer_min_max_1() {
        let a = Int32Array::from(vec![None, None, Some(5), Some(2)]);
        assert_eq!(Some(2), min(&a));
        assert_eq!(Some(5), max(&a));
    }

    #[test]
    fn test_string_min_max_with_nulls() {
        let a = StringArray::from(vec![Some("b"), None, None, Some("a"), Some("c")]);
        assert_eq!("a", min_string(&a).unwrap());
        assert_eq!("c", max_string(&a).unwrap());
    }

    #[test]
    fn test_string_min_max_all_nulls() {
        let a = StringArray::from(vec![None, None]);
        assert_eq!(None, min_string(&a));
        assert_eq!(None, max_string(&a));
    }

    #[test]
    fn test_string_min_max_1() {
        let a = StringArray::from(vec![None, None, Some("b"), Some("a")]);
        assert_eq!(Some("a"), min_string(&a));
        assert_eq!(Some("b"), max_string(&a));
    }
}
<commit_msg>ARROW-10595: [Rust] Simplify inner loop of min/max kernels for non-null case<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Defines aggregations over Arrow arrays.

use std::ops::Add;

use crate::array::{Array, GenericStringArray, PrimitiveArray, StringOffsetSizeTrait};
use crate::datatypes::ArrowNumericType;

/// Helper macro to perform min/max of strings
fn min_max_string<T: StringOffsetSizeTrait, F: Fn(&str, &str) -> bool>(
    array: &GenericStringArray<T>,
    cmp: F,
) -> Option<&str> {
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }
    let mut n = "";
    let mut has_value = false;
    let data = array.data();

    if null_count == 0 {
        for i in 0..data.len() {
            let item = array.value(i);
            if !has_value || cmp(&n, item) {
                has_value = true;
                n = item;
            }
        }
    } else {
        for i in 0..data.len() {
            let item = array.value(i);
            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
                has_value = true;
                n = item;
            }
        }
    }
    Some(n)
}

/// Returns the minimum value in the array, according to the natural order.
pub fn min<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
{
    min_max_helper(array, |a, b| a > b)
}

/// Returns the maximum value in the array, according to the natural order.
pub fn max<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
{
    min_max_helper(array, |a, b| a < b)
}

/// Returns the maximum value in the string array, according to the natural order.
pub fn max_string<T: StringOffsetSizeTrait>(
    array: &GenericStringArray<T>,
) -> Option<&str> {
    min_max_string(array, |a, b| a < b)
}

/// Returns the minimum value in the string array, according to the natural order.
pub fn min_string<T: StringOffsetSizeTrait>(
    array: &GenericStringArray<T>,
) -> Option<&str> {
    min_max_string(array, |a, b| a > b)
}

/// Helper function to perform min/max lambda function on values from a numeric array.
fn min_max_helper<T, F>(array: &PrimitiveArray<T>, cmp: F) -> Option<T::Native>
where
    T: ArrowNumericType,
    F: Fn(&T::Native, &T::Native) -> bool,
{
    let null_count = array.null_count();

    // Includes case array.len() == 0
    if null_count == array.len() {
        return None;
    }

    let data = array.data();
    let m = array.value_slice(0, data.len());
    let mut n;

    if null_count == 0 {
        // optimized path for arrays without null values
        n = m[0];

        for item in &m[1..] {
            if cmp(&n, item) {
                n = *item
            }
        }
    } else {
        n = T::default_value();
        let mut has_value = false;
        for (i, item) in m.iter().enumerate() {
            if data.is_valid(i) && (!has_value || cmp(&n, item)) {
                has_value = true;
                n = *item
            }
        }
    }
    Some(n)
}

/// Returns the sum of values in the array.
///
/// Returns `None` if the array is empty or only contains null values.
#[cfg(not(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd")))]
pub fn sum<T>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T: ArrowNumericType,
    T::Native: Add<Output = T::Native>,
{
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }

    let data: &[T::Native] = array.value_slice(0, array.len());

    match array.data().null_buffer() {
        None => {
            let sum = data.iter().fold(T::default_value(), |accumulator, value| {
                accumulator + *value
            });

            Some(sum)
        }
        Some(buffer) => {
            let mut sum = T::default_value();
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
            &data_chunks
                .zip(bit_chunks.iter())
                .for_each(|(chunk, mask)| {
                    chunk.iter().enumerate().for_each(|(i, value)| {
                        if (mask & (1 << i)) != 0 {
                            sum = sum + *value;
                        }
                    });
                });

            let remainder_bits = bit_chunks.remainder_bits();

            remainder.iter().enumerate().for_each(|(i, value)| {
                if remainder_bits & (1 << i) != 0 {
                    sum = sum + *value;
                }
            });

            Some(sum)
        }
    }
}

/// Returns the sum of values in the array.
///
/// Returns `None` if the array is empty or only contains null values.
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "simd"))]
pub fn sum<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
where
    T::Native: Add<Output = T::Native>,
{
    let null_count = array.null_count();

    if null_count == array.len() {
        return None;
    }

    let data: &[T::Native] = array.value_slice(0, array.len());

    let mut vector_sum = T::init(T::default_value());
    let mut remainder_sum = T::default_value();

    match array.data().null_buffer() {
        None => {
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            data_chunks.for_each(|chunk| {
                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
                    let chunk = T::load(&chunk);
                    vector_sum = vector_sum + chunk;
                });
            });

            remainder.iter().for_each(|value| {
                remainder_sum = remainder_sum + *value;
            });
        }
        Some(buffer) => {
            // process data in chunks of 64 elements since we also get 64 bits of validity information at a time
            let data_chunks = data.chunks_exact(64);
            let remainder = data_chunks.remainder();

            let bit_chunks = buffer.bit_chunks(array.offset(), array.len());
            let remainder_bits = bit_chunks.remainder_bits();

            data_chunks.zip(bit_chunks).for_each(|(chunk, mut mask)| {
                // split chunks further into slices corresponding to the vector length
                // the compiler is able to unroll this inner loop and remove bounds checks
                // since the outer chunk size (64) is always a multiple of the number of lanes
                chunk.chunks_exact(T::lanes()).for_each(|chunk| {
                    let zero = T::init(T::default_value());
                    let vecmask = T::mask_from_u64(mask);
                    let chunk = T::load(&chunk);
                    let blended = T::mask_select(vecmask, chunk, zero);

                    vector_sum = vector_sum + blended;

                    mask = mask >> T::lanes();
                });
            });

            remainder.iter().enumerate().for_each(|(i, value)| {
                if remainder_bits & (1 << i) != 0 {
                    remainder_sum = remainder_sum + *value;
                }
            });
        }
    }

    // calculate horizontal sum of accumulator by writing to a temporary
    // this is probably faster than extracting individual lanes
    // the compiler is free to optimize this to something faster
    let tmp = &mut [T::default_value(); 64];
    T::write(vector_sum, &mut tmp[0..T::lanes()]);

    let mut total_sum = T::default_value();
    tmp[0..T::lanes()]
        .iter()
        .for_each(|lane| total_sum = total_sum + *lane);

    total_sum = total_sum + remainder_sum;

    Some(total_sum)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::array::*;

    #[test]
    fn test_primitive_array_sum() {
        let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
        assert_eq!(15, sum(&a).unwrap());
    }

    #[test]
    fn test_primitive_array_float_sum() {
        let a = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]);
        assert!(16.5 - sum(&a).unwrap() < f64::EPSILON);
    }

    #[test]
    fn test_primitive_array_sum_with_nulls() {
        let a = Int32Array::from(vec![None, Some(2), Some(3), None, Some(5)]);
        assert_eq!(10, sum(&a).unwrap());
    }

    #[test]
    fn test_primitive_array_sum_all_nulls() {
        let a = Int32Array::from(vec![None, None, None]);
        assert_eq!(None, sum(&a));
    }

    #[test]
    fn test_buffer_array_min_max() {
        let a = Int32Array::from(vec![5, 6, 7, 8, 9]);
        assert_eq!(5, min(&a).unwrap());
        assert_eq!(9, max(&a).unwrap());
    }

    #[test]
    fn test_buffer_array_min_max_with_nulls() {
        let a = Int32Array::from(vec![Some(5), None, None, Some(8), Some(9)]);
        assert_eq!(5, min(&a).unwrap());
        assert_eq!(9, max(&a).unwrap());
    }

    #[test]
    fn test_buffer_min_max_1() {
        let a = Int32Array::from(vec![None, None, Some(5), Some(2)]);
        assert_eq!(Some(2), min(&a));
        assert_eq!(Some(5), max(&a));
    }

    #[test]
    fn test_string_min_max_with_nulls() {
        let a = StringArray::from(vec![Some("b"), None, None, Some("a"), Some("c")]);
        assert_eq!("a", min_string(&a).unwrap());
        assert_eq!("c", max_string(&a).unwrap());
    }

    #[test]
    fn test_string_min_max_all_nulls() {
        let a = StringArray::from(vec![None, None]);
        assert_eq!(None, min_string(&a));
        assert_eq!(None, max_string(&a));
    }

    #[test]
    fn test_string_min_max_1() {
        let a = StringArray::from(vec![None, None, Some("b"), Some("a")]);
        assert_eq!(Some("a"), min_string(&a));
        assert_eq!(Some("b"), max_string(&a));
    }
}
<|endoftext|>


<commit_before><commit_msg>ui: Remove unused UI instance for unit<commit_after><|endoftext|>


<commit_before>// Exercises a bug in the shape code that was exposed
// on x86_64: when there is a tag embedded in an
// interior record which is then itself interior to
// something else, shape calculations were off.
use std;
import std::list;
import std::list::list;
import std::option;

tag opt_span {

    //hack (as opposed to option::t), to make `span` compile
    os_none;
    os_some(@span);
}
type span = {lo: uint, hi: uint, expanded_from: opt_span};
type spanned<T> = { data: T, span: span };
type ty_ = uint;
type path_ = { global: bool, idents: [str], types: [@ty] };
type path = spanned<path_>;
type ty = spanned<ty_>;

fn main() {
    let sp: span = {lo: 57451u, hi: 57542u, expanded_from: os_none};
    let t: @ty = @{ data: 3u, span: sp };
    let p_: path_ = { global: true, idents: ["hi"], types: [t] };
    let p: path = { data: p_, span: sp };
    let x = { sp: sp, path: p };
    log_err x.path;
    log_err x;
}
<commit_msg>extra whitespace<commit_after>// Exercises a bug in the shape code that was exposed
// on x86_64: when there is a tag embedded in an
// interior record which is then itself interior to
// something else, shape calculations were off.
use std;
import std::list;
import std::list::list;
import std::option;

tag opt_span {

    //hack (as opposed to option::t), to make `span` compile
    os_none;
    os_some(@span);
}
type span = {lo: uint, hi: uint, expanded_from: opt_span};
type spanned<T> = { data: T, span: span };
type ty_ = uint;
type path_ = { global: bool, idents: [str], types: [@ty] };
type path = spanned<path_>;
type ty = spanned<ty_>;

fn main() {
    let sp: span = {lo: 57451u, hi: 57542u, expanded_from: os_none};
    let t: @ty = @{ data: 3u, span: sp };
    let p_: path_ = { global: true, idents: ["hi"], types: [t] };
    let p: path = { data: p_, span: sp };
    let x = { sp: sp, path: p };
    log_err x.path;
    log_err x;
}
<|endoftext|>


<commit_before>// run-pass

fn main() {
    match (return) {} // ok
    if (return) {} // ok
}
<commit_msg>Make warnings into errors<commit_after>// run-pass

#![deny(unused_parens)]
#![allow(unreachable_code)]

fn main() {
    match (return) {} // ok
    if (return) {} // ok
}
<|endoftext|>


<commit_before>use std::sync::Arc;
use super::attestation_record::{
    validate_attestation,
    AttestationValidationError,
};
use super::attestation_record::{
    AttestationRecord,
    split_one_attestation,
    split_all_attestations,
    AttestationSplitError,
};
use super::{
    AttesterMap,
    ProposerMap,
};
use super::SszBlock;
use super::db::{
    ClientDB,
    DBError,
};
use super::db::stores::{
    BlockStore,
    PoWChainStore,
    ValidatorStore,
};
use super::ssz::{
    Decodable,
    DecodeError,
};
use super::utils::types::Hash256;

#[derive(Debug, PartialEq)]
pub enum BlockStatus {
    NewBlock,
    KnownBlock,
}

#[derive(Debug, PartialEq)]
pub enum SszBlockValidationError {
    FutureSlot,
    UnknownPoWChainRef,
    BadAttestationSsz,
    AttestationValidationError(AttestationValidationError),
    AttestationSignatureFailed,
    FirstAttestationSignatureFailed,
    NoProposerSignature,
    BadProposerMap,
    DatabaseError(String),
}

/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure
/// that reads directly from encoded SSZ.
///
/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is
/// a suspicion that the block might be invalid. Such a suspicion should be applied to
/// all blocks coming from the network.
///
/// Of course, this function will only be more efficient if a block is already serialized.
/// Serializing a complete block and then validating with this function will be less
/// efficient than just validating the original block.
///
/// This function will determine if the block is new, already known or invalid (either
/// intrinsically or due to some application error.)
#[allow(dead_code)]
pub fn validate_ssz_block<T>(b: &SszBlock,
                             expected_slot: u64,
                             cycle_length: u8,
                             last_justified_slot: u64,
                             parent_hashes: &Arc<Vec<Hash256>>,
                             proposer_map: &Arc<ProposerMap>,
                             attester_map: &Arc<AttesterMap>,
                             block_store: &Arc<BlockStore<T>>,
                             validator_store: &Arc<ValidatorStore<T>>,
                             pow_store: &Arc<PoWChainStore<T>>)
    -> Result<BlockStatus, SszBlockValidationError>
    where T: ClientDB + Sized
{
    /*
     * If this block is already known, return immediately.
     */
    if block_store.block_exists(&b.block_hash())? {
        return Ok(BlockStatus::KnownBlock);
    }

    /*
     * Copy the block slot (will be used multiple times)
     */
    let block_slot = b.slot_number();

    /*
     * If the block slot corresponds to a slot in the future (according to the local time),
     * drop it.
     */
    if block_slot > expected_slot {
        return Err(SszBlockValidationError::FutureSlot);
    }

    /*
     * If the PoW chain hash is not known to us, drop it.
     *
     * We only accept blocks that reference a known PoW hash.
     *
     * Note: it is not clear what a "known" PoW chain ref is. Likely,
     * it means "sufficienty deep in the canonical PoW chain".
     */
    if !pow_store.block_hash_exists(b.pow_chain_ref())? {
        return Err(SszBlockValidationError::UnknownPoWChainRef);
    }

    /*
     * Store a reference to the serialized attestations from the block.
     */
    let attestations_ssz = &b.attestations();

    /*
     * Get a slice of the first serialized attestation (the 0th) and decode it into
     * a full AttestationRecord object.
     */
    let (first_attestation_ssz, next_index) = split_one_attestation(
        &attestations_ssz,
        0)?;
    let (first_attestation, _) = AttestationRecord::ssz_decode(
        &first_attestation_ssz, 0)?;

    /*
     * Validate this first attestation.
     *
     * It is a requirement that the block proposer for this slot
     * must have signed the 0th attestation record.
     */
    let attestation_voters = validate_attestation(
        &first_attestation,
        block_slot,
        cycle_length,
        last_justified_slot,
        &parent_hashes,
        &block_store,
        &validator_store,
        &attester_map)?;

    /*
     * If the set of voters is None, the attestation was invalid.
     */
    let attestation_voters = attestation_voters
        .ok_or(SszBlockValidationError::
               FirstAttestationSignatureFailed)?;

    /*
     * Read the proposer from the map of slot -> validator index.
     */
    let proposer = proposer_map.get(&block_slot)
        .ok_or(SszBlockValidationError::BadProposerMap)?;

    /*
     * If the proposer for this slot was not a voter, reject the block.
     */
    if !attestation_voters.contains(&proposer) {
        return Err(SszBlockValidationError::NoProposerSignature);
    }

    /*
     * Split the remaining attestations into a vector of slices, each containing
     * a single serialized attestation record.
     */
    let other_attestations = split_all_attestations(attestations_ssz,
                                                    next_index)?;

    /*
     * Verify each other AttestationRecord.
     *
     * TODO: make this parallelized.
     */
    for attestation in other_attestations {
        let (a, _) = AttestationRecord::ssz_decode(&attestation, 0)?;
        let attestation_voters = validate_attestation(
            &a,
            block_slot,
            cycle_length,
            last_justified_slot,
            &parent_hashes,
            &block_store,
            &validator_store,
            &attester_map)?;
        if attestation_voters.is_none() {
            return Err(SszBlockValidationError::
                       AttestationSignatureFailed);
        }
    }

    /*
     * If we have reached this point, the block is a new valid block that is worthy of
     * processing.
     */
    Ok(BlockStatus::NewBlock)
}

impl From<DBError> for SszBlockValidationError {
    fn from(e: DBError) -> Self {
        SszBlockValidationError::DatabaseError(e.message)
    }
}

impl From<AttestationSplitError> for SszBlockValidationError {
    fn from(e: AttestationSplitError) -> Self {
        match e {
            AttestationSplitError::TooShort =>
                SszBlockValidationError::BadAttestationSsz
        }
    }
}

impl From<DecodeError> for SszBlockValidationError {
    fn from(e: DecodeError) -> Self {
        match e {
            DecodeError::TooShort =>
                SszBlockValidationError::BadAttestationSsz,
            DecodeError::TooLong =>
                SszBlockValidationError::BadAttestationSsz,
        }
    }
}

impl From<AttestationValidationError> for SszBlockValidationError {
    fn from(e: AttestationValidationError) -> Self {
        SszBlockValidationError::AttestationValidationError(e)
    }
}
<commit_msg>Move attestation validation into iter.for_each<commit_after>use std::sync::{
    Arc,
    RwLock,
};
use super::attestation_record::{
    validate_attestation,
    AttestationValidationError,
};
use super::attestation_record::{
    AttestationRecord,
    split_one_attestation,
    split_all_attestations,
    AttestationSplitError,
};
use super::{
    AttesterMap,
    ProposerMap,
};
use super::SszBlock;
use super::db::{
    ClientDB,
    DBError,
};
use super::db::stores::{
    BlockStore,
    PoWChainStore,
    ValidatorStore,
};
use super::ssz::{
    Decodable,
    DecodeError,
};
use super::utils::types::Hash256;

#[derive(Debug, PartialEq)]
pub enum BlockStatus {
    NewBlock,
    KnownBlock,
}

#[derive(Debug, PartialEq)]
pub enum SszBlockValidationError {
    FutureSlot,
    UnknownPoWChainRef,
    BadAttestationSsz,
    AttestationValidationError(AttestationValidationError),
    AttestationSignatureFailed,
    FirstAttestationSignatureFailed,
    NoProposerSignature,
    BadProposerMap,
    DatabaseError(String),
}

/// Validate some SszBlock. An SszBlock varies from a Block in that is a read-only structure
/// that reads directly from encoded SSZ.
///
/// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is
/// a suspicion that the block might be invalid. Such a suspicion should be applied to
/// all blocks coming from the network.
///
/// Of course, this function will only be more efficient if a block is already serialized.
/// Serializing a complete block and then validating with this function will be less
/// efficient than just validating the original block.
///
/// This function will determine if the block is new, already known or invalid (either
/// intrinsically or due to some application error.)
#[allow(dead_code)]
pub fn validate_ssz_block<T>(b: &SszBlock,
                             expected_slot: u64,
                             cycle_length: u8,
                             last_justified_slot: u64,
                             parent_hashes: &Arc<Vec<Hash256>>,
                             proposer_map: &Arc<ProposerMap>,
                             attester_map: &Arc<AttesterMap>,
                             block_store: &Arc<BlockStore<T>>,
                             validator_store: &Arc<ValidatorStore<T>>,
                             pow_store: &Arc<PoWChainStore<T>>)
    -> Result<BlockStatus, SszBlockValidationError>
    where T: ClientDB + Sized
{
    /*
     * If this block is already known, return immediately.
     */
    if block_store.block_exists(&b.block_hash())? {
        return Ok(BlockStatus::KnownBlock);
    }

    /*
     * Copy the block slot (will be used multiple times)
     */
    let block_slot = b.slot_number();

    /*
     * If the block slot corresponds to a slot in the future (according to the local time),
     * drop it.
     */
    if block_slot > expected_slot {
        return Err(SszBlockValidationError::FutureSlot);
    }

    /*
     * If the PoW chain hash is not known to us, drop it.
     *
     * We only accept blocks that reference a known PoW hash.
     *
     * Note: it is not clear what a "known" PoW chain ref is. Likely,
     * it means "sufficienty deep in the canonical PoW chain".
     */
    if !pow_store.block_hash_exists(b.pow_chain_ref())? {
        return Err(SszBlockValidationError::UnknownPoWChainRef);
    }

    /*
     * Store a reference to the serialized attestations from the block.
     */
    let attestations_ssz = &b.attestations();

    /*
     * Get a slice of the first serialized attestation (the 0th) and decode it into
     * a full AttestationRecord object.
     */
    let (first_attestation_ssz, next_index) = split_one_attestation(
        &attestations_ssz,
        0)?;
    let (first_attestation, _) = AttestationRecord::ssz_decode(
        &first_attestation_ssz, 0)?;

    /*
     * Validate this first attestation.
     *
     * It is a requirement that the block proposer for this slot
     * must have signed the 0th attestation record.
     */
    let attestation_voters = validate_attestation(
        &first_attestation,
        block_slot,
        cycle_length,
        last_justified_slot,
        &parent_hashes,
        &block_store,
        &validator_store,
        &attester_map)?;

    /*
     * If the set of voters is None, the attestation was invalid.
     */
    let attestation_voters = attestation_voters
        .ok_or(SszBlockValidationError::
               FirstAttestationSignatureFailed)?;

    /*
     * Read the proposer from the map of slot -> validator index.
     */
    let proposer = proposer_map.get(&block_slot)
        .ok_or(SszBlockValidationError::BadProposerMap)?;

    /*
     * If the proposer for this slot was not a voter, reject the block.
     */
    if !attestation_voters.contains(&proposer) {
        return Err(SszBlockValidationError::NoProposerSignature);
    }

    /*
     * Split the remaining attestations into a vector of slices, each containing
     * a single serialized attestation record.
     */
    let other_attestations = split_all_attestations(attestations_ssz,
                                                    next_index)?;

    /*
     * Verify each other AttestationRecord.
     *
     * TODO: make this parallelized.
     */
    let failure: Option<SszBlockValidationError> = None;
    let failure = RwLock::new(failure);
    other_attestations.iter()
        .for_each(|attestation| {
            if let Some(_) = *failure.read().unwrap() {
                ()
            };
            match AttestationRecord::ssz_decode(&attestation, 0) {
                Ok((a, _)) => {
                    let result = validate_attestation(
                        &a,
                        block_slot,
                        cycle_length,
                        last_justified_slot,
                        &parent_hashes,
                        &block_store,
                        &validator_store,
                        &attester_map);
                    match result {
                        Err(e) => {
                            let mut failure = failure.write().unwrap();
                            *failure = Some(SszBlockValidationError::from(e));
                        }
                        Ok(None) => {
                            let mut failure = failure.write().unwrap();
                            *failure = Some(SszBlockValidationError::AttestationSignatureFailed);
                        }
                        _ => ()
                    }
                }
                Err(e) => {
                    let mut failure = failure.write().unwrap();
                    *failure = Some(SszBlockValidationError::from(e));
                }
            };
        });

    // TODO: handle validation failure. Presently, it will just pass everything

    /*
     * If we have reached this point, the block is a new valid block that is worthy of
     * processing.
     */
    Ok(BlockStatus::NewBlock)
}

impl From<DBError> for SszBlockValidationError {
    fn from(e: DBError) -> Self {
        SszBlockValidationError::DatabaseError(e.message)
    }
}

impl From<AttestationSplitError> for SszBlockValidationError {
    fn from(e: AttestationSplitError) -> Self {
        match e {
            AttestationSplitError::TooShort =>
                SszBlockValidationError::BadAttestationSsz
        }
    }
}

impl From<DecodeError> for SszBlockValidationError {
    fn from(e: DecodeError) -> Self {
        match e {
            DecodeError::TooShort =>
                SszBlockValidationError::BadAttestationSsz,
            DecodeError::TooLong =>
                SszBlockValidationError::BadAttestationSsz,
        }
    }
}

impl From<AttestationValidationError> for SszBlockValidationError {
    fn from(e: AttestationValidationError) -> Self {
        SszBlockValidationError::AttestationValidationError(e)
    }
}
<|endoftext|>


<commit_before>//! Targets the Nintendo Game Boy Advance (GBA),
//! a handheld game device from 2001.
//!
//! Please ping @Lokathor if changes are needed.
//!
//! The target profile assumes that you have the ARM binutils in your path (specifically the linker, `arm-none-eabi-ld`). They can be obtained for free for all major OSes from the ARM developer's website, and they may also be available in your system's package manager
//!
//! **Important:** This target profile **does not** specify a linker script or the ROM header. You'll still need to provide these yourself to construct a final  binary. Generally you'd do this with something like `-Clink-arg=-Tmy_script.ld` and `-Clink-arg=my_crt.o`.

use crate::spec::{LinkerFlavor, LldFlavor, Target, TargetOptions, TargetResult};

pub fn target() -> TargetResult {
    Ok(Target {
        llvm_target: "thumbv4t-none-eabi".to_string(),
        target_endian: "little".to_string(),
        target_pointer_width: "32".to_string(),
        target_c_int_width: "32".to_string(),
        target_os: "none".to_string(),
        target_env: "gba".to_string(),
        target_vendor: "nintendo".to_string(),
        arch: "arm".to_string(),
        /* Data layout args are '-' separated:
         * little endian
         * stack is 64-bit aligned (EABI)
         * pointers are 32-bit
         * i64 must be 64-bit aligned (EABI)
         * mangle names with ELF style
         * native integers are 32-bit
         * All other elements are default
         */
        data_layout: "e-S64-p:32:32-i64:64-m:e-n32".to_string(),
        linker_flavor: LinkerFlavor::Ld,
        options: TargetOptions {
            linker: Some("arm-none-eabi-ld".to_string()),
            linker_is_gnu: true,

            // extra args passed to the external assembler
            asm_args: vec!["-mcpu=arm7tdmi".to_string(), "-mthumb-interwork".to_string()],

            cpu: "arm7tdmi".to_string(),

            // minimum extra features, these cannot be disabled via -C
            features: "+soft-float,+strict-align".to_string(),

            executables: true,

            relocation_model: RelocModel::Static,

            //function_sections: bool,
            //exe_suffix: String,
            main_needs_argc_argv: false,

            // if we have thread-local storage
            has_elf_tls: false,

            // don't have atomic compare-and-swap
            atomic_cas: false,

            // always just abort
            panic_strategy: PanicStrategy::Abort,

            // ABIs to not use
            unsupported_abis: super::arm_base::unsupported_abis(),

            // The minimum alignment for global symbols.
            min_global_align: Some(4),

            // no threads here
            singlethread: true,

            // GBA has no builtins
            no_builtins: true,

            // this is off just like in the `thumb_base`
            emit_debug_gdb_scripts: false,

            ..TargetOptions::default()
        },
    })
}
<commit_msg>fix the imports.<commit_after>//! Targets the Nintendo Game Boy Advance (GBA),
//! a handheld game device from 2001.
//!
//! Please ping @Lokathor if changes are needed.
//!
//! The target profile assumes that you have the ARM binutils in your path (specifically the linker, `arm-none-eabi-ld`). They can be obtained for free for all major OSes from the ARM developer's website, and they may also be available in your system's package manager
//!
//! **Important:** This target profile **does not** specify a linker script or the ROM header. You'll still need to provide these yourself to construct a final  binary. Generally you'd do this with something like `-Clink-arg=-Tmy_script.ld` and `-Clink-arg=my_crt.o`.

use crate::spec::{LinkerFlavor, PanicStrategy, RelocModel, Target, TargetOptions, TargetResult};

pub fn target() -> TargetResult {
    Ok(Target {
        llvm_target: "thumbv4t-none-eabi".to_string(),
        target_endian: "little".to_string(),
        target_pointer_width: "32".to_string(),
        target_c_int_width: "32".to_string(),
        target_os: "none".to_string(),
        target_env: "gba".to_string(),
        target_vendor: "nintendo".to_string(),
        arch: "arm".to_string(),
        /* Data layout args are '-' separated:
         * little endian
         * stack is 64-bit aligned (EABI)
         * pointers are 32-bit
         * i64 must be 64-bit aligned (EABI)
         * mangle names with ELF style
         * native integers are 32-bit
         * All other elements are default
         */
        data_layout: "e-S64-p:32:32-i64:64-m:e-n32".to_string(),
        linker_flavor: LinkerFlavor::Ld,
        options: TargetOptions {
            linker: Some("arm-none-eabi-ld".to_string()),
            linker_is_gnu: true,

            // extra args passed to the external assembler
            asm_args: vec!["-mcpu=arm7tdmi".to_string(), "-mthumb-interwork".to_string()],

            cpu: "arm7tdmi".to_string(),

            // minimum extra features, these cannot be disabled via -C
            features: "+soft-float,+strict-align".to_string(),

            executables: true,

            relocation_model: RelocModel::Static,

            //function_sections: bool,
            //exe_suffix: String,
            main_needs_argc_argv: false,

            // if we have thread-local storage
            has_elf_tls: false,

            // don't have atomic compare-and-swap
            atomic_cas: false,

            // always just abort
            panic_strategy: PanicStrategy::Abort,

            // ABIs to not use
            unsupported_abis: super::arm_base::unsupported_abis(),

            // The minimum alignment for global symbols.
            min_global_align: Some(4),

            // no threads here
            singlethread: true,

            // GBA has no builtins
            no_builtins: true,

            // this is off just like in the `thumb_base`
            emit_debug_gdb_scripts: false,

            ..TargetOptions::default()
        },
    })
}
<|endoftext|>


<commit_before>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

use anyhow::{bail, Result};
use regex::Regex;
use std::time::Duration;
use structopt::StructOpt;

use opentitanlib::app::TransportWrapper;
use opentitanlib::spiflash::{BlockEraseSize, SpiFlash, SupportedAddressModes, WriteGranularity};
use opentitanlib::test_utils::init::InitializeTest;
use opentitanlib::uart::console::{ExitStatus, UartConsole};
use opentitanlib::util::parse_int::ParseInt;

#[derive(Debug, StructOpt)]
struct Opts {
    #[structopt(flatten)]
    init: InitializeTest,

    #[structopt(
        long, parse(try_from_str=humantime::parse_duration),
        default_value = "5s",
        help = "Bootstrap detection timeout",
    )]
    timeout: Duration,

    #[structopt(
        long, parse(try_from_str=usize::from_str),
        default_value = "12",
        help = "JEDEC page of manufacturer",
    )]
    jedec_page: usize,

    #[structopt(
        long, parse(try_from_str=u8::from_str),
        default_value = "0xEF",
        help = "JEDEC ID of manufacturer",
    )]
    jedec_id: u8,

    #[structopt(
        long, parse(try_from_str=u8::from_str),
        default_value = "0x08",
        help = "JEDEC manufacturer product ID",
    )]
    jedec_product: u8,

    #[structopt(
        long, parse(try_from_str=u32::from_str),
        default_value = "0x100000",
        help = "Size of the internal flash",
    )]
    flash_size: u32,
}

#[derive(Debug, PartialEq, Eq)]
enum BootstrapRequest {
    No,
    Yes,
}

fn test_bootstrap_entry(
    opts: &Opts,
    transport: &TransportWrapper,
    request: BootstrapRequest,
) -> Result<()> {
    // TODO: Should really `opts.init.uart_params.create()` here, but we need to refactor
    // BootstrapOptions first.
    //let uart = opts.init.uart_params.create(&transport)?;
    let uart = transport.uart("0")?;

    // Look for the bootstrap signal from the ROM.
    // When enabled, we expect the uart to emit a message indicating bootstrap entry.
    // When disabled, we expect the ROM to indicate boot failure.
    let (success, failure) = match request {
        BootstrapRequest::No => (Regex::new(r"BFV:")?, Regex::new(r"bootstrap:1\r\n")?),
        BootstrapRequest::Yes => (Regex::new(r"bootstrap:1\r\n")?, Regex::new(r"BFV:")?),
    };
    let mut console = UartConsole {
        timeout: Some(opts.timeout),
        exit_success: Some(success),
        exit_failure: Some(failure),
        ..Default::default()
    };

    match request {
        BootstrapRequest::No => transport.remove_pin_strapping("ROM_BOOTSTRAP")?,
        BootstrapRequest::Yes => transport.apply_pin_strapping("ROM_BOOTSTRAP")?,
    }
    let reset_delay = opts
        .init
        .bootstrap
        .options
        .reset_delay
        .unwrap_or(Duration::from_millis(50));
    transport.reset_target(reset_delay, true)?;

    // Now watch the console for the exit conditions.
    let mut stdout = std::io::stdout();
    let result = console.interact(&*uart, None, Some(&mut stdout))?;
    match result {
        ExitStatus::ExitSuccess => {}
        _ => {
            bail!("FAIL: {:?}", result);
        }
    };

    // Now check whether the SPI device is responding to status messages
    let spi = transport.spi("0")?;
    let status = SpiFlash::read_status(&*spi)?;
    match request {
        BootstrapRequest::No => assert_eq!(status, 0xff),
        BootstrapRequest::Yes => assert_eq!(status, 0x00),
    }
    Ok(())
}

fn test_jedec_id(opts: &Opts, transport: &TransportWrapper) -> Result<()> {
    let spi = transport.spi("0")?;
    let id = SpiFlash::read_jedec_id(&*spi, 16)?;
    log::info!("Read jedec id: {:x?}", id);
    let mut index = 0;
    while id[index] == 0x7f {
        index += 1;
    }
    let page = index;
    let manufacturer = id[index];
    let product = id[index + 1];
    let density = id[index + 2];

    assert_eq!(page, opts.jedec_page);
    assert_eq!(manufacturer, opts.jedec_id);
    assert_eq!(product, opts.jedec_product);
    assert_eq!(1u32 << density, opts.flash_size);
    Ok(())
}

fn test_sfdp(opts: &Opts, transport: &TransportWrapper) -> Result<()> {
    let spi = transport.spi("0")?;
    let sfdp = SpiFlash::read_sfdp(&*spi)?;
    log::info!("SFDP: {:#x?}", sfdp);

    // We expect a SFDP header of version 1.10 with no additional phdrs
    // except the mandatory JDEC phdr.
    assert_eq!(sfdp.header.signature, 0x50444653);
    assert_eq!(sfdp.header.minor, 0xa);
    assert_eq!(sfdp.header.major, 0x1);
    assert_eq!(sfdp.header.nph, 0x0);
    assert_eq!(sfdp.header.reserved, 0xff);

    // Check the parameters of the mandatory phdr.
    assert_eq!(sfdp.phdr[0].id, 0x0);
    assert_eq!(sfdp.phdr[0].minor, 0x7);
    assert_eq!(sfdp.phdr[0].major, 0x1);
    assert_eq!(sfdp.phdr[0].dwords, 0x17);
    assert_eq!(sfdp.phdr[0].offset, 0x10);

    // Check the values of the JEDEC parameters.
    // TODO(lowRISC/opentitan#14457): Update these checks after updating the
    // SFDP parser in opentitanlib.
    assert_eq!(sfdp.jedec.block_erase_size, BlockEraseSize::Block4KiB);
    assert_eq!(
        sfdp.jedec.write_granularity,
        WriteGranularity::Granularity64Bytes
    );
    assert_eq!(sfdp.jedec.write_en_required, true);
    assert_eq!(sfdp.jedec.write_en_opcode, 0x50);
    assert_eq!(sfdp.jedec.erase_opcode_4kib, 0x20);
    assert_eq!(sfdp.jedec.support_fast_read_112, false);
    assert_eq!(sfdp.jedec.address_modes, SupportedAddressModes::Mode3b);
    assert_eq!(sfdp.jedec.support_double_rate_clocking, false);
    assert_eq!(sfdp.jedec.support_fast_read_122, false);
    assert_eq!(sfdp.jedec.support_fast_read_144, false);
    assert_eq!(sfdp.jedec.support_fast_read_114, false);
    assert_eq!(sfdp.jedec.support_fast_read_222, false);
    assert_eq!(sfdp.jedec.support_fast_read_444, false);
    assert_eq!(sfdp.jedec.density, opts.flash_size);
    Ok(())
}

fn main() -> Result<()> {
    let opts = Opts::from_args();
    opts.init.init_logging();

    let transport = opts.init.init_target()?;
    test_bootstrap_entry(&opts, &transport, BootstrapRequest::No)?;
    test_bootstrap_entry(&opts, &transport, BootstrapRequest::Yes)?;
    test_jedec_id(&opts, &transport)?;
    test_sfdp(&opts, &transport)?;
    Ok(())
}
<commit_msg>[test] Add rom_e2e_bootstrap_write_enable_disable<commit_after>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

use anyhow::{bail, Result};
use regex::Regex;
use std::time::Duration;
use structopt::StructOpt;

use opentitanlib::app::TransportWrapper;
use opentitanlib::spiflash::{BlockEraseSize, SpiFlash, SupportedAddressModes, WriteGranularity};
use opentitanlib::test_utils::init::InitializeTest;
use opentitanlib::uart::console::{ExitStatus, UartConsole};
use opentitanlib::util::parse_int::ParseInt;

#[derive(Debug, StructOpt)]
struct Opts {
    #[structopt(flatten)]
    init: InitializeTest,

    #[structopt(
        long, parse(try_from_str=humantime::parse_duration),
        default_value = "5s",
        help = "Bootstrap detection timeout",
    )]
    timeout: Duration,

    #[structopt(
        long, parse(try_from_str=usize::from_str),
        default_value = "12",
        help = "JEDEC page of manufacturer",
    )]
    jedec_page: usize,

    #[structopt(
        long, parse(try_from_str=u8::from_str),
        default_value = "0xEF",
        help = "JEDEC ID of manufacturer",
    )]
    jedec_id: u8,

    #[structopt(
        long, parse(try_from_str=u8::from_str),
        default_value = "0x08",
        help = "JEDEC manufacturer product ID",
    )]
    jedec_product: u8,

    #[structopt(
        long, parse(try_from_str=u32::from_str),
        default_value = "0x100000",
        help = "Size of the internal flash",
    )]
    flash_size: u32,
}

#[derive(Debug, PartialEq, Eq)]
enum BootstrapRequest {
    No,
    Yes,
}

fn test_bootstrap_entry(
    opts: &Opts,
    transport: &TransportWrapper,
    request: BootstrapRequest,
) -> Result<()> {
    // TODO: Should really `opts.init.uart_params.create()` here, but we need to refactor
    // BootstrapOptions first.
    //let uart = opts.init.uart_params.create(&transport)?;
    let uart = transport.uart("0")?;

    // Look for the bootstrap signal from the ROM.
    // When enabled, we expect the uart to emit a message indicating bootstrap entry.
    // When disabled, we expect the ROM to indicate boot failure.
    let (success, failure) = match request {
        BootstrapRequest::No => (Regex::new(r"BFV:")?, Regex::new(r"bootstrap:1\r\n")?),
        BootstrapRequest::Yes => (Regex::new(r"bootstrap:1\r\n")?, Regex::new(r"BFV:")?),
    };
    let mut console = UartConsole {
        timeout: Some(opts.timeout),
        exit_success: Some(success),
        exit_failure: Some(failure),
        ..Default::default()
    };

    match request {
        BootstrapRequest::No => transport.remove_pin_strapping("ROM_BOOTSTRAP")?,
        BootstrapRequest::Yes => transport.apply_pin_strapping("ROM_BOOTSTRAP")?,
    }
    let reset_delay = opts
        .init
        .bootstrap
        .options
        .reset_delay
        .unwrap_or(Duration::from_millis(50));
    transport.reset_target(reset_delay, true)?;

    // Now watch the console for the exit conditions.
    let mut stdout = std::io::stdout();
    let result = console.interact(&*uart, None, Some(&mut stdout))?;
    match result {
        ExitStatus::ExitSuccess => {}
        _ => {
            bail!("FAIL: {:?}", result);
        }
    };

    // Now check whether the SPI device is responding to status messages
    let spi = transport.spi("0")?;
    let status = SpiFlash::read_status(&*spi)?;
    match request {
        BootstrapRequest::No => assert_eq!(status, 0xff),
        BootstrapRequest::Yes => assert_eq!(status, 0x00),
    }
    Ok(())
}

fn test_jedec_id(opts: &Opts, transport: &TransportWrapper) -> Result<()> {
    let spi = transport.spi("0")?;
    let id = SpiFlash::read_jedec_id(&*spi, 16)?;
    log::info!("Read jedec id: {:x?}", id);
    let mut index = 0;
    while id[index] == 0x7f {
        index += 1;
    }
    let page = index;
    let manufacturer = id[index];
    let product = id[index + 1];
    let density = id[index + 2];

    assert_eq!(page, opts.jedec_page);
    assert_eq!(manufacturer, opts.jedec_id);
    assert_eq!(product, opts.jedec_product);
    assert_eq!(1u32 << density, opts.flash_size);
    Ok(())
}

fn test_write_enable_disable(transport: &TransportWrapper) -> Result<()> {
    let spi = transport.spi("0")?;

    assert_eq!(SpiFlash::read_status(&*spi)?, 0x0);

    SpiFlash::set_write_enable(&*spi)?;
    assert_eq!(SpiFlash::read_status(&*spi)?, 0x2);

    SpiFlash::set_write_disable(&*spi)?;
    assert_eq!(SpiFlash::read_status(&*spi)?, 0x0);

    Ok(())
}

fn test_sfdp(opts: &Opts, transport: &TransportWrapper) -> Result<()> {
    let spi = transport.spi("0")?;
    let sfdp = SpiFlash::read_sfdp(&*spi)?;
    log::info!("SFDP: {:#x?}", sfdp);

    // We expect a SFDP header of version 1.10 with no additional phdrs
    // except the mandatory JDEC phdr.
    assert_eq!(sfdp.header.signature, 0x50444653);
    assert_eq!(sfdp.header.minor, 0xa);
    assert_eq!(sfdp.header.major, 0x1);
    assert_eq!(sfdp.header.nph, 0x0);
    assert_eq!(sfdp.header.reserved, 0xff);

    // Check the parameters of the mandatory phdr.
    assert_eq!(sfdp.phdr[0].id, 0x0);
    assert_eq!(sfdp.phdr[0].minor, 0x7);
    assert_eq!(sfdp.phdr[0].major, 0x1);
    assert_eq!(sfdp.phdr[0].dwords, 0x17);
    assert_eq!(sfdp.phdr[0].offset, 0x10);

    // Check the values of the JEDEC parameters.
    // TODO(lowRISC/opentitan#14457): Update these checks after updating the
    // SFDP parser in opentitanlib.
    assert_eq!(sfdp.jedec.block_erase_size, BlockEraseSize::Block4KiB);
    assert_eq!(
        sfdp.jedec.write_granularity,
        WriteGranularity::Granularity64Bytes
    );
    assert_eq!(sfdp.jedec.write_en_required, true);
    assert_eq!(sfdp.jedec.write_en_opcode, 0x50);
    assert_eq!(sfdp.jedec.erase_opcode_4kib, 0x20);
    assert_eq!(sfdp.jedec.support_fast_read_112, false);
    assert_eq!(sfdp.jedec.address_modes, SupportedAddressModes::Mode3b);
    assert_eq!(sfdp.jedec.support_double_rate_clocking, false);
    assert_eq!(sfdp.jedec.support_fast_read_122, false);
    assert_eq!(sfdp.jedec.support_fast_read_144, false);
    assert_eq!(sfdp.jedec.support_fast_read_114, false);
    assert_eq!(sfdp.jedec.support_fast_read_222, false);
    assert_eq!(sfdp.jedec.support_fast_read_444, false);
    assert_eq!(sfdp.jedec.density, opts.flash_size);
    Ok(())
}

fn main() -> Result<()> {
    let opts = Opts::from_args();
    opts.init.init_logging();

    let transport = opts.init.init_target()?;
    test_bootstrap_entry(&opts, &transport, BootstrapRequest::No)?;
    test_bootstrap_entry(&opts, &transport, BootstrapRequest::Yes)?;
    test_jedec_id(&opts, &transport)?;
    test_sfdp(&opts, &transport)?;
    test_write_enable_disable(&transport)?;
    Ok(())
}
<|endoftext|>


<commit_before>//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
//!
//! SeaHash is a hash function with performance similar (within around ±7% difference) to XXHash
//! and MetroHash, but with stronger guarantees.
//!
//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
//! to be used for on-disk/permanent storage (e.g. checksums).
//!
//! SeaHash beats xxHash by 3-10% depending on the architecture in use. SeaHash has better quality
//! guarantees as well.
//!
//! # Achieving the performance
//!
//! Like any good general-purpose hash function, SeaHash reads 8 bytes at once effectively reducing
//! the running time by an order of ~5.
//!
//! Secondly, SeaHash achieves the performance by heavily exploiting Instruction-Level Parallelism.
//! In particular, it fetches 4 integers in every round and independently diffuses them. This
//! yields four different states, which are finally combined.
//!
//! # Advantages over other hash functions
//!
//! - Portability: SeaHash always gives the same hash across any platforms, and can thus be used
//!   for e.g. on-disk structures.
//! - Performance: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function that
//!   I know off.
//! - Hardware accelerateable: SeaHash is designed such that ASICs can implement it with really
//!   high performance.
//! - Provable quality guarantees: Contrary to most other non-cryptographic hash function, SeaHash
//!   can be proved to satisfy the avalanche criterion as well as BIC.
//!
//! # Statistical guarantees
//!
//! SeaHash comes with certain proven guarantees about the statistical properties of the output:
//!
//! 1. Pick some _n_-byte sequence, _s_. The number of _n_-byte sequence colliding with _s_ is
//!    independent of the choice of _s_ (all equivalence class have equal size).
//! 2. If you flip any bit in the input, the probability for any bit in the output to be flipped is
//!    0.5.
//!
//! The first guarantee can be derived through deduction, by proving that the diffusion function is
//! bijective (reverse the XORs and find the congruence inverses to the primes).
//!
//! The second guarantee requires more complex calculations: Construct a matrix of probabilities
//! and set one to certain (1), then apply transformations through the respective operations. The
//! proof is a bit long, but relatively simple.
//!
//! # Inner workings
//!
//! In technical terms, SeaHash follows a alternating 4-state length-padded Merkle–Damgård
//! construction with an add-diffuse compression function:
//!
//! ![A diagram.](http://ticki.github.io/img/seahash_construction_diagram.svg)
//!
//! It starts with 4 initial states, then it alternates between them (increment, wrap on 4) and
//! does modular addition with the respective block. When a state has been visited the diffusion
//! function (f) is applied. The very last block is padded with zeros.
//!
//! After all the blocks have been gone over, all the states are added to the number of bytes
//! written. The sum is then passed through the diffusion function, which produces the final hash
//! value.
//!
//! The diffusion function is drawn below.
//!
//! ```notest
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! ```
//!
//! The advantage of having four completely segregated (note that there is no mix round, so they're
//! entirely independent) states is that fast parallelism is possible. For example, if I were to
//! hash 1 TB, I can spawn up four threads which can run independently without _any_
//! intercommunication or syncronization before the last round.
//!
//! If the diffusion function (f) was cryptographically secure, it would pass cryptoanalysis
//! trivially. This might seem irrelavant, as it clearly isn't cryptographically secure, but it
//! tells us something about the inner semantics. In particular, any diffusion function with
//! sufficient statistical quality will make up a good hash function in this construction.

#![no_std]
#![warn(missing_docs)]

use core::num::Wrapping as W;

pub use buffer::hash;
pub use stream::SeaHasher;

pub mod reference;
mod buffer;
mod stream;

/// The diffusion function.
///
/// This is a bijective function emitting chaotic behavior. Such functions are used as building
/// blocks for hash functions.
fn diffuse(mut x: W<u64>) -> W<u64> {
    // Move entropy down by XOR with shifting.
    x ^= x >> 32;
    // Move entropy up by scattering through multiplication.
    x *= W(0x7ed0e9fa0d94a33);
    // We still need more entropy downwards. Flipping higher bits won't flip lower ones, so far.
    // For example, if you flip the most significant bit, the 32'th bit will flip per the XOR-shift
    // subdiffusion, but this flip will only be scattered by the multiplication to flipping bits
    // higher than the 32'th, meaning that the ones lower will be unaffected. As such, we need to
    // get some entropy down.
    x ^= x >> 32;
    // So far, the avalanche diagram looks pretty good, but it still emits stripe patterns. For
    // example, flipping the 5'th lowest bit won't flip the least significant bit because of the
    // choice of scalar (in particular, observe how it leaves the 32'th bit unflipped after the
    // multiplication, which means that the XOR-shift never affects the lowest bit). No choice of
    // scalar will make this go away, it will merely change the unaffected bits. Instead, we need
    // to make the behavior more undeterministic by scattering bits through multiplication.
    x *= W(0x7ed0e9fa0d94a33);
    // This is the final stage of the diffusion function. There are still issues with the lowest
    // bits, which are still unaffected by the multiplication above. However, the multiplication
    // solved the higher bits' dependence, so lending entropy from the higher half will fix the
    // issues with the lower half.
    x ^= x >> 32;

    x
}
<commit_msg>Describe another statistical guarantee.<commit_after>//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
//!
//! SeaHash is a hash function with performance similar (within around ±7% difference) to XXHash
//! and MetroHash, but with stronger guarantees.
//!
//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
//! to be used for on-disk/permanent storage (e.g. checksums).
//!
//! SeaHash beats xxHash by 3-10% depending on the architecture in use. SeaHash has better quality
//! guarantees as well.
//!
//! # Achieving the performance
//!
//! Like any good general-purpose hash function, SeaHash reads 8 bytes at once effectively reducing
//! the running time by an order of ~5.
//!
//! Secondly, SeaHash achieves the performance by heavily exploiting Instruction-Level Parallelism.
//! In particular, it fetches 4 integers in every round and independently diffuses them. This
//! yields four different states, which are finally combined.
//!
//! # Advantages over other hash functions
//!
//! - Portability: SeaHash always gives the same hash across any platforms, and can thus be used
//!   for e.g. on-disk structures.
//! - Performance: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function that
//!   I know off.
//! - Hardware accelerateable: SeaHash is designed such that ASICs can implement it with really
//!   high performance.
//! - Provable quality guarantees: Contrary to most other non-cryptographic hash function, SeaHash
//!   can be proved to satisfy the avalanche criterion as well as BIC.
//!
//! # Statistical guarantees
//!
//! SeaHash comes with certain proven guarantees about the statistical properties of the output:
//!
//! 1. Pick some _n_-byte sequence, _s_. The number of _n_-byte sequence colliding with _s_ is
//!    independent of the choice of _s_ (all equivalence class have equal size).
//! 2. If you flip any bit in the input, the probability for any bit in the output to be flipped is
//!    0.5.
//! 3. The hash value of a sequence of uniformly distributed bytes is itself uniformly distributed.
//!
//! The first guarantee can be derived through deduction, by proving that the diffusion function is
//! bijective (reverse the XORs and find the congruence inverses to the primes).
//!
//! The second guarantee requires more complex calculations: Construct a matrix of probabilities
//! and set one to certain (1), then apply transformations through the respective operations. The
//! proof is a bit long, but relatively simple.
//!
//! The third guarantee requires proving that the hash value is a tree, such that:
//! - Leafs represents the input values.
//! - Single-child nodes reduce to the diffusion of the child.
//! - Multiple-child nodes reduce to the sum of the children.
//!
//! Then simply show that each of these reductions transform uniformly distributed variables to
//! uniformly distributed variables.
//!
//! # Inner workings
//!
//! In technical terms, SeaHash follows a alternating 4-state length-padded Merkle–Damgård
//! construction with an add-diffuse compression function:
//!
//! ![A diagram.](http://ticki.github.io/img/seahash_construction_diagram.svg)
//!
//! It starts with 4 initial states, then it alternates between them (increment, wrap on 4) and
//! does modular addition with the respective block. When a state has been visited the diffusion
//! function (f) is applied. The very last block is padded with zeros.
//!
//! After all the blocks have been gone over, all the states are added to the number of bytes
//! written. The sum is then passed through the diffusion function, which produces the final hash
//! value.
//!
//! The diffusion function is drawn below.
//!
//! ```notest
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! x ← px
//! x ← x ≫ 32
//! ```
//!
//! The advantage of having four completely segregated (note that there is no mix round, so they're
//! entirely independent) states is that fast parallelism is possible. For example, if I were to
//! hash 1 TB, I can spawn up four threads which can run independently without _any_
//! intercommunication or syncronization before the last round.
//!
//! If the diffusion function (f) was cryptographically secure, it would pass cryptoanalysis
//! trivially. This might seem irrelavant, as it clearly isn't cryptographically secure, but it
//! tells us something about the inner semantics. In particular, any diffusion function with
//! sufficient statistical quality will make up a good hash function in this construction.

#![no_std]
#![warn(missing_docs)]

use core::num::Wrapping as W;

pub use buffer::hash;
pub use stream::SeaHasher;

pub mod reference;
mod buffer;
mod stream;

/// The diffusion function.
///
/// This is a bijective function emitting chaotic behavior. Such functions are used as building
/// blocks for hash functions.
fn diffuse(mut x: W<u64>) -> W<u64> {
    // Move entropy down by XOR with shifting.
    x ^= x >> 32;
    // Move entropy up by scattering through multiplication.
    x *= W(0x7ed0e9fa0d94a33);
    // We still need more entropy downwards. Flipping higher bits won't flip lower ones, so far.
    // For example, if you flip the most significant bit, the 32'th bit will flip per the XOR-shift
    // subdiffusion, but this flip will only be scattered by the multiplication to flipping bits
    // higher than the 32'th, meaning that the ones lower will be unaffected. As such, we need to
    // get some entropy down.
    x ^= x >> 32;
    // So far, the avalanche diagram looks pretty good, but it still emits stripe patterns. For
    // example, flipping the 5'th lowest bit won't flip the least significant bit because of the
    // choice of scalar (in particular, observe how it leaves the 32'th bit unflipped after the
    // multiplication, which means that the XOR-shift never affects the lowest bit). No choice of
    // scalar will make this go away, it will merely change the unaffected bits. Instead, we need
    // to make the behavior more undeterministic by scattering bits through multiplication.
    x *= W(0x7ed0e9fa0d94a33);
    // This is the final stage of the diffusion function. There are still issues with the lowest
    // bits, which are still unaffected by the multiplication above. However, the multiplication
    // solved the higher bits' dependence, so lending entropy from the higher half will fix the
    // issues with the lower half.
    x ^= x >> 32;

    x
}
<|endoftext|>