<commit_before>use super::*;
use crate::case_result::compare_result;
use cached_tree_hash::{CachedTreeHash, TreeHashCache};
use rayon::prelude::*;
use serde_derive::Deserialize;
use ssz::Decode;
use std::fmt::Debug;
use tree_hash::TreeHash;
use types::{
    test_utils::{SeedableRng, TestRandom, XorShiftRng},
    Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock,
    BeaconBlockBody, BeaconBlockHeader, BeaconState, Crosslink, Deposit, DepositData, Eth1Data,
    EthSpec, Fork, Hash256, HistoricalBatch, IndexedAttestation, PendingAttestation,
    ProposerSlashing, Transfer, Validator, VoluntaryExit,
};

#[derive(Debug, Clone, Deserialize)]
pub struct SszStatic {
    pub type_name: String,
    pub serialized: String,
    pub root: String,
    #[serde(skip)]
    pub raw_yaml: String,
}

#[derive(Debug, Clone, Deserialize)]
pub struct Value<T> {
    value: T,
}

impl YamlDecode for SszStatic {
    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
        let mut ssz_static: SszStatic = serde_yaml::from_str(&yaml.as_str()).unwrap();

        ssz_static.raw_yaml = yaml.clone();

        Ok(ssz_static)
    }
}

impl SszStatic {
    fn value<T: serde::de::DeserializeOwned>(&self) -> Result<T, Error> {
        let wrapper: Value<T> = serde_yaml::from_str(&self.raw_yaml.as_str()).map_err(|e| {
            Error::FailedToParseTest(format!("Unable to parse {} YAML: {:?}", self.type_name, e))
        })?;

        Ok(wrapper.value)
    }
}

impl EfTest for Cases<SszStatic> {
    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
        self.test_cases
            .par_iter()
            .enumerate()
            .map(|(i, tc)| {
                let result = match tc.type_name.as_ref() {
                    "Fork" => ssz_static_test::<Fork>(tc),
                    "Crosslink" => ssz_static_test::<Crosslink>(tc),
                    "Eth1Data" => ssz_static_test::<Eth1Data>(tc),
                    "AttestationData" => ssz_static_test::<AttestationData>(tc),
                    "AttestationDataAndCustodyBit" => {
                        ssz_static_test::<AttestationDataAndCustodyBit>(tc)
                    }
                    "IndexedAttestation" => ssz_static_test::<IndexedAttestation>(tc),
                    "DepositData" => ssz_static_test::<DepositData>(tc),
                    "BeaconBlockHeader" => ssz_static_test::<BeaconBlockHeader>(tc),
                    "Validator" => ssz_static_test::<Validator>(tc),
                    "PendingAttestation" => ssz_static_test::<PendingAttestation>(tc),
                    "HistoricalBatch" => ssz_static_test::<HistoricalBatch<E>>(tc),
                    "ProposerSlashing" => ssz_static_test::<ProposerSlashing>(tc),
                    "AttesterSlashing" => ssz_static_test::<AttesterSlashing>(tc),
                    "Attestation" => ssz_static_test::<Attestation>(tc),
                    "Deposit" => ssz_static_test::<Deposit>(tc),
                    "VoluntaryExit" => ssz_static_test::<VoluntaryExit>(tc),
                    "Transfer" => ssz_static_test::<Transfer>(tc),
                    "BeaconBlockBody" => ssz_static_test::<BeaconBlockBody>(tc),
                    "BeaconBlock" => ssz_static_test::<BeaconBlock>(tc),
                    "BeaconState" => ssz_static_test::<BeaconState<E>>(tc),
                    _ => Err(Error::FailedToParseTest(format!(
                        "Unknown type: {}",
                        tc.type_name
                    ))),
                };

                CaseResult::new(i, tc, result)
            })
            .collect()
    }
}

fn ssz_static_test<T>(tc: &SszStatic) -> Result<(), Error>
where
    T: Decode
        + Debug
        + PartialEq<T>
        + serde::de::DeserializeOwned
        + TreeHash
        + CachedTreeHash
        + TestRandom,
{
    // Verify we can decode SSZ in the same way we can decode YAML.
    let ssz = hex::decode(&tc.serialized[2..])
        .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
    let expected = tc.value::<T>()?;
    let decode_result = T::from_ssz_bytes(&ssz);
    compare_result(&decode_result, &Some(expected))?;

    // Verify the TreeHash root of the decoded struct matches the test.
    let decoded = decode_result.unwrap();
    let expected_root =
        &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
    let expected_root = Hash256::from_slice(&expected_root);
    let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root());
    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))?;

    // Verify a _new_ CachedTreeHash root of the decoded struct matches the test.
    let cache = TreeHashCache::new(&decoded).unwrap();
    let cached_tree_hash_root = Hash256::from_slice(cache.tree_hash_root().unwrap());
    compare_result::<Hash256, Error>(&Ok(cached_tree_hash_root), &Some(expected_root))?;

    // Verify the root after an update from a random CachedTreeHash to the decoded struct.
    let mut rng = XorShiftRng::from_seed([42; 16]);
    let random_instance = T::random_for_test(&mut rng);
    let mut cache = TreeHashCache::new(&random_instance).unwrap();
    cache.update(&decoded).unwrap();
    let updated_root = Hash256::from_slice(cache.tree_hash_root().unwrap());
    compare_result::<Hash256, Error>(&Ok(updated_root), &Some(expected_root))?;

    Ok(())
}
<commit_msg>Add functionality to check ssz_static encoding as well as existing decoding<commit_after>use super::*;
use crate::case_result::compare_result;
use cached_tree_hash::{CachedTreeHash, TreeHashCache};
use rayon::prelude::*;
use serde_derive::Deserialize;
use ssz::{Decode, Encode, ssz_encode};
use std::fmt::Debug;
use tree_hash::TreeHash;
use types::{
    test_utils::{SeedableRng, TestRandom, XorShiftRng},
    Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock,
    BeaconBlockBody, BeaconBlockHeader, BeaconState, Crosslink, Deposit, DepositData, Eth1Data,
    EthSpec, Fork, Hash256, HistoricalBatch, IndexedAttestation, PendingAttestation,
    ProposerSlashing, Transfer, Validator, VoluntaryExit,
};

#[derive(Debug, Clone, Deserialize)]
pub struct SszStatic {
    pub type_name: String,
    pub serialized: String,
    pub root: String,
    #[serde(skip)]
    pub raw_yaml: String,
}

#[derive(Debug, Clone, Deserialize)]
pub struct Value<T> {
    value: T,
}

impl YamlDecode for SszStatic {
    fn yaml_decode(yaml: &String) -> Result<Self, Error> {
        let mut ssz_static: SszStatic = serde_yaml::from_str(&yaml.as_str()).unwrap();

        ssz_static.raw_yaml = yaml.clone();

        Ok(ssz_static)
    }
}

impl SszStatic {
    fn value<T: serde::de::DeserializeOwned>(&self) -> Result<T, Error> {
        let wrapper: Value<T> = serde_yaml::from_str(&self.raw_yaml.as_str()).map_err(|e| {
            Error::FailedToParseTest(format!("Unable to parse {} YAML: {:?}", self.type_name, e))
        })?;

        Ok(wrapper.value)
    }
}

impl EfTest for Cases<SszStatic> {
    fn test_results<E: EthSpec>(&self) -> Vec<CaseResult> {
        self.test_cases
            .par_iter()
            .enumerate()
            .map(|(i, tc)| {
                let result = match tc.type_name.as_ref() {
                    "Fork" => ssz_static_test::<Fork>(tc),
                    "Crosslink" => ssz_static_test::<Crosslink>(tc),
                    "Eth1Data" => ssz_static_test::<Eth1Data>(tc),
                    "AttestationData" => ssz_static_test::<AttestationData>(tc),
                    "AttestationDataAndCustodyBit" => {
                        ssz_static_test::<AttestationDataAndCustodyBit>(tc)
                    }
                    "IndexedAttestation" => ssz_static_test::<IndexedAttestation>(tc),
                    "DepositData" => ssz_static_test::<DepositData>(tc),
                    "BeaconBlockHeader" => ssz_static_test::<BeaconBlockHeader>(tc),
                    "Validator" => ssz_static_test::<Validator>(tc),
                    "PendingAttestation" => ssz_static_test::<PendingAttestation>(tc),
                    "HistoricalBatch" => ssz_static_test::<HistoricalBatch<E>>(tc),
                    "ProposerSlashing" => ssz_static_test::<ProposerSlashing>(tc),
                    "AttesterSlashing" => ssz_static_test::<AttesterSlashing>(tc),
                    "Attestation" => ssz_static_test::<Attestation>(tc),
                    "Deposit" => ssz_static_test::<Deposit>(tc),
                    "VoluntaryExit" => ssz_static_test::<VoluntaryExit>(tc),
                    "Transfer" => ssz_static_test::<Transfer>(tc),
                    "BeaconBlockBody" => ssz_static_test::<BeaconBlockBody>(tc),
                    "BeaconBlock" => ssz_static_test::<BeaconBlock>(tc),
                    "BeaconState" => ssz_static_test::<BeaconState<E>>(tc),
                    _ => Err(Error::FailedToParseTest(format!(
                        "Unknown type: {}",
                        tc.type_name
                    ))),
                };

                CaseResult::new(i, tc, result)
            })
            .collect()
    }
}

fn ssz_static_test<T>(tc: &SszStatic) -> Result<(), Error>
where
    T: Clone
        + Decode
        + Debug
        + Encode
        + PartialEq<T>
        + serde::de::DeserializeOwned
        + TreeHash
        + CachedTreeHash
        + TestRandom,
{
    // Verify we can decode SSZ in the same way we can decode YAML.
    let ssz = hex::decode(&tc.serialized[2..])
        .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
    let expected = tc.value::<T>()?;
    let decode_result = T::from_ssz_bytes(&ssz);
    compare_result(&decode_result, &Some(expected))?;

    // Verify we can encode the result back into original ssz bytes
    let decoded = decode_result.unwrap();
    let encoded_result = decoded.as_ssz_bytes();
    compare_result::<Vec<u8>, Error>(&Ok(encoded_result), &Some(ssz));

    // Verify the TreeHash root of the decoded struct matches the test.
    let expected_root =
        &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?;
    let expected_root = Hash256::from_slice(&expected_root);
    let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root());
    compare_result::<Hash256, Error>(&Ok(tree_hash_root), &Some(expected_root))?;

    // Verify a _new_ CachedTreeHash root of the decoded struct matches the test.
    let cache = TreeHashCache::new(&decoded).unwrap();
    let cached_tree_hash_root = Hash256::from_slice(cache.tree_hash_root().unwrap());
    compare_result::<Hash256, Error>(&Ok(cached_tree_hash_root), &Some(expected_root))?;

    // Verify the root after an update from a random CachedTreeHash to the decoded struct.
    let mut rng = XorShiftRng::from_seed([42; 16]);
    let random_instance = T::random_for_test(&mut rng);
    let mut cache = TreeHashCache::new(&random_instance).unwrap();
    cache.update(&decoded).unwrap();
    let updated_root = Hash256::from_slice(cache.tree_hash_root().unwrap());
    compare_result::<Hash256, Error>(&Ok(updated_root), &Some(expected_root))?;

    Ok(())
}
<|endoftext|>


<commit_before>use CL::*;
use hl::*;
use error::check;

// These are basically types that can be safely memcpyed.
pub trait VectorType {}

impl int: VectorType;
impl i32: VectorType;
impl u32: VectorType;
impl float: VectorType;
impl f64: VectorType;
impl f32: VectorType;

pub struct Vector<T: VectorType> {
    cl_buffer: cl_mem,
    size: uint,
    context: @ComputeContext,

    drop {
        clReleaseMemObject(self.cl_buffer);
    }
}

pub impl<T: VectorType> Vector<T> {
    static pub fn from_vec(ctx: @ComputeContext, v: &[const T]) -> Vector<T> {
        do vec::as_const_buf(v) |p, len| {
            let mut status = 0;
            let byte_size = len * sys::size_of::<T>() as libc::size_t;

            let buf = clCreateBuffer(ctx.ctx.ctx,
                                     CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
                                     byte_size,
                                     p as *libc::c_void,
                                     ptr::addr_of(&status));
            check(status, "Could not allocate buffer");

            //let status = clEnqueueWriteBuffer(
            //    ctx.q, buf, CL_TRUE, 0, byte_size, p as *libc::c_void,
            //    0, ptr::null(), ptr::null());

            Vector {
                cl_buffer: buf,
                size: len,
                context: ctx,
            }
        }
    }

    pub fn to_vec(self) -> ~[T] unsafe {
        let mut result = ~[];
        vec::reserve(&mut result, self.size);
        vec::raw::set_len(&mut result, self.size);
        do vec::as_imm_buf(result) |p, len| {
            clEnqueueReadBuffer(
                self.context.q.cqueue, self.cl_buffer, CL_TRUE, 0,
                len * sys::size_of::<T>() as libc::size_t,
                p as *libc::c_void, 0, ptr::null(), ptr::null());

        }
        move result
    }
}

impl<T: VectorType> Vector<T>: ::hl::KernelArg {
    pure fn get_value(&self) -> (libc::size_t, *libc::c_void) {
        (sys::size_of::<cl_mem>() as libc::size_t,
         ptr::addr_of(&self.cl_buffer) as *libc::c_void)
    }
}

pub struct Unique<T: VectorType> {
    cl_buffer: cl_mem,
    size: uint,
    context: @ComputeContext,

    drop {
        clReleaseMemObject(self.cl_buffer);
    }
}

pub impl<T: VectorType> Unique<T> {
    static pub fn from_vec(ctx: @ComputeContext, v: ~[const T]) -> Unique<T>
        unsafe
    {
        let byte_size
            = 6 * sys::size_of::<uint>()
            + v.len() * sys::size_of::<T>();
        let byte_size = byte_size as libc::size_t;

        let len = v.len();

        let addr: *libc::c_void = cast::reinterpret_cast(&v);

        let mut status = 0;
        let buf = clCreateBuffer(ctx.ctx.ctx,
                                 CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
                                 byte_size,
                                 addr,
                                 ptr::addr_of(&status));
        check(status, "Could not allocate buffer");

        Unique {
            cl_buffer: buf,
            size: len,
            context: ctx,
        }
    }

    pub fn to_vec(self) -> ~[T] unsafe {
        let mut result = ~[];
        vec::reserve(&mut result, self.size);
        vec::raw::set_len(&mut result, self.size);
        do vec::as_imm_buf(result) |p, len| {
            clEnqueueReadBuffer(
                self.context.q.cqueue, self.cl_buffer, CL_TRUE,
                // Skip the header, we have a new one here.
                (6 * sys::size_of::<uint>()) as libc::size_t,
                len * sys::size_of::<T>() as libc::size_t,
                p as *libc::c_void, 0, ptr::null(), ptr::null());

        }
        move result
    }
}

impl<T: VectorType> Unique<T>: ::hl::KernelArg {
    pure fn get_value(&self) -> (libc::size_t, *libc::c_void) {
        (sys::size_of::<cl_mem>() as libc::size_t,
         ptr::addr_of(&self.cl_buffer) as *libc::c_void)
    }
}

#[cfg(test)]
mod test {
    use hl::*;
    use vector::*;

    macro_rules! expect (
        ($test: expr, $expected: expr) => ({
            let test = $test;
            let expected = $expected;
            if test != expected {
                fail fmt!("Test failure in %s: expected %?, got %?",
                          stringify!($test),
                          expected, test)
            }
        })
    )

    #[test]
    fn gpu_vector() {
        let ctx = create_compute_context();

        let x = ~[1, 2, 3, 4, 5];
        let gx = Vector::from_vec(ctx, x);
        let y = gx.to_vec();
        expect!(y, x);
    }

    #[test]
    fn unique_vector() {
        let ctx = create_compute_context();

        let x = ~[1, 2, 3, 4, 5];
        let gx = Unique::from_vec(ctx, copy x);
        let y = gx.to_vec();
        expect!(y, x);
    }
}
<commit_msg>Adding uint VectorType impl.<commit_after>use CL::*;
use hl::*;
use error::check;

// These are basically types that can be safely memcpyed.
pub trait VectorType {}

impl int: VectorType;
impl i32: VectorType;
impl uint: VectorType;
impl u32: VectorType;
impl float: VectorType;
impl f64: VectorType;
impl f32: VectorType;

pub struct Vector<T: VectorType> {
    cl_buffer: cl_mem,
    size: uint,
    context: @ComputeContext,

    drop {
        clReleaseMemObject(self.cl_buffer);
    }
}

pub impl<T: VectorType> Vector<T> {
    static pub fn from_vec(ctx: @ComputeContext, v: &[const T]) -> Vector<T> {
        do vec::as_const_buf(v) |p, len| {
            let mut status = 0;
            let byte_size = len * sys::size_of::<T>() as libc::size_t;

            let buf = clCreateBuffer(ctx.ctx.ctx,
                                     CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
                                     byte_size,
                                     p as *libc::c_void,
                                     ptr::addr_of(&status));
            check(status, "Could not allocate buffer");

            //let status = clEnqueueWriteBuffer(
            //    ctx.q, buf, CL_TRUE, 0, byte_size, p as *libc::c_void,
            //    0, ptr::null(), ptr::null());

            Vector {
                cl_buffer: buf,
                size: len,
                context: ctx,
            }
        }
    }

    pub fn to_vec(self) -> ~[T] unsafe {
        let mut result = ~[];
        vec::reserve(&mut result, self.size);
        vec::raw::set_len(&mut result, self.size);
        do vec::as_imm_buf(result) |p, len| {
            clEnqueueReadBuffer(
                self.context.q.cqueue, self.cl_buffer, CL_TRUE, 0,
                len * sys::size_of::<T>() as libc::size_t,
                p as *libc::c_void, 0, ptr::null(), ptr::null());

        }
        move result
    }
}

impl<T: VectorType> Vector<T>: ::hl::KernelArg {
    pure fn get_value(&self) -> (libc::size_t, *libc::c_void) {
        (sys::size_of::<cl_mem>() as libc::size_t,
         ptr::addr_of(&self.cl_buffer) as *libc::c_void)
    }
}

pub struct Unique<T: VectorType> {
    cl_buffer: cl_mem,
    size: uint,
    context: @ComputeContext,

    drop {
        clReleaseMemObject(self.cl_buffer);
    }
}

pub impl<T: VectorType> Unique<T> {
    static pub fn from_vec(ctx: @ComputeContext, v: ~[const T]) -> Unique<T>
        unsafe
    {
        let byte_size
            = 6 * sys::size_of::<uint>()
            + v.len() * sys::size_of::<T>();
        let byte_size = byte_size as libc::size_t;

        let len = v.len();

        let addr: *libc::c_void = cast::reinterpret_cast(&v);

        let mut status = 0;
        let buf = clCreateBuffer(ctx.ctx.ctx,
                                 CL_MEM_READ_WRITE | CL_MEM_COPY_HOST_PTR,
                                 byte_size,
                                 addr,
                                 ptr::addr_of(&status));
        check(status, "Could not allocate buffer");

        Unique {
            cl_buffer: buf,
            size: len,
            context: ctx,
        }
    }

    pub fn to_vec(self) -> ~[T] unsafe {
        let mut result = ~[];
        vec::reserve(&mut result, self.size);
        vec::raw::set_len(&mut result, self.size);
        do vec::as_imm_buf(result) |p, len| {
            clEnqueueReadBuffer(
                self.context.q.cqueue, self.cl_buffer, CL_TRUE,
                // Skip the header, we have a new one here.
                (6 * sys::size_of::<uint>()) as libc::size_t,
                len * sys::size_of::<T>() as libc::size_t,
                p as *libc::c_void, 0, ptr::null(), ptr::null());

        }
        move result
    }
}

impl<T: VectorType> Unique<T>: ::hl::KernelArg {
    pure fn get_value(&self) -> (libc::size_t, *libc::c_void) {
        (sys::size_of::<cl_mem>() as libc::size_t,
         ptr::addr_of(&self.cl_buffer) as *libc::c_void)
    }
}

#[cfg(test)]
mod test {
    use hl::*;
    use vector::*;

    macro_rules! expect (
        ($test: expr, $expected: expr) => ({
            let test = $test;
            let expected = $expected;
            if test != expected {
                fail fmt!("Test failure in %s: expected %?, got %?",
                          stringify!($test),
                          expected, test)
            }
        })
    )

    #[test]
    fn gpu_vector() {
        let ctx = create_compute_context();

        let x = ~[1, 2, 3, 4, 5];
        let gx = Vector::from_vec(ctx, x);
        let y = gx.to_vec();
        expect!(y, x);
    }

    #[test]
    fn unique_vector() {
        let ctx = create_compute_context();

        let x = ~[1, 2, 3, 4, 5];
        let gx = Unique::from_vec(ctx, copy x);
        let y = gx.to_vec();
        expect!(y, x);
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    collections::HashMap,
    convert::TryInto,
    env::args,
    error::Error,
    fmt::{self, Debug, Display},
    os::unix::net::UnixDatagram,
    time::Duration,
};

use dbus::{
    arg::{RefArg, Variant},
    blocking::Connection,
    Path,
};
use lazy_static::lazy_static;
use libdbus_sys::DBUS_TIMEOUT_INFINITE;
use regex::Regex;
use uuid::Uuid;

const DEV_LOG: &str = "/dev/log";
/// Syslog priority syntax
/// (3 (SYSTEM) << 3) | 7 (DEBUG)
const SYSTEM_DAEMON_DEBUG: &str = "<31>";
/// Syslog priority syntax
/// (3 (SYSTEM) << 3) | 3 (ERROR)
const SYSTEM_DAEMON_ERROR: &str = "<27>";
const STRATIS_BUS_NAME: &str = "org.storage.stratis2";
const STRATIS_MANAGER_OBJECT: &str = "/org/storage/stratis2";
const STRATIS_POOL_IFACE: &str = "org.storage.stratis2.pool.r1";
const STRATIS_FS_IFACE: &str = "org.storage.stratis2.filesystem";
const DBUS_OM_IFACE: &str = "org.freedesktop.DBus.ObjectManager";

lazy_static! {
    static ref TIMEOUT: Duration = Duration::from_millis(
        DBUS_TIMEOUT_INFINITE
            .try_into()
            .expect("statically verified")
    );
}

struct StratisUdevError(Option<String>);

impl StratisUdevError {
    fn new<D>(display: D) -> StratisUdevError
    where
        D: Display,
    {
        StratisUdevError(Some(display.to_string()))
    }
}

impl<E> From<E> for StratisUdevError
where
    E: Error,
{
    fn from(e: E) -> StratisUdevError {
        StratisUdevError::new(e)
    }
}

impl Debug for StratisUdevError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Could not convert Stratis UUIDs to names.")?;
        if let Some(ref msg) = self.0 {
            write!(f, " {}", msg)
        } else {
            Ok(())
        }
    }
}

type GMORet =
    HashMap<Path<'static>, HashMap<String, HashMap<String, Variant<Box<dyn RefArg + 'static>>>>>;

fn get_managed_objects() -> Result<GMORet, StratisUdevError> {
    let connection = Connection::new_system()?;
    let proxy = connection.with_proxy(STRATIS_BUS_NAME, STRATIS_MANAGER_OBJECT, *TIMEOUT);
    Ok(proxy
        .method_call(DBUS_OM_IFACE, "GetManagedObjects", ())
        .map(|r: (GMORet,)| r.0)?)
}

fn udev_name_to_uuids(dm_name: &str) -> Result<Option<(Uuid, Uuid)>, StratisUdevError> {
    let regex = Regex::new("stratis-1-(?P<pool>[0-9a-f]{32})-thin-fs-(?P<fs>[0-9a-f]{32})")?;
    let captures = match regex.captures(dm_name) {
        Some(cap) => cap,
        None => return Ok(None),
    };
    let pool_uuid = captures
        .name("pool")
        .map(|pu| Uuid::parse_str(pu.as_str()).expect("Format validated by regex"));
    let fs_uuid = captures
        .name("fs")
        .map(|pu| Uuid::parse_str(pu.as_str()).expect("Format validated by regex"));
    Ok(pool_uuid.and_then(|pu| fs_uuid.map(|fu| (pu, fu))))
}

fn uuid_to_stratis_name(
    managed_objects: &GMORet,
    iface_name: &'static str,
    uuid: Uuid,
) -> Result<Option<String>, StratisUdevError> {
    let mut names: Vec<_> = managed_objects
        .values()
        .filter_map(|map| {
            map.get(iface_name).and_then(|submap| {
                if submap.get("Uuid").and_then(|uuid_val| uuid_val.as_str())
                    == Some(&uuid.to_simple_ref().to_string())
                {
                    submap
                        .get("Name")
                        .and_then(|name| name.as_str().map(|n| n.to_string()))
                } else {
                    None
                }
            })
        })
        .collect();

    if names.len() > 1 {
        Err(StratisUdevError::new(format!(
            "More than one device has the UUID {}",
            uuid
        )))
    } else {
        Ok(names.pop())
    }
}

fn main_report_error() -> Result<Option<(String, String)>, StratisUdevError> {
    let mut args = args();
    let dm_name = match args.nth(1) {
        Some(dm_name) => dm_name,
        None => {
            return Err(StratisUdevError::new(
                "Thinly provisioned filesystem devicemapper name required as argument.",
            ));
        }
    };

    let managed_objects = get_managed_objects()?;

    if let Some((pool_uuid, fs_uuid)) = udev_name_to_uuids(&dm_name)? {
        let pool_name = uuid_to_stratis_name(&managed_objects, STRATIS_POOL_IFACE, pool_uuid)?
            .ok_or_else(|| StratisUdevError::new("Could not get pool name from UUID."))?;
        let fs_name = uuid_to_stratis_name(&managed_objects, STRATIS_FS_IFACE, fs_uuid)?
            .ok_or_else(|| StratisUdevError::new("Could not get filesystem name from UUID."))?;
        Ok(Some((pool_name, fs_name)))
    } else {
        Ok(None)
    }
}

fn main() -> Result<(), StratisUdevError> {
    let sock = UnixDatagram::unbound().map_err(StratisUdevError::new)?;
    match main_report_error() {
        Ok(Some((pool_name, fs_name))) => {
            sock.send_to(
                format!(
                    "{}stratisd reported pool name {} and filesystem name {} to udev for given Stratis filesystem.",
                    SYSTEM_DAEMON_DEBUG, pool_name, fs_name,
                )
                .as_bytes(),
                DEV_LOG,
            )
            .map_err(StratisUdevError::new)?;

            println!("STRATIS_SYMLINK=stratis/{}/{}", pool_name, fs_name);
            Ok(())
        }
        Ok(None) => Ok(()),
        Err(e) => {
            sock.send_to(
                format!("{}{:?}", SYSTEM_DAEMON_ERROR, e).as_bytes(),
                DEV_LOG,
            )
            .map_err(StratisUdevError::new)?;
            Err(e)
        }
    }
}
<commit_msg>Print UUID for unacquirable fs or pool name<commit_after>// Copyright 2020 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    collections::HashMap,
    convert::TryInto,
    env::args,
    error::Error,
    fmt::{self, Debug, Display},
    os::unix::net::UnixDatagram,
    time::Duration,
};

use dbus::{
    arg::{RefArg, Variant},
    blocking::Connection,
    Path,
};
use lazy_static::lazy_static;
use libdbus_sys::DBUS_TIMEOUT_INFINITE;
use regex::Regex;
use uuid::Uuid;

const DEV_LOG: &str = "/dev/log";
/// Syslog priority syntax
/// (3 (SYSTEM) << 3) | 7 (DEBUG)
const SYSTEM_DAEMON_DEBUG: &str = "<31>";
/// Syslog priority syntax
/// (3 (SYSTEM) << 3) | 3 (ERROR)
const SYSTEM_DAEMON_ERROR: &str = "<27>";
const STRATIS_BUS_NAME: &str = "org.storage.stratis2";
const STRATIS_MANAGER_OBJECT: &str = "/org/storage/stratis2";
const STRATIS_POOL_IFACE: &str = "org.storage.stratis2.pool.r1";
const STRATIS_FS_IFACE: &str = "org.storage.stratis2.filesystem";
const DBUS_OM_IFACE: &str = "org.freedesktop.DBus.ObjectManager";

lazy_static! {
    static ref TIMEOUT: Duration = Duration::from_millis(
        DBUS_TIMEOUT_INFINITE
            .try_into()
            .expect("statically verified")
    );
}

struct StratisUdevError(Option<String>);

impl StratisUdevError {
    fn new<D>(display: D) -> StratisUdevError
    where
        D: Display,
    {
        StratisUdevError(Some(display.to_string()))
    }
}

impl<E> From<E> for StratisUdevError
where
    E: Error,
{
    fn from(e: E) -> StratisUdevError {
        StratisUdevError::new(e)
    }
}

impl Debug for StratisUdevError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Could not convert Stratis UUIDs to names.")?;
        if let Some(ref msg) = self.0 {
            write!(f, " {}", msg)
        } else {
            Ok(())
        }
    }
}

type GMORet =
    HashMap<Path<'static>, HashMap<String, HashMap<String, Variant<Box<dyn RefArg + 'static>>>>>;

fn get_managed_objects() -> Result<GMORet, StratisUdevError> {
    let connection = Connection::new_system()?;
    let proxy = connection.with_proxy(STRATIS_BUS_NAME, STRATIS_MANAGER_OBJECT, *TIMEOUT);
    Ok(proxy
        .method_call(DBUS_OM_IFACE, "GetManagedObjects", ())
        .map(|r: (GMORet,)| r.0)?)
}

fn udev_name_to_uuids(dm_name: &str) -> Result<Option<(Uuid, Uuid)>, StratisUdevError> {
    let regex = Regex::new("stratis-1-(?P<pool>[0-9a-f]{32})-thin-fs-(?P<fs>[0-9a-f]{32})")?;
    let captures = match regex.captures(dm_name) {
        Some(cap) => cap,
        None => return Ok(None),
    };
    let pool_uuid = captures
        .name("pool")
        .map(|pu| Uuid::parse_str(pu.as_str()).expect("Format validated by regex"));
    let fs_uuid = captures
        .name("fs")
        .map(|pu| Uuid::parse_str(pu.as_str()).expect("Format validated by regex"));
    Ok(pool_uuid.and_then(|pu| fs_uuid.map(|fu| (pu, fu))))
}

fn uuid_to_stratis_name(
    managed_objects: &GMORet,
    iface_name: &'static str,
    uuid: Uuid,
) -> Result<Option<String>, StratisUdevError> {
    let mut names: Vec<_> = managed_objects
        .values()
        .filter_map(|map| {
            map.get(iface_name).and_then(|submap| {
                if submap.get("Uuid").and_then(|uuid_val| uuid_val.as_str())
                    == Some(&uuid.to_simple_ref().to_string())
                {
                    submap
                        .get("Name")
                        .and_then(|name| name.as_str().map(|n| n.to_string()))
                } else {
                    None
                }
            })
        })
        .collect();

    if names.len() > 1 {
        Err(StratisUdevError::new(format!(
            "More than one device has the UUID {}",
            uuid
        )))
    } else {
        Ok(names.pop())
    }
}

fn main_report_error() -> Result<Option<(String, String)>, StratisUdevError> {
    let mut args = args();
    let dm_name = match args.nth(1) {
        Some(dm_name) => dm_name,
        None => {
            return Err(StratisUdevError::new(
                "Thinly provisioned filesystem devicemapper name required as argument.",
            ));
        }
    };

    let managed_objects = get_managed_objects()?;

    if let Some((pool_uuid, fs_uuid)) = udev_name_to_uuids(&dm_name)? {
        let pool_name = uuid_to_stratis_name(&managed_objects, STRATIS_POOL_IFACE, pool_uuid)?
            .ok_or_else(|| {
                StratisUdevError::new(format!("Could not get pool name from UUID {}.", pool_uuid))
            })?;
        let fs_name = uuid_to_stratis_name(&managed_objects, STRATIS_FS_IFACE, fs_uuid)?
            .ok_or_else(|| {
                StratisUdevError::new(format!(
                    "Could not get filesystem name from UUID {}.",
                    fs_uuid
                ))
            })?;
        Ok(Some((pool_name, fs_name)))
    } else {
        Ok(None)
    }
}

fn main() -> Result<(), StratisUdevError> {
    let sock = UnixDatagram::unbound().map_err(StratisUdevError::new)?;
    match main_report_error() {
        Ok(Some((pool_name, fs_name))) => {
            sock.send_to(
                format!(
                    "{}stratisd reported pool name {} and filesystem name {} to udev for given Stratis filesystem.",
                    SYSTEM_DAEMON_DEBUG, pool_name, fs_name,
                )
                .as_bytes(),
                DEV_LOG,
            )
            .map_err(StratisUdevError::new)?;

            println!("STRATIS_SYMLINK=stratis/{}/{}", pool_name, fs_name);
            Ok(())
        }
        Ok(None) => Ok(()),
        Err(e) => {
            sock.send_to(
                format!("{}{:?}", SYSTEM_DAEMON_ERROR, e).as_bytes(),
                DEV_LOG,
            )
            .map_err(StratisUdevError::new)?;
            Err(e)
        }
    }
}
<|endoftext|>


<commit_before>use std::{
    sync::{
        atomic::{AtomicBool, AtomicUsize, Ordering},
        Arc,
    },
    thread,
};

#[cfg(feature = "jemalloc")]
use jemallocator::Jemalloc;

#[cfg(feature = "dh")]
use dhat::{Dhat, DhatAlloc};

use num_format::{Locale, ToFormattedString};
use rand::{thread_rng, Rng};

#[global_allocator]
#[cfg(feature = "jemalloc")]
static ALLOCATOR: Jemalloc = Jemalloc;

#[global_allocator]
#[cfg(feature = "dh")]
static ALLOCATOR: DhatAlloc = DhatAlloc;

static TOTAL: AtomicUsize = AtomicUsize::new(0);

const USAGE: &str = "
Usage: stress [--threads=<#>] [--burn-in] [--duration=<s>] \
    [--key-len=<l>] [--val-len=<l>] \
    [--get-prop=<p>] \
    [--set-prop=<p>] \
    [--del-prop=<p>] \
    [--cas-prop=<p>] \
    [--scan-prop=<p>] \
    [--merge-prop=<p>] \
    [--entries=<n>] \
    [--sequential] \
    [--total-ops=<n>] \
    [--flush-every=<ms>]

Options:
    --threads=<#>      Number of threads [default: 4].
    --burn-in          Don't halt until we receive a signal.
    --duration=<s>     Seconds to run for [default: 10].
    --key-len=<l>      The length of keys [default: 10].
    --val-len=<l>      The length of values [default: 100].
    --get-prop=<p>     The relative proportion of get requests [default: 94].
    --set-prop=<p>     The relative proportion of set requests [default: 2].
    --del-prop=<p>     The relative proportion of del requests [default: 1].
    --cas-prop=<p>     The relative proportion of cas requests [default: 1].
    --scan-prop=<p>    The relative proportion of scan requests [default: 1].
    --merge-prop=<p>   The relative proportion of merge requests [default: 1].
    --entries=<n>      The total keyspace [default: 100000].
    --sequential       Run the test in sequential mode instead of random.
    --total-ops=<n>    Stop test after executing a total number of operations.
    --flush-every=<m>  Flush and sync the database every ms [default: 200].
    --cache-mb=<mb>    Size of the page cache in megabytes [default: 1024].
";

#[derive(Clone, Copy)]
struct Args {
    threads: usize,
    burn_in: bool,
    duration: u64,
    key_len: usize,
    val_len: usize,
    get_prop: usize,
    set_prop: usize,
    del_prop: usize,
    cas_prop: usize,
    scan_prop: usize,
    merge_prop: usize,
    entries: usize,
    sequential: bool,
    total_ops: Option<usize>,
    flush_every: u64,
    cache_mb: usize,
}

impl Default for Args {
    fn default() -> Args {
        Args {
            threads: 4,
            burn_in: false,
            duration: 10,
            key_len: 10,
            val_len: 100,
            get_prop: 94,
            set_prop: 2,
            del_prop: 1,
            cas_prop: 1,
            scan_prop: 1,
            merge_prop: 1,
            entries: 100000,
            sequential: false,
            total_ops: None,
            flush_every: 200,
            cache_mb: 1024,
        }
    }
}

fn parse<'a, I, T>(mut iter: I) -> T
where
    I: Iterator<Item = &'a str>,
    T: std::str::FromStr,
    <T as std::str::FromStr>::Err: std::fmt::Debug,
{
    iter.next().expect(USAGE).parse().expect(USAGE)
}

impl Args {
    fn parse() -> Args {
        let mut args = Args::default();
        for raw_arg in std::env::args().skip(1) {
            let mut splits = raw_arg[2..].split('=');
            match splits.next().unwrap() {
                "threads" => args.threads = parse(&mut splits),
                "burn-in" => args.burn_in = true,
                "duration" => args.duration = parse(&mut splits),
                "key-len" => args.key_len = parse(&mut splits),
                "val-len" => args.val_len = parse(&mut splits),
                "get-prop" => args.get_prop = parse(&mut splits),
                "set-prop" => args.set_prop = parse(&mut splits),
                "del-prop" => args.del_prop = parse(&mut splits),
                "cas-prop" => args.cas_prop = parse(&mut splits),
                "scan-prop" => args.scan_prop = parse(&mut splits),
                "merge-prop" => args.merge_prop = parse(&mut splits),
                "entries" => args.entries = parse(&mut splits),
                "sequential" => args.sequential = true,
                "total-ops" => args.total_ops = Some(parse(&mut splits)),
                "flush-every" => args.flush_every = parse(&mut splits),
                other => panic!("unknown option: {}, {}", other, USAGE),
            }
        }
        args
    }
}

fn report(shutdown: Arc<AtomicBool>) {
    let mut last = 0;
    while !shutdown.load(Ordering::Relaxed) {
        thread::sleep(std::time::Duration::from_secs(1));
        let total = TOTAL.load(Ordering::Acquire);

        println!(
            "did {} ops, {}mb RSS",
            (total - last).to_formatted_string(&Locale::en),
            rss() / (1024 * 1024)
        );

        last = total;
    }
}

fn concatenate_merge(
    _key: &[u8],              // the key being merged
    old_value: Option<&[u8]>, // the previous value, if one existed
    merged_bytes: &[u8],      // the new bytes being merged in
) -> Option<Vec<u8>> {
    // set the new value, return None to delete
    let mut ret = old_value.map(|ov| ov.to_vec()).unwrap_or_else(Vec::new);

    ret.extend_from_slice(merged_bytes);

    Some(ret)
}

fn run(args: Args, tree: Arc<sled::Db>, shutdown: Arc<AtomicBool>) {
    let get_max = args.get_prop;
    let set_max = get_max + args.set_prop;
    let del_max = set_max + args.del_prop;
    let cas_max = del_max + args.cas_prop;
    let merge_max = cas_max + args.merge_prop;
    let scan_max = merge_max + args.scan_prop;

    let keygen = |len| -> sled::IVec {
        static SEQ: AtomicUsize = AtomicUsize::new(0);
        let i = if args.sequential {
            SEQ.fetch_add(1, Ordering::Relaxed)
        } else {
            thread_rng().gen::<usize>()
        } % args.entries;

        let start = if len < 8 { 8 - len } else { 0 };

        let i_keygen = &i.to_be_bytes()[start..];

        i_keygen.iter().cycle().take(len).copied().collect()
    };

    let valgen = |len| -> sled::IVec {
        if len == 0 {
            return vec![].into();
        }

        let i: usize = thread_rng().gen::<usize>() % (len * 8);

        let i_keygen = i.to_be_bytes();

        i_keygen
            .iter()
            .skip_while(|v| **v == 0)
            .cycle()
            .take(len)
            .copied()
            .collect()
    };

    let mut rng = thread_rng();

    while !shutdown.load(Ordering::Relaxed) {
        let op = TOTAL.fetch_add(1, Ordering::Release);
        let key = keygen(args.key_len);
        let choice = rng.gen_range(0, scan_max + 1);

        match choice {
            v if v <= get_max => {
                tree.get_zero_copy(&key, |_| {}).unwrap();
            }
            v if v > get_max && v <= set_max => {
                let value = valgen(args.val_len);
                tree.insert(&key, value).unwrap();
            }
            v if v > set_max && v <= del_max => {
                tree.remove(&key).unwrap();
            }
            v if v > del_max && v <= cas_max => {
                let old = if rng.gen::<bool>() {
                    let value = valgen(args.val_len);
                    Some(value)
                } else {
                    None
                };

                let new = if rng.gen::<bool>() {
                    let value = valgen(args.val_len);
                    Some(value)
                } else {
                    None
                };

                if let Err(e) = tree.compare_and_swap(&key, old, new) {
                    panic!("operational error: {:?}", e);
                }
            }
            v if v > cas_max && v <= merge_max => {
                let value = valgen(args.val_len);
                tree.merge(&key, value).unwrap();
            }
            _ => {
                let iter = tree.range(key..).map(|res| res.unwrap());

                if op % 2 == 0 {
                    let _ = iter.take(rng.gen_range(0, 15)).collect::<Vec<_>>();
                } else {
                    let _ = iter
                        .rev()
                        .take(rng.gen_range(0, 15))
                        .collect::<Vec<_>>();
                }
            }
        }
    }
}

fn rss() -> usize {
    #[cfg(target_os = "linux")]
    {
        use std::io::prelude::*;
        use std::io::BufReader;

        let mut buf = String::new();
        let mut f =
            BufReader::new(std::fs::File::open("/proc/self/statm").unwrap());
        f.read_line(&mut buf).unwrap();
        let mut parts = buf.split_whitespace();
        let rss_pages = parts.nth(1).unwrap().parse::<usize>().unwrap();
        rss_pages * 4096
    }
    #[cfg(not(target_os = "linux"))]
    {
        0
    }
}

fn main() {
    #[cfg(feature = "logging")]
    setup_logger();

    #[cfg(feature = "dh")]
    let _dh = Dhat::start_heap_profiling();

    let args = Args::parse();

    let shutdown = Arc::new(AtomicBool::new(false));

    let config = sled::Config::new()
        .cache_capacity(args.cache_mb as u64 * 1024 * 1024)
        .flush_every_ms(if args.flush_every == 0 {
            None
        } else {
            Some(args.flush_every)
        });

    let tree = Arc::new(config.open().unwrap());
    tree.set_merge_operator(concatenate_merge);

    let mut threads = vec![];

    let now = std::time::Instant::now();

    let n_threads = args.threads;

    for i in 0..=n_threads {
        let tree = tree.clone();
        let shutdown = shutdown.clone();

        let t = if i == 0 {
            thread::Builder::new()
                .name("reporter".into())
                .spawn(move || report(shutdown))
                .unwrap()
        } else {
            thread::spawn(move || run(args, tree, shutdown))
        };

        threads.push(t);
    }

    if let Some(ops) = args.total_ops {
        assert!(!args.burn_in, "don't set both --burn-in and --total-ops");
        while TOTAL.load(Ordering::Relaxed) < ops {
            thread::sleep(std::time::Duration::from_millis(50));
        }
        shutdown.store(true, Ordering::SeqCst);
    } else if !args.burn_in {
        thread::sleep(std::time::Duration::from_secs(args.duration));
        shutdown.store(true, Ordering::SeqCst);
    }

    for t in threads.into_iter() {
        t.join().unwrap();
    }
    let ops = TOTAL.load(Ordering::SeqCst);
    let time = now.elapsed().as_secs() as usize;

    println!(
        "did {} total ops in {} seconds. {} ops/s",
        ops.to_formatted_string(&Locale::en),
        time,
        ((ops * 1_000) / (time * 1_000)).to_formatted_string(&Locale::en)
    );

    #[cfg(feature = "metrics")]
    sled::print_profile();
}

#[cfg(feature = "logging")]
pub fn setup_logger() {
    use std::io::Write;

    color_backtrace::install();

    fn tn() -> String {
        std::thread::current().name().unwrap_or("unknown").to_owned()
    }

    let mut builder = env_logger::Builder::new();
    builder
        .format(|buf, record| {
            writeln!(
                buf,
                "{:05} {:25} {:10} {}",
                record.level(),
                tn(),
                record.module_path().unwrap().split("::").last().unwrap(),
                record.args()
            )
        })
        .filter(None, log::LevelFilter::Info);

    if let Ok(env) = std::env::var("RUST_LOG") {
        builder.parse_filters(&env);
    }

    let _r = builder.try_init();
}
<commit_msg>Fix cache-mb stress2 config<commit_after>use std::{
    sync::{
        atomic::{AtomicBool, AtomicUsize, Ordering},
        Arc,
    },
    thread,
};

#[cfg(feature = "jemalloc")]
use jemallocator::Jemalloc;

#[cfg(feature = "dh")]
use dhat::{Dhat, DhatAlloc};

use num_format::{Locale, ToFormattedString};
use rand::{thread_rng, Rng};

#[global_allocator]
#[cfg(feature = "jemalloc")]
static ALLOCATOR: Jemalloc = Jemalloc;

#[global_allocator]
#[cfg(feature = "dh")]
static ALLOCATOR: DhatAlloc = DhatAlloc;

static TOTAL: AtomicUsize = AtomicUsize::new(0);

const USAGE: &str = "
Usage: stress [--threads=<#>] [--burn-in] [--duration=<s>] \
    [--key-len=<l>] [--val-len=<l>] \
    [--get-prop=<p>] \
    [--set-prop=<p>] \
    [--del-prop=<p>] \
    [--cas-prop=<p>] \
    [--scan-prop=<p>] \
    [--merge-prop=<p>] \
    [--entries=<n>] \
    [--sequential] \
    [--total-ops=<n>] \
    [--flush-every=<ms>]

Options:
    --threads=<#>      Number of threads [default: 4].
    --burn-in          Don't halt until we receive a signal.
    --duration=<s>     Seconds to run for [default: 10].
    --key-len=<l>      The length of keys [default: 10].
    --val-len=<l>      The length of values [default: 100].
    --get-prop=<p>     The relative proportion of get requests [default: 94].
    --set-prop=<p>     The relative proportion of set requests [default: 2].
    --del-prop=<p>     The relative proportion of del requests [default: 1].
    --cas-prop=<p>     The relative proportion of cas requests [default: 1].
    --scan-prop=<p>    The relative proportion of scan requests [default: 1].
    --merge-prop=<p>   The relative proportion of merge requests [default: 1].
    --entries=<n>      The total keyspace [default: 100000].
    --sequential       Run the test in sequential mode instead of random.
    --total-ops=<n>    Stop test after executing a total number of operations.
    --flush-every=<m>  Flush and sync the database every ms [default: 200].
    --cache-mb=<mb>    Size of the page cache in megabytes [default: 1024].
";

#[derive(Debug, Clone, Copy)]
struct Args {
    threads: usize,
    burn_in: bool,
    duration: u64,
    key_len: usize,
    val_len: usize,
    get_prop: usize,
    set_prop: usize,
    del_prop: usize,
    cas_prop: usize,
    scan_prop: usize,
    merge_prop: usize,
    entries: usize,
    sequential: bool,
    total_ops: Option<usize>,
    flush_every: u64,
    cache_mb: usize,
}

impl Default for Args {
    fn default() -> Args {
        Args {
            threads: 4,
            burn_in: false,
            duration: 10,
            key_len: 10,
            val_len: 100,
            get_prop: 94,
            set_prop: 2,
            del_prop: 1,
            cas_prop: 1,
            scan_prop: 1,
            merge_prop: 1,
            entries: 100000,
            sequential: false,
            total_ops: None,
            flush_every: 200,
            cache_mb: 1024,
        }
    }
}

fn parse<'a, I, T>(mut iter: I) -> T
where
    I: Iterator<Item = &'a str>,
    T: std::str::FromStr,
    <T as std::str::FromStr>::Err: std::fmt::Debug,
{
    iter.next().expect(USAGE).parse().expect(USAGE)
}

impl Args {
    fn parse() -> Args {
        let mut args = Args::default();
        for raw_arg in std::env::args().skip(1) {
            let mut splits = raw_arg[2..].split('=');
            match splits.next().unwrap() {
                "threads" => args.threads = parse(&mut splits),
                "burn-in" => args.burn_in = true,
                "duration" => args.duration = parse(&mut splits),
                "key-len" => args.key_len = parse(&mut splits),
                "val-len" => args.val_len = parse(&mut splits),
                "get-prop" => args.get_prop = parse(&mut splits),
                "set-prop" => args.set_prop = parse(&mut splits),
                "del-prop" => args.del_prop = parse(&mut splits),
                "cas-prop" => args.cas_prop = parse(&mut splits),
                "scan-prop" => args.scan_prop = parse(&mut splits),
                "merge-prop" => args.merge_prop = parse(&mut splits),
                "entries" => args.entries = parse(&mut splits),
                "sequential" => args.sequential = true,
                "total-ops" => args.total_ops = Some(parse(&mut splits)),
                "flush-every" => args.flush_every = parse(&mut splits),
                "cache-mb" => args.cache_mb = parse(&mut splits),
                other => panic!("unknown option: {}, {}", other, USAGE),
            }
        }
        args
    }
}

fn report(shutdown: Arc<AtomicBool>) {
    let mut last = 0;
    while !shutdown.load(Ordering::Relaxed) {
        thread::sleep(std::time::Duration::from_secs(1));
        let total = TOTAL.load(Ordering::Acquire);

        println!(
            "did {} ops, {}mb RSS",
            (total - last).to_formatted_string(&Locale::en),
            rss() / (1024 * 1024)
        );

        last = total;
    }
}

fn concatenate_merge(
    _key: &[u8],              // the key being merged
    old_value: Option<&[u8]>, // the previous value, if one existed
    merged_bytes: &[u8],      // the new bytes being merged in
) -> Option<Vec<u8>> {
    // set the new value, return None to delete
    let mut ret = old_value.map(|ov| ov.to_vec()).unwrap_or_else(Vec::new);

    ret.extend_from_slice(merged_bytes);

    Some(ret)
}

fn run(args: Args, tree: Arc<sled::Db>, shutdown: Arc<AtomicBool>) {
    let get_max = args.get_prop;
    let set_max = get_max + args.set_prop;
    let del_max = set_max + args.del_prop;
    let cas_max = del_max + args.cas_prop;
    let merge_max = cas_max + args.merge_prop;
    let scan_max = merge_max + args.scan_prop;

    let keygen = |len| -> sled::IVec {
        static SEQ: AtomicUsize = AtomicUsize::new(0);
        let i = if args.sequential {
            SEQ.fetch_add(1, Ordering::Relaxed)
        } else {
            thread_rng().gen::<usize>()
        } % args.entries;

        let start = if len < 8 { 8 - len } else { 0 };

        let i_keygen = &i.to_be_bytes()[start..];

        i_keygen.iter().cycle().take(len).copied().collect()
    };

    let valgen = |len| -> sled::IVec {
        if len == 0 {
            return vec![].into();
        }

        let i: usize = thread_rng().gen::<usize>() % (len * 8);

        let i_keygen = i.to_be_bytes();

        i_keygen
            .iter()
            .skip_while(|v| **v == 0)
            .cycle()
            .take(len)
            .copied()
            .collect()
    };

    let mut rng = thread_rng();

    while !shutdown.load(Ordering::Relaxed) {
        let op = TOTAL.fetch_add(1, Ordering::Release);
        let key = keygen(args.key_len);
        let choice = rng.gen_range(0, scan_max + 1);

        match choice {
            v if v <= get_max => {
                tree.get_zero_copy(&key, |_| {}).unwrap();
            }
            v if v > get_max && v <= set_max => {
                let value = valgen(args.val_len);
                tree.insert(&key, value).unwrap();
            }
            v if v > set_max && v <= del_max => {
                tree.remove(&key).unwrap();
            }
            v if v > del_max && v <= cas_max => {
                let old = if rng.gen::<bool>() {
                    let value = valgen(args.val_len);
                    Some(value)
                } else {
                    None
                };

                let new = if rng.gen::<bool>() {
                    let value = valgen(args.val_len);
                    Some(value)
                } else {
                    None
                };

                if let Err(e) = tree.compare_and_swap(&key, old, new) {
                    panic!("operational error: {:?}", e);
                }
            }
            v if v > cas_max && v <= merge_max => {
                let value = valgen(args.val_len);
                tree.merge(&key, value).unwrap();
            }
            _ => {
                let iter = tree.range(key..).map(|res| res.unwrap());

                if op % 2 == 0 {
                    let _ = iter.take(rng.gen_range(0, 15)).collect::<Vec<_>>();
                } else {
                    let _ = iter
                        .rev()
                        .take(rng.gen_range(0, 15))
                        .collect::<Vec<_>>();
                }
            }
        }
    }
}

fn rss() -> usize {
    #[cfg(target_os = "linux")]
    {
        use std::io::prelude::*;
        use std::io::BufReader;

        let mut buf = String::new();
        let mut f =
            BufReader::new(std::fs::File::open("/proc/self/statm").unwrap());
        f.read_line(&mut buf).unwrap();
        let mut parts = buf.split_whitespace();
        let rss_pages = parts.nth(1).unwrap().parse::<usize>().unwrap();
        rss_pages * 4096
    }
    #[cfg(not(target_os = "linux"))]
    {
        0
    }
}

fn main() {
    #[cfg(feature = "logging")]
    setup_logger();

    #[cfg(feature = "dh")]
    let _dh = Dhat::start_heap_profiling();

    let args = Args::parse();

    let shutdown = Arc::new(AtomicBool::new(false));

    dbg!(args);

    let config = sled::Config::new()
        .cache_capacity(args.cache_mb * 1024 * 1024)
        .flush_every_ms(if args.flush_every == 0 {
            None
        } else {
            Some(args.flush_every)
        });

    let tree = Arc::new(config.open().unwrap());
    tree.set_merge_operator(concatenate_merge);

    let mut threads = vec![];

    let now = std::time::Instant::now();

    let n_threads = args.threads;

    for i in 0..=n_threads {
        let tree = tree.clone();
        let shutdown = shutdown.clone();

        let t = if i == 0 {
            thread::Builder::new()
                .name("reporter".into())
                .spawn(move || report(shutdown))
                .unwrap()
        } else {
            thread::spawn(move || run(args, tree, shutdown))
        };

        threads.push(t);
    }

    if let Some(ops) = args.total_ops {
        assert!(!args.burn_in, "don't set both --burn-in and --total-ops");
        while TOTAL.load(Ordering::Relaxed) < ops {
            thread::sleep(std::time::Duration::from_millis(50));
        }
        shutdown.store(true, Ordering::SeqCst);
    } else if !args.burn_in {
        thread::sleep(std::time::Duration::from_secs(args.duration));
        shutdown.store(true, Ordering::SeqCst);
    }

    for t in threads.into_iter() {
        t.join().unwrap();
    }
    let ops = TOTAL.load(Ordering::SeqCst);
    let time = now.elapsed().as_secs() as usize;

    println!(
        "did {} total ops in {} seconds. {} ops/s",
        ops.to_formatted_string(&Locale::en),
        time,
        ((ops * 1_000) / (time * 1_000)).to_formatted_string(&Locale::en)
    );

    #[cfg(feature = "metrics")]
    sled::print_profile();
}

#[cfg(feature = "logging")]
pub fn setup_logger() {
    use std::io::Write;

    color_backtrace::install();

    fn tn() -> String {
        std::thread::current().name().unwrap_or("unknown").to_owned()
    }

    let mut builder = env_logger::Builder::new();
    builder
        .format(|buf, record| {
            writeln!(
                buf,
                "{:05} {:25} {:10} {}",
                record.level(),
                tn(),
                record.module_path().unwrap().split("::").last().unwrap(),
                record.args()
            )
        })
        .filter(None, log::LevelFilter::Info);

    if let Ok(env) = std::env::var("RUST_LOG") {
        builder.parse_filters(&env);
    }

    let _r = builder.try_init();
}
<|endoftext|>


<commit_before>//
// Mnemosyne: a functional systems programming language.
// (c) 2015 Hawk Weisman
//
// Mnemosyne is released under the MIT License. Please refer to
// the LICENSE file at the top-level directory of this distribution
// or at https://github.com/hawkw/mnemosyne/.
//

use std::iter;
use std::fmt::Write;
use std::rc::Rc;

use ::forktable::ForkTable;

use ast::*;
use self::annotations::*;
use self::types::Type;

/// A symbol table is a `ForkTable` mapping `String`s to `Type` annotations.
///
/// This table should be forked upon entering a new scope.
pub type SymbolTable<'a> = ForkTable<'a, String, SymbolAnnotation<'a>>;

#[macro_use]
macro_rules! indent {
    ($to:expr) => ( iter::repeat('\t')
                        .take($to)
                        .collect::<String>() )
}

pub mod ast;
pub mod types;
pub mod annotations;

impl<'a> AnnotateTypes<'a> for Unscoped<'a, Form<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self> {
        unimplemented!() //TODO: implement
    }
}

impl<'a> AnnotateTypes<'a> for Unscoped<'a, DefForm<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self>{
        unimplemented!() //TODO: implement
    }
}

#[derive(Clone,Debug,PartialEq)]
pub struct SymbolAnnotation<'a> {
    /// The type of the symbol
    pub ty: Type,
    /// An optional proven value for the symbol.
    ///
    /// This should be defined iff the symbol signifies a constant value
    /// or a constant expression, or if we were able to prove that the value
    /// remains constant within the current scope.
    pub proven_value: Option<Rc<Expr<'a, ScopedState>>>
}
<commit_msg>SymbolAnnotations can now be for type defs too<commit_after>//
// Mnemosyne: a functional systems programming language.
// (c) 2015 Hawk Weisman
//
// Mnemosyne is released under the MIT License. Please refer to
// the LICENSE file at the top-level directory of this distribution
// or at https://github.com/hawkw/mnemosyne/.
//

use std::iter;
use std::fmt::Write;
use std::rc::Rc;

use ::forktable::ForkTable;

use ast::*;
use self::annotations::*;
use self::types::Type;

/// A symbol table is a `ForkTable` mapping `String`s to `Type` annotations.
///
/// This table should be forked upon entering a new scope.
pub type SymbolTable<'a> = ForkTable<'a, String, SymbolAnnotation<'a>>;

#[macro_use]
macro_rules! indent {
    ($to:expr) => ( iter::repeat('\t')
                        .take($to)
                        .collect::<String>() )
}

pub mod ast;
pub mod types;
pub mod annotations;

impl<'a> AnnotateTypes<'a> for Unscoped<'a, Form<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self> {
        unimplemented!() //TODO: implement
    }
}

impl<'a> AnnotateTypes<'a> for Unscoped<'a, DefForm<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self>{
        unimplemented!() //TODO: implement
    }
}

#[derive(Clone,Debug,PartialEq)]
pub enum SymbolAnnotation<'a> {
    TypeDef(Type)
  , Value { /// The type of the symbol
            ty: Type,
            /// An optional proven value for the symbol.
            ///
            /// This should be defined iff the symbol signifies a constant value
            /// or a constant expression, or if we were able to prove that the value
            /// remains constant within the current scope.
            proven_value: Option<Rc<Expr<'a, ScopedState>>>
          }
}
<|endoftext|>


<commit_before>/**
 * Platform specific functions for ash
 */
use std::os;
use std::env;

#[cfg(windows)]
pub fn get_username() -> String {
  match env::var_os("USERNAME") {
    Some(e) => e,
    None => "?".to_string()
  }
}

#[cfg(unix)]
pub fn get_username() -> String {
  match os::getenv("USER") {
    Some(e) => e,
    None => "?".to_string()
  }
}

//I don't know how to get this yet
pub fn get_hostname() -> String {
  "hostname".to_string()
}
<commit_msg>Properly fixed building<commit_after>/**
 * Platform specific functions for ash
 */
use std::env;

#[cfg(windows)]
pub fn get_username() -> String {
  match env::var_os("USERNAME") {
    Some(e) => e.into_string().unwrap(),
    None => "?".to_string()
  }
}

#[cfg(unix)]
pub fn get_username() -> String {
  match env::var_os("USER") {
    Some(e) => e.into_string().unwrap(),
    None => "?".to_string()
  }
}

//I don't know how to get this yet
pub fn get_hostname() -> String {
  "hostname".to_string()
}
<|endoftext|>


<commit_before>// Copyright (c) 2021 Anatoly Ikorsky
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use std::{convert::TryFrom, io};

use crate::{
    binlog::{decimal, jsonb, jsondiff::JsonDiff, misc::*},
    constants::{ColumnFlags, ColumnType},
    io::ParseBuf,
    misc::raw::int::*,
    proto::MyDeserialize,
    value::Value::{self, *},
};

/// Value of a binlog event.
#[derive(Debug, Clone, PartialEq)]
pub enum BinlogValue<'a> {
    /// MySql value.
    Value(Value),
    /// JSONB value.
    Jsonb(jsonb::Value<'a>),
    /// Value of a partial JSON modification event.
    JsonDiff(Vec<JsonDiff<'a>>),
}

impl<'a> BinlogValue<'a> {
    /// Returns a `'static` version of `self`.
    pub fn into_owned(self) -> BinlogValue<'static> {
        match self {
            BinlogValue::Value(x) => BinlogValue::Value(x),
            BinlogValue::Jsonb(x) => BinlogValue::Jsonb(x.into_owned()),
            BinlogValue::JsonDiff(x) => {
                BinlogValue::JsonDiff(x.into_iter().map(|x| x.into_owned()).collect())
            }
        }
    }
}

impl<'de> MyDeserialize<'de> for BinlogValue<'de> {
    const SIZE: Option<usize> = None;
    /// <col_type, col_meta, is_unsigned, is_partial>
    type Ctx = (ColumnType, &'de [u8], bool, bool);

    fn deserialize(
        (mut col_type, col_meta, is_unsigned, is_partial): Self::Ctx,
        buf: &mut ParseBuf<'de>,
    ) -> io::Result<Self> {
        use ColumnType::*;

        let mut length = 0_usize;

        if col_type == MYSQL_TYPE_TYPED_ARRAY {
            let type_byte = col_meta[0];
            col_type = ColumnType::try_from(type_byte).unwrap_or(col_type);
        }

        if col_type == MYSQL_TYPE_STRING {
            if col_meta[0] >= 1 {
                let byte0 = col_meta[0] as usize;
                let byte1 = col_meta[1] as usize;

                if (byte0 & 0x30) != 0x30 {
                    // a long CHAR() field: see #37426
                    length = byte1 | (((byte0 & 0x30) ^ 0x30) << 4);
                    col_type = ColumnType::try_from(byte0 as u8 | 0x30).unwrap_or(col_type);
                } else {
                    length = byte1;
                }
            } else {
                length = (ParseBuf(col_meta)).eat_u16_le() as usize;
            }
        }

        match col_type {
            MYSQL_TYPE_TINY | MYSQL_TYPE_SHORT | MYSQL_TYPE_LONG | MYSQL_TYPE_LONGLONG
            | MYSQL_TYPE_FLOAT | MYSQL_TYPE_DOUBLE => {
                let mut flags = ColumnFlags::empty();
                flags.set(ColumnFlags::UNSIGNED_FLAG, is_unsigned);
                Value::deserialize_bin((col_type, flags), &mut *buf).map(BinlogValue::Value)
            }
            MYSQL_TYPE_TIMESTAMP => {
                let val: RawInt<LeU32> = buf.parse(())?;
                Ok(BinlogValue::Value(Int(*val as i64)))
            }
            MYSQL_TYPE_INT24 => {
                if is_unsigned {
                    let val: RawInt<LeU24> = buf.parse(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                } else {
                    let val: RawInt<LeI24> = buf.parse(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
            }
            MYSQL_TYPE_TIME => {
                let tmp: RawInt<LeU24> = buf.parse(())?;
                let h = *tmp / 10000;
                let m = (*tmp % 10000) / 100;
                let s = *tmp % 100;
                Ok(BinlogValue::Value(Time(
                    false, 0, h as u8, m as u8, s as u8, 0,
                )))
            }
            MYSQL_TYPE_DATETIME => {
                // read YYYYMMDDHHMMSS representaion
                let raw: RawInt<LeU64> = buf.parse(())?;
                let d_part = *raw / 1_000_000;
                let t_part = *raw % 1_000_000;
                Ok(BinlogValue::Value(Date(
                    (d_part / 10000) as u16,
                    ((d_part % 10000) / 100) as u8,
                    (d_part % 100) as u8,
                    (t_part / 10000) as u8,
                    ((t_part % 10000) / 100) as u8,
                    (t_part % 100) as u8,
                    0,
                )))
            }
            MYSQL_TYPE_YEAR => {
                let y = *buf.parse::<RawInt<u8>>(())? as i32;
                Ok(BinlogValue::Value(Bytes(
                    (1900 + y).to_string().into_bytes(),
                )))
            }
            MYSQL_TYPE_NEWDATE => {
                let tmp = *buf.parse::<RawInt<LeU24>>(())?;
                let d = tmp & 31;
                let m = (tmp >> 5) & 15;
                let y = tmp >> 9;
                Ok(BinlogValue::Value(Date(
                    y as u16, m as u8, d as u8, 0, 0, 0, 0,
                )))
            }
            MYSQL_TYPE_BIT => {
                let nbits = col_meta[0] as usize * 8 + (col_meta[1] as usize);
                let nbytes = (nbits + 7) / 8;
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_TIMESTAMP2 => {
                let dec = col_meta[0];
                let (sec, usec) = my_timestamp_from_binary(&mut *buf, dec)?;
                if usec == 0 {
                    Ok(BinlogValue::Value(Bytes(sec.to_string().into_bytes())))
                } else {
                    Ok(BinlogValue::Value(Bytes(
                        format!("{}.{:06}", sec, usec).into_bytes(),
                    )))
                }
            }
            MYSQL_TYPE_DATETIME2 => {
                let dec = col_meta[0];
                my_datetime_packed_from_binary(&mut *buf, dec as u32)
                    .map(datetime_from_packed)
                    .map(BinlogValue::Value)
            }
            MYSQL_TYPE_TIME2 => {
                let dec = col_meta[0];
                my_time_packed_from_binary(&mut *buf, dec as u32)
                    .map(time_from_packed)
                    .map(BinlogValue::Value)
            }
            MYSQL_TYPE_JSON => {
                length = *buf.parse::<RawInt<LeU32>>(())? as usize;
                let mut json_value_buf: ParseBuf = buf.parse(length)?;
                if is_partial {
                    let mut diffs = Vec::new();
                    while !json_value_buf.is_empty() {
                        diffs.push(json_value_buf.parse(())?);
                    }
                    Ok(BinlogValue::JsonDiff(diffs))
                } else {
                    Ok(BinlogValue::Jsonb(json_value_buf.parse(())?))
                }
            }
            MYSQL_TYPE_NEWDECIMAL => {
                // precision is the maximum number of decimal digits
                let precision = col_meta[0] as usize;
                // scale (aka decimals) is the number of decimal digits after the point
                let scale = col_meta[1] as usize;

                let dec = decimal::Decimal::read_bin(&mut *buf, precision, scale, false)?;

                Ok(BinlogValue::Value(Bytes(dec.to_string().into_bytes())))
            }
            MYSQL_TYPE_ENUM => match col_meta[1] {
                1 => {
                    let val = buf.parse::<RawInt<u8>>(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
                2 => {
                    let val = buf.parse::<RawInt<LeU16>>(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
                _ => Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown ENUM")),
            },
            MYSQL_TYPE_SET => {
                let nbytes = col_meta[1] as usize;
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_TINY_BLOB
            | MYSQL_TYPE_MEDIUM_BLOB
            | MYSQL_TYPE_LONG_BLOB
            | MYSQL_TYPE_BLOB
            | MYSQL_TYPE_GEOMETRY => {
                let nbytes = match col_meta[0] {
                    1 => *buf.parse::<RawInt<u8>>(())? as usize,
                    2 => *buf.parse::<RawInt<LeU16>>(())? as usize,
                    3 => *buf.parse::<RawInt<LeU24>>(())? as usize,
                    4 => *buf.parse::<RawInt<LeU32>>(())? as usize,
                    _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown BLOB")),
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_VARCHAR | MYSQL_TYPE_VAR_STRING => {
                let type_len = (col_meta[0] as u16 | ((col_meta[1] as u16) << 8)) as usize;
                let nbytes = if type_len < 256 {
                    *buf.parse::<RawInt<u8>>(())? as usize
                } else {
                    *buf.parse::<RawInt<LeU16>>(())? as usize
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_STRING => {
                let nbytes = if length < 256 {
                    *buf.parse::<RawInt<u8>>(())? as usize
                } else {
                    *buf.parse::<RawInt<LeU16>>(())? as usize
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            _ => Err(io::Error::new(
                io::ErrorKind::InvalidData,
                "Don't know how to handle column",
            )),
        }
    }
}
<commit_msg>fix: get nbits from "MYSQL_TYPE_BIT" column meta<commit_after>// Copyright (c) 2021 Anatoly Ikorsky
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use std::{convert::TryFrom, io};

use crate::{
    binlog::{decimal, jsonb, jsondiff::JsonDiff, misc::*},
    constants::{ColumnFlags, ColumnType},
    io::ParseBuf,
    misc::raw::int::*,
    proto::MyDeserialize,
    value::Value::{self, *},
};

/// Value of a binlog event.
#[derive(Debug, Clone, PartialEq)]
pub enum BinlogValue<'a> {
    /// MySql value.
    Value(Value),
    /// JSONB value.
    Jsonb(jsonb::Value<'a>),
    /// Value of a partial JSON modification event.
    JsonDiff(Vec<JsonDiff<'a>>),
}

impl<'a> BinlogValue<'a> {
    /// Returns a `'static` version of `self`.
    pub fn into_owned(self) -> BinlogValue<'static> {
        match self {
            BinlogValue::Value(x) => BinlogValue::Value(x),
            BinlogValue::Jsonb(x) => BinlogValue::Jsonb(x.into_owned()),
            BinlogValue::JsonDiff(x) => {
                BinlogValue::JsonDiff(x.into_iter().map(|x| x.into_owned()).collect())
            }
        }
    }
}

impl<'de> MyDeserialize<'de> for BinlogValue<'de> {
    const SIZE: Option<usize> = None;
    /// <col_type, col_meta, is_unsigned, is_partial>
    type Ctx = (ColumnType, &'de [u8], bool, bool);

    fn deserialize(
        (mut col_type, col_meta, is_unsigned, is_partial): Self::Ctx,
        buf: &mut ParseBuf<'de>,
    ) -> io::Result<Self> {
        use ColumnType::*;

        let mut length = 0_usize;

        if col_type == MYSQL_TYPE_TYPED_ARRAY {
            let type_byte = col_meta[0];
            col_type = ColumnType::try_from(type_byte).unwrap_or(col_type);
        }

        if col_type == MYSQL_TYPE_STRING {
            if col_meta[0] >= 1 {
                let byte0 = col_meta[0] as usize;
                let byte1 = col_meta[1] as usize;

                if (byte0 & 0x30) != 0x30 {
                    // a long CHAR() field: see #37426
                    length = byte1 | (((byte0 & 0x30) ^ 0x30) << 4);
                    col_type = ColumnType::try_from(byte0 as u8 | 0x30).unwrap_or(col_type);
                } else {
                    length = byte1;
                }
            } else {
                length = (ParseBuf(col_meta)).eat_u16_le() as usize;
            }
        }

        match col_type {
            MYSQL_TYPE_TINY | MYSQL_TYPE_SHORT | MYSQL_TYPE_LONG | MYSQL_TYPE_LONGLONG
            | MYSQL_TYPE_FLOAT | MYSQL_TYPE_DOUBLE => {
                let mut flags = ColumnFlags::empty();
                flags.set(ColumnFlags::UNSIGNED_FLAG, is_unsigned);
                Value::deserialize_bin((col_type, flags), &mut *buf).map(BinlogValue::Value)
            }
            MYSQL_TYPE_TIMESTAMP => {
                let val: RawInt<LeU32> = buf.parse(())?;
                Ok(BinlogValue::Value(Int(*val as i64)))
            }
            MYSQL_TYPE_INT24 => {
                if is_unsigned {
                    let val: RawInt<LeU24> = buf.parse(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                } else {
                    let val: RawInt<LeI24> = buf.parse(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
            }
            MYSQL_TYPE_TIME => {
                let tmp: RawInt<LeU24> = buf.parse(())?;
                let h = *tmp / 10000;
                let m = (*tmp % 10000) / 100;
                let s = *tmp % 100;
                Ok(BinlogValue::Value(Time(
                    false, 0, h as u8, m as u8, s as u8, 0,
                )))
            }
            MYSQL_TYPE_DATETIME => {
                // read YYYYMMDDHHMMSS representaion
                let raw: RawInt<LeU64> = buf.parse(())?;
                let d_part = *raw / 1_000_000;
                let t_part = *raw % 1_000_000;
                Ok(BinlogValue::Value(Date(
                    (d_part / 10000) as u16,
                    ((d_part % 10000) / 100) as u8,
                    (d_part % 100) as u8,
                    (t_part / 10000) as u8,
                    ((t_part % 10000) / 100) as u8,
                    (t_part % 100) as u8,
                    0,
                )))
            }
            MYSQL_TYPE_YEAR => {
                let y = *buf.parse::<RawInt<u8>>(())? as i32;
                Ok(BinlogValue::Value(Bytes(
                    (1900 + y).to_string().into_bytes(),
                )))
            }
            MYSQL_TYPE_NEWDATE => {
                let tmp = *buf.parse::<RawInt<LeU24>>(())?;
                let d = tmp & 31;
                let m = (tmp >> 5) & 15;
                let y = tmp >> 9;
                Ok(BinlogValue::Value(Date(
                    y as u16, m as u8, d as u8, 0, 0, 0, 0,
                )))
            }
            MYSQL_TYPE_BIT => {
                let nbits = col_meta[1] as usize * 8 + (col_meta[0] as usize);
                let nbytes = (nbits + 7) / 8;
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_TIMESTAMP2 => {
                let dec = col_meta[0];
                let (sec, usec) = my_timestamp_from_binary(&mut *buf, dec)?;
                if usec == 0 {
                    Ok(BinlogValue::Value(Bytes(sec.to_string().into_bytes())))
                } else {
                    Ok(BinlogValue::Value(Bytes(
                        format!("{}.{:06}", sec, usec).into_bytes(),
                    )))
                }
            }
            MYSQL_TYPE_DATETIME2 => {
                let dec = col_meta[0];
                my_datetime_packed_from_binary(&mut *buf, dec as u32)
                    .map(datetime_from_packed)
                    .map(BinlogValue::Value)
            }
            MYSQL_TYPE_TIME2 => {
                let dec = col_meta[0];
                my_time_packed_from_binary(&mut *buf, dec as u32)
                    .map(time_from_packed)
                    .map(BinlogValue::Value)
            }
            MYSQL_TYPE_JSON => {
                length = *buf.parse::<RawInt<LeU32>>(())? as usize;
                let mut json_value_buf: ParseBuf = buf.parse(length)?;
                if is_partial {
                    let mut diffs = Vec::new();
                    while !json_value_buf.is_empty() {
                        diffs.push(json_value_buf.parse(())?);
                    }
                    Ok(BinlogValue::JsonDiff(diffs))
                } else {
                    Ok(BinlogValue::Jsonb(json_value_buf.parse(())?))
                }
            }
            MYSQL_TYPE_NEWDECIMAL => {
                // precision is the maximum number of decimal digits
                let precision = col_meta[0] as usize;
                // scale (aka decimals) is the number of decimal digits after the point
                let scale = col_meta[1] as usize;

                let dec = decimal::Decimal::read_bin(&mut *buf, precision, scale, false)?;

                Ok(BinlogValue::Value(Bytes(dec.to_string().into_bytes())))
            }
            MYSQL_TYPE_ENUM => match col_meta[1] {
                1 => {
                    let val = buf.parse::<RawInt<u8>>(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
                2 => {
                    let val = buf.parse::<RawInt<LeU16>>(())?;
                    Ok(BinlogValue::Value(Int(*val as i64)))
                }
                _ => Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown ENUM")),
            },
            MYSQL_TYPE_SET => {
                let nbytes = col_meta[1] as usize;
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_TINY_BLOB
            | MYSQL_TYPE_MEDIUM_BLOB
            | MYSQL_TYPE_LONG_BLOB
            | MYSQL_TYPE_BLOB
            | MYSQL_TYPE_GEOMETRY => {
                let nbytes = match col_meta[0] {
                    1 => *buf.parse::<RawInt<u8>>(())? as usize,
                    2 => *buf.parse::<RawInt<LeU16>>(())? as usize,
                    3 => *buf.parse::<RawInt<LeU24>>(())? as usize,
                    4 => *buf.parse::<RawInt<LeU32>>(())? as usize,
                    _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "Unknown BLOB")),
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_VARCHAR | MYSQL_TYPE_VAR_STRING => {
                let type_len = (col_meta[0] as u16 | ((col_meta[1] as u16) << 8)) as usize;
                let nbytes = if type_len < 256 {
                    *buf.parse::<RawInt<u8>>(())? as usize
                } else {
                    *buf.parse::<RawInt<LeU16>>(())? as usize
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            MYSQL_TYPE_STRING => {
                let nbytes = if length < 256 {
                    *buf.parse::<RawInt<u8>>(())? as usize
                } else {
                    *buf.parse::<RawInt<LeU16>>(())? as usize
                };
                let bytes: &[u8] = buf.parse(nbytes)?;
                Ok(BinlogValue::Value(Bytes(bytes.into())))
            }
            _ => Err(io::Error::new(
                io::ErrorKind::InvalidData,
                "Don't know how to handle column",
            )),
        }
    }
}
<|endoftext|>